#8371 Migrate jenkins builds - due Aug 15

unreleased
closed
nobody
asf (28)
General
nobody
2020-08-14
2020-07-19
No

For details see https://lists.apache.org/thread.html/re974eed417a1bc294694701d5c91b4bf92689fcf32a4c91f169be87d%40%3Cbuilds.apache.org%3E

Basically we need to move these:

Over to https://ci-builds.apache.org/ Shouldn't be too hard since their configs are mostly inline bash scripts, no fancy plugin usage etc.

Discussion

  • Dave Brondsema

    Dave Brondsema - 2020-08-14

    current config is

    Allura

    • Discard old builds - 5
    • git https://gitbox.apache.org/repos/asf/allura.git/
    • poll H/5 * * * *
    • Delete workspace before build starts
    • Abort the build if it's stuck - 50 min
     1
     2
     3
     4
     5
     6
     7
     8
     9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    #!/bin/bash
    # need to specify the shell, else Jenkins will quit after the first non-zero exit
    
    echo -n 'cpu count: '; grep -c processor /proc/cpuinfo 
    python -V
    which python
    which python27
    which python2.7
    echo path: $PATH
    echo workspace: $WORKSPACE
    echo jenkins_home: $JENKINS_HOME
    echo home: $HOME
    echo pwd: `pwd`
    echo REBUILD_VENV: $REBUILD_VENV
    echo LANG: $LANG
    env
    svn --version
    echo pip: `pip --version`
    echo virtualenv: `virtualenv --version`
    echo setuptools: `easy_install --version`
    echo npm: `npm --version`
    #pip freeze
    
    #if [ $REBUILD_VENV == "true" ]; then
       rm -rf ".allura-venv"
    #fi
    
    # hosts have a new enough virtualenv/pip/setuptools we don't need to bootstrap from source:
    #rm -rf virtualenv-*
    #if [ ! -d ".allura-venv" ]; then
    #    if [ ! -f virtualenv-*/virtualenv.py ]; then
    #        #wget --no-check-certificate https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz || exit
    #        wget https://pypi.org/packages/source/v/virtualenv/virtualenv-16.0.0.tar.gz || exit
    #        tar xvfz virtualenv-*.tar.gz
    #    fi
    #    python virtualenv-*/virtualenv.py .allura-venv
    #fi
    
    virtualenv .allura-venv
    
    . .allura-venv/bin/activate
    
    git clean -f -x  # remove test.log, nosetest.xml etc (don't use -d since it'd remove our venv dir)
    
    echo venv-pip: `pip --version`
    
    # upgrading beforehand used to be necessary, but not any more
    #pip install 'pip>=9'
    #pip install 'setuptools>=11.3'
    #python -m pip install --upgrade pip setuptools wheel
    
    # retry a few times
    MAIN_PIP="pip install -r requirements.txt"
    $MAIN_PIP || (echo "retrying pip install after short sleep"; sleep 2; $MAIN_PIP) || exit
    
    pip install pycodestyle pyflakes coverage nose-xunitmp || exit
    
    ln -s /usr/lib/python2.7/dist-packages/pysvn $VIRTUAL_ENV/lib/python2.7/site-packages/
    
    # use "Allura* Forge* scripts" instead of "." so that .allura-venv doesn't get checked too (and '.' gives './' prefixed results which don't work out)
    pyflakes Allura* Forge* scripts | awk -F\: '{printf "%s:%s: [E]%s\n", $1, $2, $3}' > pyflakes.txt
    pycodestyle Allura* Forge* scripts > pep8.txt
    
    ./rebuild-all.bash
    
    # fresh start with npm
    #rm -rf node_modules
    npm install || (echo "retrying npm install"; sleep 5; npm install) || (echo "retrying npm install"; sleep 5; npm install) || (echo "retrying npm install"; sleep 5; npm install) || (echo "retrying npm install"; sleep 5; npm install) || (echo "retrying npm install"; sleep 5; npm install) 
    
    # TODO: ALLURA_VALIDATION=all
    LANG=en_US.UTF-8 ./run_tests --with-xunitmp # --with-coverage --cover-erase
    retcode=$?
    
    #find . -name .coverage -maxdepth 2 | while read coveragefile; do pushd `dirname $coveragefile`; coverage xml --include='forge*,allura*'; popd; done;
    
    rm -f call_count.csv
    ./scripts/perf/call_count.py --data-file call_count.csv
    
    
    # debugging
    echo npm: `npm --version`
    echo hostname: `hostname --short`
    echo NODE_NAME: $NODE_NAME
    
    exit $retcode
    
    • post-build: Plot "Call Counts of 1 "typical" request" from call_count.csv
    • post-build: Cobertura from **/coverage.xml
    • post-build: junit from **/nosetests.xml
    • post-build: email dev@ every unstable buld

    Allura-licensing-check

    • Discard old builds - 10
    • git https://gitbox.apache.org/repos/asf/allura.git/
    • build after Allura
    • Delete workspace before build starts
    • Abort the build if it's stuck - 15 min
     1
     2
     3
     4
     5
     6
     7
     8
     9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    #!/bin/bash -x
    
    # clean start
    #rm -rf tmp
    
    mkdir -p tmp
    cd tmp
    
    if [ ! -e apache-ant-*/bin/ant ]; then
        rm -rf apache-ant-*  # empty/partial dir may exist
        # ant 1.10.x requires Java8 runtime
        # download from archive since mirrors delete older releases when new ones come out
        wget -nv http://archive.apache.org/dist/ant/binaries/apache-ant-1.9.9-bin.tar.gz || exit
        tar xzf apache-ant-*.tar.gz
        rm -rf apache-ant-*.tar.gz
    fi
    
    if [ ! -e apache-rat-*/*.jar ]; then
        rm -rf apache-rat-*  # empty/partial dir may exist
        wget -nv http://apache.mirrors.tds.net/creadur/apache-rat-0.13/apache-rat-0.13-bin.tar.gz || exit
        tar xzf apache-rat-*.tar.gz
        rm -rf apache-rat-*.tar.gz
    fi
    
    ANT=$WORKSPACE/tmp/apache-ant-*/bin/ant
    OUTFILE=$WORKSPACE/tmp/apache_rat_results.txt
    
    cd $WORKSPACE/scripts/src-license-check/
    $ANT -lib $WORKSPACE/tmp/apache-rat-*/ > $OUTFILE
    
    cat $OUTFILE
    
    # exit OK if we found no unknowns
    # TODO: better check.  I think this will pass on known license that are incompatible
    grep ' 0 Unknown Licenses' $OUTFILE -q
    
    • post-build: email dev@ every unstable buld
     
  • Dave Brondsema

    Dave Brondsema - 2020-08-14
    • status: open --> closed
     

Log in to post a comment.