Newer
Older
#!/usr/bin/env bash
set -e
###############################################################################
#
# edx-all-tests.sh
#
# Execute all tests for edx-platform.
#
# This script can be called from a Jenkins
# multiconfiguration job that defines these environment
# variables:
#
# `TEST_SUITE` defines which kind of test to run.
# Possible values are:
#
# - "quality": Run the quality (pep8/pylint) checks
# - "unit": Run the JavaScript and Python unit tests
# (also tests building the Sphinx documentation,
# because we couldn't think of a better place to put it)
# - "lms-acceptance": Run the acceptance (Selenium) tests for the LMS
# - "cms-acceptance": Run the acceptance (Selenium) tests for Studio
# - "bok-choy": Run acceptance tests that use the bok-choy framework
#
# `SHARD` is a number (1, 2, or 3) indicating which subset of the tests
# to build. Currently, "lms-acceptance" and "bok-choy" each have two
# shards (1 and 2), "cms-acceptance" has three shards (1, 2, and 3),
# and all the other test suites have one shard.
#
# For the "bok-choy", the tests are put into shard groups using the nose
# 'attr' decorator (e.g. "@attr('shard_1')"). Currently, anything with
# the 'shard_1' attribute will run in the first shard. All other bok-choy
# tests will run in shard 2.
#
# For the lettuce acceptance tests, ("lms-" and "cms-acceptance") they
# are decorated with "@shard_{}" (e.g. @shard_1 for the first shard).
# The lettuce tests must have a shard specified to be run in jenkins,
# as there is no shard that runs unspecified tests.
#
# Jenkins configuration:
#
# - The edx-platform git repository is checked out by the Jenkins git plugin.
#
# - Jenkins logs in as user "jenkins"
#
# - The Jenkins file system root is "/home/jenkins"
#
# - An init script creates a virtualenv at "/home/jenkins/edx-venv"
# with some requirements pre-installed (such as scipy)
#
# Jenkins worker setup:
# See the edx/configuration repo for Jenkins worker provisioning scripts.
# The provisioning scripts install requirements that this script depends on!
#
###############################################################################
# Violations thresholds for failing the build
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
source $HOME/jenkins_env
# Clean up previous builds
git clean -qxfd
# Clear the mongo database
# Note that this prevents us from running jobs in parallel on a single worker.
mongo --quiet --eval 'db.getMongo().getDBNames().forEach(function(i){db.getSiblingDB(i).dropDatabase()})'
# Ensure we have fetched origin/master
# Some of the reporting tools compare the checked out branch to origin/master;
# depending on how the GitHub plugin refspec is configured, this may
# not already be fetched.
git fetch origin master:refs/remotes/origin/master
# Reset the jenkins worker's ruby environment back to
# the state it was in when the instance was spun up.
if [ -e $HOME/edx-rbenv_clean.tar.gz ]; then
rm -rf $HOME/.rbenv
tar -C $HOME -xf $HOME/edx-rbenv_clean.tar.gz
fi
# Bootstrap Ruby requirements so we can run the tests
bundle install
# Ensure the Ruby environment contains no stray gems
bundle clean --force
# Reset the jenkins worker's virtualenv back to the
# state it was in when the instance was spun up.
if [ -e $HOME/edx-venv_clean.tar.gz ]; then
rm -rf $HOME/edx-venv
tar -C $HOME -xf $HOME/edx-venv_clean.tar.gz
fi
# Activate the Python virtualenv
source $HOME/edx-venv/bin/activate
# If the environment variable 'SHARD' is not set, default to 'all'.
# This could happen if you are trying to use this script from
# jenkins and do not define 'SHARD' in your multi-config project.
# Note that you will still need to pass a value for 'TEST_SUITE'
# or else no tests will be executed.
SHARD=${SHARD:="all"}
case "$TEST_SUITE" in
"quality")
paver find_fixme > fixme.log || { cat fixme.log; EXIT=1; }
paver run_pep8 -l $PEP8_THRESHOLD > pep8.log || { cat pep8.log; EXIT=1; }
paver run_pylint -l $PYLINT_THRESHOLD > pylint.log || { cat pylint.log; EXIT=1; }
# Run quality task. Pass in the 'fail-under' percentage to diff-quality
paver run_quality -p 100
# Need to create an empty test result so the post-build
# action doesn't fail the build.
mkdir -p reports
cat > reports/quality.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="quality" tests="1" errors="0" failures="0" skip="0">
<testcase classname="quality" name="quality" time="0.604"></testcase>
</testsuite>
END
case "$SHARD" in
"lms")
paver test_system -s lms
paver coverage
;;
"other")
paver test_system -s cms
paver test_js --coverage --skip_clean
paver test_lib --skip_clean
paver coverage
;;
*)
paver test
paver coverage
;;
esac
;;
"lms-acceptance")
case "$SHARD" in
"all")
paver test_acceptance -s lms --extra_args="-v 3"
;;
"2")
mkdir -p reports
mkdir -p reports/acceptance
cat > reports/acceptance/xunit.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
<testcase classname="lettuce.tests" name="shard_placeholder" time="0.001"></testcase>
</testsuite>
END
;;
paver test_acceptance -s lms --extra_args="-v 3"
;;
"cms-acceptance")
case "$SHARD" in
paver test_acceptance -s cms --extra_args="-v 3"
;;
"2"|"3")
mkdir -p reports/acceptance
cat > reports/acceptance/xunit.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
<testcase classname="lettuce.tests" name="shard_placeholder" time="0.001"></testcase>
</testsuite>
END
"all")
paver test_bokchoy
paver bokchoy_coverage
;;
paver test_bokchoy --extra_args="-a shard_1"
paver bokchoy_coverage
paver test_bokchoy --extra_args="-a 'shard_2'"
paver bokchoy_coverage
;;
"3")
paver test_bokchoy --extra_args="-a 'shard_3'"
paver bokchoy_coverage
;;
"4")
paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False"
paver bokchoy_coverage
;;
# Default case because if we later define another bok-choy shard on Jenkins
# (e.g. Shard 5) in the multi-config project and expand this file
# with an additional case condition, old branches without that commit
# would not execute any tests on the worker assigned to that shard
# and thus their build would fail.
# This way they will just report 1 test executed and passed.
*)
# Need to create an empty test result so the post-build
# action doesn't fail the build.
# May be unnecessary if we changed the "Skip if there are no test files"
# option to True in the jenkins job definitions.
cat > reports/bok_choy/xunit.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
<testcase classname="acceptance.tests" name="shard_placeholder" time="0.001"></testcase>
</testsuite>
END
;;