diff --git a/Jenkinsfile b/Jenkinsfile
index c593b2e9162c7ec96d8176ec888b4bb51a470178..f0f69cae08964c6947726dc5ec6b47d1f825c972 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,42 +1,46 @@
def runPythonTests() {
- ansiColor('gnome-terminal') {
- sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
- checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${sha1}']],
- doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
- userRemoteConfigs: [[credentialsId: 'jenkins-worker',
- refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
- url: 'git@github.com:edx/edx-platform.git']]]
- console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
- dir('stdout') {
- writeFile file: "${TEST_SUITE}-${SHARD}-stdout.log", text: console_output
- }
- stash includes: 'reports/**/*coverage*', name: "${TEST_SUITE}-${SHARD}-reports"
+ sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) {
+ checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
+ doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
+ userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+ refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
+ url: 'git@github.com:edx/edx-platform.git']]]
+ console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
+ dir('stdout') {
+ writeFile file: "${TEST_SUITE}-stdout.log", text: console_output
}
+ stash includes: 'reports/**/*coverage*', name: "${TEST_SUITE}-reports"
}
}
-def savePythonTestArtifacts() {
+def pythonTestCleanup() {
archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,stdout/*.log,*.log'
junit '**/nosetests.xml'
+ sh '''source $HOME/edx-venv/bin/activate
+ bash scripts/xdist/terminate_xdist_nodes.sh'''
}
pipeline {
-
agent { label "coverage-worker" }
-
options {
timestamps()
- timeout(75)
+ timeout(60)
+ }
+ environment {
+ XDIST_CONTAINER_SUBNET = credentials('XDIST_CONTAINER_SUBNET')
+ XDIST_CONTAINER_SECURITY_GROUP = credentials('XDIST_CONTAINER_SECURITY_GROUP')
+ XDIST_CONTAINER_TASK_NAME = "jenkins-worker-task"
+ XDIST_GIT_BRANCH = "${ghprbActualCommit}"
}
-
stages {
stage('Run Tests') {
parallel {
- stage('lms-unit-1') {
+ stage("lms-unit") {
agent { label "jenkins-worker" }
environment {
- SHARD = 1
- TEST_SUITE = 'lms-unit'
+ TEST_SUITE = "lms-unit"
+ XDIST_NUM_TASKS = 10
+ XDIST_REMOTE_NUM_PROCESSES = 2
}
steps {
script {
@@ -46,35 +50,17 @@ pipeline {
post {
always {
script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('lms-unit-2') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 2
- TEST_SUITE = 'lms-unit'
- }
- steps{
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
+ pythonTestCleanup()
}
}
}
}
- stage('lms-unit-3') {
+ stage("cms-unit") {
agent { label "jenkins-worker" }
environment {
- SHARD = 3
- TEST_SUITE = 'lms-unit'
+ TEST_SUITE = "cms-unit"
+ XDIST_NUM_TASKS = 4
+ XDIST_REMOTE_NUM_PROCESSES = 2
}
steps {
script {
@@ -84,16 +70,17 @@ pipeline {
post {
always {
script {
- savePythonTestArtifacts()
+ pythonTestCleanup()
}
}
}
}
- stage('lms-unit-4') {
+ stage("commonlib-unit") {
agent { label "jenkins-worker" }
environment {
- SHARD = 4
- TEST_SUITE = 'lms-unit'
+ TEST_SUITE = "commonlib-unit"
+ XDIST_NUM_TASKS = 3
+ XDIST_REMOTE_NUM_PROCESSES = 2
}
steps {
script {
@@ -103,216 +90,7 @@ pipeline {
post {
always {
script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('lms-unit-5') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 5
- TEST_SUITE = 'lms-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('lms-unit-6') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 6
- TEST_SUITE = 'lms-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('lms-unit-7') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 7
- TEST_SUITE = 'lms-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('lms-unit-8') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 8
- TEST_SUITE = 'lms-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('lms-unit-9') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 9
- TEST_SUITE = 'lms-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('lms-unit-10') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 10
- TEST_SUITE = 'lms-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('cms-unit-1') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 1
- TEST_SUITE = 'cms-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('cms-unit-2') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 2
- TEST_SUITE = 'cms-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('commonlib-unit-1') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 1
- TEST_SUITE = 'commonlib-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('commonlib-unit-2') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 2
- TEST_SUITE = 'commonlib-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
- }
- }
- }
- }
- stage('commonlib-unit-3') {
- agent { label "jenkins-worker" }
- environment {
- SHARD = 3
- TEST_SUITE = 'commonlib-unit'
- }
- steps {
- script {
- runPythonTests()
- }
- }
- post {
- always {
- script {
- savePythonTestArtifacts()
+ pythonTestCleanup()
}
}
}
@@ -327,30 +105,16 @@ pipeline {
SUBSET_JOB = "null" // Keep this variable until we can remove the $SUBSET_JOB path from .coveragerc
}
steps {
- ansiColor('gnome-terminal') {
- sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
- checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${sha1}']],
- doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
- userRemoteConfigs: [[credentialsId: 'jenkins-worker',
- refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
- url: 'git@github.com:edx/edx-platform.git']]]
- unstash 'lms-unit-1-reports'
- unstash 'lms-unit-2-reports'
- unstash 'lms-unit-3-reports'
- unstash 'lms-unit-4-reports'
- unstash 'lms-unit-5-reports'
- unstash 'lms-unit-6-reports'
- unstash 'lms-unit-7-reports'
- unstash 'lms-unit-8-reports'
- unstash 'lms-unit-9-reports'
- unstash 'lms-unit-10-reports'
- unstash 'cms-unit-1-reports'
- unstash 'cms-unit-2-reports'
- unstash 'commonlib-unit-1-reports'
- unstash 'commonlib-unit-2-reports'
- unstash 'commonlib-unit-3-reports'
- sh "./scripts/jenkins-report.sh"
- }
+ sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+ checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
+ doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
+ userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+ refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
+ url: 'git@github.com:edx/edx-platform.git']]]
+ unstash 'lms-unit-reports'
+ unstash 'cms-unit-reports'
+ unstash 'commonlib-unit-reports'
+ sh "./scripts/jenkins-report.sh"
}
}
post {
diff --git a/pavelib/paver_tests/test_paver_pytest_cmds.py b/pavelib/paver_tests/test_paver_pytest_cmds.py
new file mode 100644
index 0000000000000000000000000000000000000000..70dd8494aafc100d37b70f50d6f5ca92e56b76e1
--- /dev/null
+++ b/pavelib/paver_tests/test_paver_pytest_cmds.py
@@ -0,0 +1,163 @@
+"""
+Tests for the pytest paver commands themselves.
+Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_pytest_cmds.py
+"""
+import unittest
+import os
+import ddt
+
+from pavelib.utils.test.suites import SystemTestSuite, LibTestSuite
+from pavelib.utils.envs import Env
+
+
+XDIST_TESTING_IP_ADDRESS_LIST = '0.0.0.1,0.0.0.2,0.0.0.3'
+
+
+@ddt.ddt
+class TestPaverPytestCmd(unittest.TestCase):
+ """
+ Test Paver pytest commands
+ """
+
+ def _expected_command(self, root, test_id, pytestSubclass, run_under_coverage=True,
+ processes=0, xdist_ip_addresses=None):
+ """
+ Returns the command that is expected to be run for the given test spec
+ and store.
+ """
+ report_dir = Env.REPORT_DIR / root
+ shard = os.environ.get('SHARD')
+ if shard:
+ report_dir = report_dir / 'shard_' + shard
+
+ expected_statement = [
+ "python",
+ "-Wd",
+ "-m",
+ "pytest"
+ ]
+ if pytestSubclass == "SystemTestSuite":
+ expected_statement.append("--ds={}".format('{}.envs.{}'.format(root, Env.TEST_SETTINGS)))
+ expected_statement.append("--junitxml={}".format(report_dir / "nosetests.xml"))
+
+ if xdist_ip_addresses:
+ expected_statement.append('--dist=loadscope')
+ for ip in xdist_ip_addresses.split(','):
+ if processes <= 0:
+ processes = 1
+
+ if pytestSubclass == "SystemTestSuite":
+ django_env_var_cmd = "export DJANGO_SETTINGS_MODULE={}.envs.test".format(root)
+ elif pytestSubclass == "LibTestSuite":
+ if 'pavelib/paver_tests' in test_id:
+ django_env_var_cmd = "export DJANGO_SETTINGS_MODULE={}.envs.test".format(root)
+ else:
+ django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='openedx.tests.settings'"
+
+ xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
+ '//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
+ '//chdir="/edx/app/edxapp/edx-platform"' \
+ .format(processes, ip, django_env_var_cmd)
+ expected_statement.append(xdist_string)
+ for rsync_dir in Env.rsync_dirs():
+ expected_statement.append('--rsyncdir {}'.format(rsync_dir))
+ else:
+ if processes == -1:
+ expected_statement.append('-n auto')
+ expected_statement.append('--dist=loadscope')
+ elif processes != 0:
+ expected_statement.append('-n {}'.format(processes))
+ expected_statement.append('--dist=loadscope')
+
+ expected_statement.extend([
+ "-p no:randomly",
+ test_id
+ ])
+
+ if run_under_coverage:
+ if xdist_ip_addresses:
+ for module in Env.covered_modules():
+ expected_statement.append('--cov')
+ expected_statement.append(module)
+ else:
+ expected_statement.append('--cov')
+ expected_statement.append('--cov-report=')
+ return expected_statement
+
+ @ddt.data('lms', 'cms')
+ def test_SystemTestSuite_suites(self, system):
+ test_id = 'tests'
+ suite = SystemTestSuite(system, test_id=test_id)
+ assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite")
+
+ @ddt.data('lms', 'cms')
+ def test_SystemTestSuite_auto_processes(self, system):
+ test_id = 'tests'
+ suite = SystemTestSuite(system, test_id=test_id, processes=-1)
+ assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite", processes=-1)
+
+ @ddt.data('lms', 'cms')
+ def test_SystemTestSuite_multi_processes(self, system):
+ test_id = 'tests'
+ suite = SystemTestSuite(system, test_id=test_id, processes=3)
+ assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite", processes=3)
+
+ @ddt.data('lms', 'cms')
+ def test_SystemTestSuite_with_xdist(self, system):
+ test_id = 'tests'
+ suite = SystemTestSuite(system, test_id=test_id, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+ assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite",
+ xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+
+ @ddt.data('lms', 'cms')
+ def test_SystemTestSuite_with_xdist_multi_processes(self, system):
+ test_id = 'tests'
+ suite = SystemTestSuite(system, test_id=test_id, processes=2, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+ assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite", processes=2,
+ xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+
+ @ddt.data('lms', 'cms')
+ def test_SystemTestSuite_with_xdist_negative_processes(self, system):
+ test_id = 'tests'
+ suite = SystemTestSuite(system, test_id=test_id, processes=-1, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+ assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite", processes=-1,
+ xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+
+ @ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
+ def test_LibTestSuite_suites(self, system):
+ test_id = 'tests'
+ suite = LibTestSuite(system, test_id=test_id)
+ assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite")
+
+ @ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
+ def test_LibTestSuite_auto_processes(self, system):
+ test_id = 'tests'
+ suite = LibTestSuite(system, test_id=test_id, processes=-1)
+ assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite", processes=-1)
+
+ @ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
+ def test_LibTestSuite_multi_processes(self, system):
+ test_id = 'tests'
+ suite = LibTestSuite(system, test_id=test_id, processes=3)
+ assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite", processes=3)
+
+ @ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
+ def test_LibTestSuite_with_xdist(self, system):
+ test_id = 'tests'
+ suite = LibTestSuite(system, test_id=test_id, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+ assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite",
+ xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+
+ @ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
+ def test_LibTestSuite_with_xdist_multi_processes(self, system):
+ test_id = 'tests'
+ suite = LibTestSuite(system, test_id=test_id, processes=2, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+ assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite", processes=2,
+ xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+
+ @ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
+ def test_LibTestSuite_with_xdist_negative_processes(self, system):
+ test_id = 'tests'
+ suite = LibTestSuite(system, test_id=test_id, processes=-1, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
+ assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite", processes=-1,
+ xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
diff --git a/pavelib/tests.py b/pavelib/tests.py
index c19ebad9303e69a62f1888fa433a44757cd4f8ca..587e793aa66cab801e021a028c89f7110dd94d5a 100644
--- a/pavelib/tests.py
+++ b/pavelib/tests.py
@@ -72,7 +72,7 @@ __test__ = False # do not collect
make_option(
'--xdist_ip_addresses',
dest='xdist_ip_addresses',
- help="Space separated string of ip addresses to shard tests to via xdist."
+ help="Comma separated string of ip addresses to shard tests to via xdist."
)
], share_with=['pavelib.utils.test.utils.clean_reports_dir'])
@PassthroughTask
@@ -160,8 +160,10 @@ def test_system(options, passthrough_options):
make_option(
'--xdist_ip_addresses',
dest='xdist_ip_addresses',
- help="Space separated string of ip addresses to shard tests to via xdist."
- )
+ help="Comma separated string of ip addresses to shard tests to via xdist."
+ ),
+ make_option('-p', '--processes', dest='processes', default=0, help='number of processes to use running tests'),
+ make_option('-r', '--randomize', action='store_true', help='run the tests in a random order'),
], share_with=['pavelib.utils.test.utils.clean_reports_dir'])
@PassthroughTask
@timed
diff --git a/pavelib/utils/test/suites/pytest_suite.py b/pavelib/utils/test/suites/pytest_suite.py
index 6f2cc120a8e88dcbb8d8fca92a3b783f33d0e749..e814e75608aa23fb73532ffda993766b3eeb4a06 100644
--- a/pavelib/utils/test/suites/pytest_suite.py
+++ b/pavelib/utils/test/suites/pytest_suite.py
@@ -158,12 +158,20 @@ class SystemTestSuite(PytestSuite):
if self.disable_capture:
cmd.append("-s")
-
if self.xdist_ip_addresses:
cmd.append('--dist=loadscope')
- for ip in self.xdist_ip_addresses.split(' '):
- xdist_string = '--tx ssh=ubuntu@{}//python="source /edx/app/edxapp/edxapp_env; ' \
- 'python"//chdir="/edx/app/edxapp/edx-platform"'.format(ip)
+ if self.processes <= 0:
+ xdist_remote_processes = 1
+ else:
+ xdist_remote_processes = self.processes
+ for ip in self.xdist_ip_addresses.split(','):
+ # The django settings runtime command does not propagate to xdist remote workers
+ django_env_var_cmd = 'export DJANGO_SETTINGS_MODULE={}' \
+ .format('{}.envs.{}'.format(self.root, self.settings))
+ xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
+ '//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
+ '//chdir="/edx/app/edxapp/edx-platform"' \
+ .format(xdist_remote_processes, ip, django_env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append('--rsyncdir {}'.format(rsync_dir))
@@ -239,6 +247,14 @@ class LibTestSuite(PytestSuite):
self.test_id = kwargs.get('test_id', self.root)
self.eval_attr = kwargs.get('eval_attr', None)
self.xdist_ip_addresses = kwargs.get('xdist_ip_addresses', None)
+ self.randomize = kwargs.get('randomize', None)
+ self.processes = kwargs.get('processes', None)
+
+ if self.processes is None:
+ # Don't use multiprocessing by default
+ self.processes = 0
+
+ self.processes = int(self.processes)
@property
def cmd(self):
@@ -251,8 +267,6 @@ class LibTestSuite(PytestSuite):
'-Wd',
'-m',
'pytest',
- '-p',
- 'no:randomly',
'--junitxml={}'.format(self.xunit_report),
])
cmd.extend(self.passthrough_options + self.test_options_flags)
@@ -265,13 +279,33 @@ class LibTestSuite(PytestSuite):
if self.xdist_ip_addresses:
cmd.append('--dist=loadscope')
- for ip in self.xdist_ip_addresses.split(' '):
- xdist_string = '--tx ssh=ubuntu@{}//python="source /edx/app/edxapp/edxapp_env; ' \
- 'python"//chdir="/edx/app/edxapp/edx-platform"'.format(ip)
+ if self.processes <= 0:
+ xdist_remote_processes = 1
+ else:
+ xdist_remote_processes = self.processes
+ for ip in self.xdist_ip_addresses.split(','):
+ # The django settings runtime command does not propagate to xdist remote workers
+ if 'pavelib/paver_tests' in self.test_id:
+ django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='lms.envs.test'"
+ else:
+ django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='openedx.tests.settings'"
+ xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
+ '//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
+ '//chdir="/edx/app/edxapp/edx-platform"' \
+ .format(xdist_remote_processes, ip, django_env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append('--rsyncdir {}'.format(rsync_dir))
+ else:
+ if self.processes == -1:
+ cmd.append('-n auto')
+ cmd.append('--dist=loadscope')
+ elif self.processes != 0:
+ cmd.append('-n {}'.format(self.processes))
+ cmd.append('--dist=loadscope')
+ if not self.randomize:
+ cmd.append("-p no:randomly")
if self.eval_attr:
cmd.append("-a '{}'".format(self.eval_attr))
diff --git a/scripts/unit-tests.sh b/scripts/unit-tests.sh
index 837dd8e73e70a7a48eb20788e1de0dbc3dbecaa2..be47bdf0e2e807c332e869401989502beda07dc7 100755
--- a/scripts/unit-tests.sh
+++ b/scripts/unit-tests.sh
@@ -29,8 +29,6 @@ set -e
#
###############################################################################
-PAVER_ARGS="-v"
-PARALLEL="--processes=-1"
export SKIP_NPM_INSTALL="True"
# Skip re-installation of Python prerequisites inside a tox execution.
@@ -38,6 +36,20 @@ if [[ -n "$TOXENV" ]]; then
export NO_PREREQ_INSTALL="True"
fi
+if [[ -n "$XDIST_NUM_TASKS" ]]; then
+ bash scripts/xdist/prepare_xdist_nodes.sh
+ PAVER_ARGS="-v --xdist_ip_addresses="$(<pytest_task_ips.txt)""
+ export SHARD="all"
+ if [[ -n "$XDIST_REMOTE_NUM_PROCESSES" ]]; then
+ PARALLEL="--processes=$XDIST_REMOTE_NUM_PROCESSES"
+ else
+ PARALLEL="--processes=1"
+ fi
+else
+ PAVER_ARGS="-v"
+ PARALLEL="--processes=-1"
+fi
+
case "${TEST_SUITE}" in
"lms-unit")
@@ -65,7 +77,7 @@ case "${TEST_SUITE}" in
"cms-unit")
case "$SHARD" in
"all")
- paver test_system -s cms --disable_capture ${PAVER_ARGS} 2> cms-tests.log
+ paver test_system -s cms --disable_capture ${PAVER_ARGS} ${PARALLEL} 2> cms-tests.log
;;
1)
paver test_system -s cms --disable_capture --eval-attr="shard==$SHARD" ${PAVER_ARGS} 2> cms-tests.${SHARD}.log
@@ -87,7 +99,7 @@ case "${TEST_SUITE}" in
"commonlib-unit")
case "$SHARD" in
"all")
- paver test_lib --disable_capture ${PAVER_ARGS} 2> common-tests.log
+ paver test_lib --disable_capture ${PAVER_ARGS} ${PARALLEL} 2> common-tests.log
;;
[1-2])
paver test_lib -l common/lib/xmodule --disable_capture --eval-attr="shard==$SHARD" ${PAVER_ARGS} 2> common-tests.${SHARD}.log
diff --git a/scripts/xdist/prepare_xdist_nodes.sh b/scripts/xdist/prepare_xdist_nodes.sh
index 3e41956386c7eca7d8982e7ce1518cabf1415585..763fca9b51ba3dcd38f58ebbe0965a388548eda8 100644
--- a/scripts/xdist/prepare_xdist_nodes.sh
+++ b/scripts/xdist/prepare_xdist_nodes.sh
@@ -1,16 +1,16 @@
#!/bin/bash
set -e
+echo "Spinning up xdist containers with pytest_container_manager.py"
python scripts/xdist/pytest_container_manager.py -a up -n ${XDIST_NUM_TASKS} \
-t ${XDIST_CONTAINER_TASK_NAME} \
-s ${XDIST_CONTAINER_SUBNET} \
-sg ${XDIST_CONTAINER_SECURITY_GROUP}
ip_list=$(<pytest_task_ips.txt)
-
-for ip in $ip_list
+for ip in $(echo $ip_list | sed "s/,/ /g")
do
- container_reqs_cmd="ssh ubuntu@$ip 'cd /edx/app/edxapp/edx-platform;
+ container_reqs_cmd="ssh -o StrictHostKeyChecking=no ubuntu@$ip 'cd /edx/app/edxapp/edx-platform;
git pull -q; git checkout -q ${XDIST_GIT_BRANCH};
source /edx/app/edxapp/edxapp_env; pip install -qr requirements/edx/testing.txt' & "
diff --git a/scripts/xdist/pytest_container_manager.py b/scripts/xdist/pytest_container_manager.py
index e7dd0b384b9373cc40148e62ca5c56a3ee9cac30..4fc22e3397be0a6777720f696543745de4eb587c 100644
--- a/scripts/xdist/pytest_container_manager.py
+++ b/scripts/xdist/pytest_container_manager.py
@@ -104,13 +104,13 @@ class PytestContainerManager():
logger.info("Successfully booted up {} tasks.".format(number_of_tasks))
# Generate .txt files containing IP addresses and task arns
- ip_list_string = " ".join(ip_addresses)
+ ip_list_string = ",".join(ip_addresses)
logger.info("Task IP list: {}".format(ip_list_string))
ip_list_file = open("pytest_task_ips.txt", "w")
ip_list_file.write(ip_list_string)
ip_list_file.close()
- task_arn_list_string = " ".join(task_arns)
+ task_arn_list_string = ",".join(task_arns)
logger.info("Task arn list: {}".format(task_arn_list_string))
task_arn_file = open("pytest_task_arns.txt", "w")
task_arn_file.write(task_arn_list_string)
@@ -120,7 +120,7 @@ class PytestContainerManager():
"""
Terminates tasks based on a list of task_arns.
"""
- for task_arn in task_arns:
+ for task_arn in task_arns.split(','):
response = self.ecs.stop_task(
cluster=self.cluster_name,
task=task_arn,
@@ -134,23 +134,26 @@ if __name__ == "__main__":
description="PytestContainerManager, manages ECS tasks in an AWS cluster."
)
- parser.add_argument('--region', '-g', default='us-east-1',
- help="AWS region where ECS infrastructure lives. Defaults to us-east-1")
+ parser.add_argument('--action', '-a', choices=['up', 'down'], default=None,
+ help="Action for PytestContainerManager to perform. "
+ "Either up for spinning up AWS ECS tasks or down for stopping them")
parser.add_argument('--cluster', '-c', default="jenkins-worker-containers",
help="AWS Cluster name where the tasks run. Defaults to"
"the testeng cluster: jenkins-worker-containers")
- parser.add_argument('--action', '-a', choices=['up', 'down'], default=None,
- help="Action for PytestContainerManager to perform. "
- "Either up for spinning up AWS ECS tasks or down for stopping them")
+ parser.add_argument('--region', '-g', default='us-east-1',
+ help="AWS region where ECS infrastructure lives. Defaults to us-east-1")
# Spinning up tasks
+ parser.add_argument('--launch_type', default='FARGATE', choices=['EC2', 'FARGATE'],
+ help="ECS launch type for tasks. Defaults to FARGATE")
+
parser.add_argument('--num_tasks', '-n', type=int, default=None,
help="Number of ECS tasks to spin up")
- parser.add_argument('--task_name', '-t', default=None,
- help="Name of the task definition")
+ parser.add_argument('--public_ip', choices=['ENABLED', 'DISABLED'],
+ default='DISABLED', help="Whether the tasks should have a public IP")
parser.add_argument('--subnets', '-s', nargs='+', default=None,
help="List of subnets for the tasks to exist in")
@@ -158,19 +161,16 @@ if __name__ == "__main__":
parser.add_argument('--security_groups', '-sg', nargs='+', default=None,
help="List of security groups to apply to the tasks")
- parser.add_argument('--public_ip', choices=['ENABLED', 'DISABLED'],
- default='DISABLED', help="Whether the tasks should have a public IP")
-
- parser.add_argument('--launch_type', default='FARGATE', choices=['EC2', 'FARGATE'],
- help="ECS launch type for tasks. Defaults to FARGATE")
+ parser.add_argument('--task_name', '-t', default=None,
+ help="Name of the task definition")
# Terminating tasks
- parser.add_argument('--task_arns', '-arns', nargs='+', default=None,
- help="Task arns to terminate")
-
parser.add_argument('--reason', '-r', default="Finished executing tests",
help="Reason for terminating tasks")
+ parser.add_argument('--task_arns', '-arns', default=None,
+ help="Task arns to terminate")
+
args = parser.parse_args()
containerManager = PytestContainerManager(args.region, args.cluster)
diff --git a/scripts/xdist/terminate_xdist_nodes.sh b/scripts/xdist/terminate_xdist_nodes.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ee077a2c5739434fda772b5e65b45563ace27b18
--- /dev/null
+++ b/scripts/xdist/terminate_xdist_nodes.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+if [ -f pytest_task_arns.txt ]; then
+ echo "Terminating xdist containers with pytest_container_manager.py"
+ xdist_task_arns=$(<pytest_task_arns.txt)
+ python scripts/xdist/pytest_container_manager.py -a down --task_arns ${xdist_task_arns}
+else
+ echo "File: pytest_task_arns.txt not found"
+fi
diff --git a/tox.ini b/tox.ini
index c0b3d9475f531be991cd0a34d17f89ad54c737c0..4e592b2917c81cc127e6ef1c25bc4f3a99369778 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,6 +51,7 @@ passenv =
XDIST_CONTAINER_TASK_NAME
XDIST_GIT_BRANCH
XDIST_NUM_TASKS
+ XDIST_REMOTE_NUM_PROCESSES
deps =
django18: Django>=1.8,<1.9
django19: Django>=1.9,<1.10