diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index dcff051155d468e86bcf8c56d78e99c521de3138..2d775c0998b597d5459794ac5cde9d5b21bf3aeb 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -78,6 +78,9 @@ class Env(object): "lib" / "custom_a11y_rules.js" ) + # Which Python version should be used in xdist workers? + PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7") + # If set, put reports for run in "unique" directories. # The main purpose of this is to ensure that the reports can be 'slurped' # in the main jenkins flow job without overwriting the reports from other diff --git a/pavelib/utils/test/suites/pytest_suite.py b/pavelib/utils/test/suites/pytest_suite.py index 02f5fd2882a0217aa78832e015cbb5cbd5b09249..68f6aa37b3f1a69258fb2e8dfb5cffc18830b1fd 100644 --- a/pavelib/utils/test/suites/pytest_suite.py +++ b/pavelib/utils/test/suites/pytest_suite.py @@ -184,9 +184,9 @@ class SystemTestSuite(PytestSuite): .format('{}.envs.{}'.format(self.root, self.settings), self.disable_courseenrollment_history) xdist_string = u'--tx {}*ssh="jenkins@{} -o StrictHostKeyChecking=no"' \ - '//python="source edx-venv/bin/activate; {}; python"' \ + '//python="source edx-venv-{}/edx-venv/bin/activate; {}; python"' \ '//chdir="edx-platform"' \ - .format(xdist_remote_processes, ip, env_var_cmd) + .format(xdist_remote_processes, ip, Env.PYTHON_VERSION, env_var_cmd) cmd.append(xdist_string) for rsync_dir in Env.rsync_dirs(): cmd.append(u'--rsyncdir {}'.format(rsync_dir)) @@ -310,9 +310,9 @@ class LibTestSuite(PytestSuite): .format(django_env_var_cmd, self.disable_courseenrollment_history) xdist_string = u'--tx {}*ssh="jenkins@{} -o StrictHostKeyChecking=no"' \ - '//python="source edx-venv/bin/activate; {}; python"' \ + '//python="source edx-venv-{}/edx-venv/bin/activate; {}; python"' \ '//chdir="edx-platform"' \ - .format(xdist_remote_processes, ip, env_var_cmd) + .format(xdist_remote_processes, ip, Env.PYTHON_VERSION, env_var_cmd) cmd.append(xdist_string) for rsync_dir in Env.rsync_dirs(): cmd.append(u'--rsyncdir {}'.format(rsync_dir)) diff --git a/scripts/Jenkinsfiles/python b/scripts/Jenkinsfiles/python index beb4c6f482958d8fc591e77854a4fc1d0b751119..8aa44faa22fdafda5c04e6f2ff92845e2fdb409c 100644 --- a/scripts/Jenkinsfiles/python +++ b/scripts/Jenkinsfiles/python @@ -16,7 +16,7 @@ def pythonTestCleanup() { archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,*.log' sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB' junit '**/nosetests.xml' - sh '''source $HOME/edx-venv/bin/activate + sh '''source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate bash scripts/xdist/terminate_xdist_nodes.sh''' } diff --git a/scripts/jenkins-common.sh b/scripts/jenkins-common.sh index e585809641540e0551c45bb421a8b0340e74b872..b1a56b1f39cded8e35b94affd491c623ee25dc58 100644 --- a/scripts/jenkins-common.sh +++ b/scripts/jenkins-common.sh @@ -20,14 +20,16 @@ fi # Reset the jenkins worker's virtualenv back to the # state it was in when the instance was spun up. if [ -z ${PYTHON_VERSION+x} ] || [[ ${PYTHON_VERSION} == 'null' ]]; then + VENV_PATH="$HOME" ARCHIVED_VENV="edx-venv_clean.tar.gz" else + VENV_PATH="$HOME/edx-venv-$PYTHON_VERSION" ARCHIVED_VENV="edx-venv_clean-$PYTHON_VERSION.tar.gz" fi if [ -e $HOME/$ARCHIVED_VENV ]; then - rm -rf $HOME/edx-venv - tar -C $HOME -xf $HOME/$ARCHIVED_VENV + rm -rf $VENV_PATH/edx-venv + tar -C $VENV_PATH -xf $HOME/$ARCHIVED_VENV fi # Load the npm packages from the time the worker was built @@ -41,7 +43,7 @@ if [ -e $HOME/edx-npm-cache_clean.tar.gz ]; then fi # Activate the Python virtualenv -source $HOME/edx-venv/bin/activate +source $VENV_PATH/edx-venv/bin/activate # add the node packages dir to PATH PATH=$PATH:node_modules/.bin diff --git a/scripts/xdist/prepare_xdist_nodes.sh b/scripts/xdist/prepare_xdist_nodes.sh index 0029c75e6eea57816aca8ab5e59b0cb3192e9cc3..c5ec85919d7722ceb39051d4ec2b04c45255cf66 100644 --- a/scripts/xdist/prepare_xdist_nodes.sh +++ b/scripts/xdist/prepare_xdist_nodes.sh @@ -23,9 +23,9 @@ do worker_reqs_cmd="ssh -o StrictHostKeyChecking=no jenkins@$ip 'git clone --branch master --depth 1 -q https://github.com/edx/edx-platform.git; cd edx-platform; git fetch -fq origin ${XDIST_GIT_REFSPEC}; git checkout -q ${XDIST_GIT_BRANCH}; - rm -rf /home/jenkins/edx-venv; - tar -C /home/jenkins -xf /home/jenkins/edx-venv_clean-${PYTHON_VERSION}.tar.gz; - source ../edx-venv/bin/activate; + rm -rf /home/jenkins/edx-venv-${PYTHON_VERSION}/edx-venv; + tar -C /home/jenkins/edx-venv-${PYTHON_VERSION} -xf /home/jenkins/edx-venv_clean-${PYTHON_VERSION}.tar.gz; + source ../edx-venv-${PYTHON_VERSION}/edx-venv/bin/activate; pip install -q ${DJANGO_REQUIREMENT} -r requirements/edx/testing.txt; mkdir reports' & " cmd=$cmd$worker_reqs_cmd