Fix Jenkins virtualenv cleanup (#22356)

pavelib/utils/envs.py

@@ -78,6 +78,9 @@ class Env(object):
         "lib" / "custom_a11y_rules.js"
     )
 
+    # Which Python version should be used in xdist workers?
+    PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7")
+
     # If set, put reports for run in "unique" directories.
     # The main purpose of this is to ensure that the reports can be 'slurped'
     # in the main jenkins flow job without overwriting the reports from other

pavelib/utils/test/suites/pytest_suite.py

@@ -184,9 +184,9 @@ class SystemTestSuite(PytestSuite):
                     .format('{}.envs.{}'.format(self.root, self.settings),
                             self.disable_courseenrollment_history)
                 xdist_string = u'--tx {}*ssh="jenkins@{} -o StrictHostKeyChecking=no"' \
-                               '//python="source edx-venv/bin/activate; {}; python"' \
+                               '//python="source edx-venv-{}/edx-venv/bin/activate; {}; python"' \
                                '//chdir="edx-platform"' \
-                               .format(xdist_remote_processes, ip, env_var_cmd)
+                               .format(xdist_remote_processes, ip, Env.PYTHON_VERSION, env_var_cmd)
                 cmd.append(xdist_string)
             for rsync_dir in Env.rsync_dirs():
                 cmd.append(u'--rsyncdir {}'.format(rsync_dir))
@@ -310,9 +310,9 @@ class LibTestSuite(PytestSuite):
                     .format(django_env_var_cmd, self.disable_courseenrollment_history)
 
                 xdist_string = u'--tx {}*ssh="jenkins@{} -o StrictHostKeyChecking=no"' \
-                               '//python="source edx-venv/bin/activate; {}; python"' \
+                               '//python="source edx-venv-{}/edx-venv/bin/activate; {}; python"' \
                                '//chdir="edx-platform"' \
-                               .format(xdist_remote_processes, ip, env_var_cmd)
+                               .format(xdist_remote_processes, ip, Env.PYTHON_VERSION, env_var_cmd)
                 cmd.append(xdist_string)
             for rsync_dir in Env.rsync_dirs():
                 cmd.append(u'--rsyncdir {}'.format(rsync_dir))

scripts/Jenkinsfiles/python

@@ -16,7 +16,7 @@ def pythonTestCleanup() {
     archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,*.log'
     sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB'
     junit '**/nosetests.xml'
-    sh '''source $HOME/edx-venv/bin/activate
+    sh '''source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate
     bash scripts/xdist/terminate_xdist_nodes.sh'''
 }
 

scripts/jenkins-common.sh

@@ -20,14 +20,16 @@ fi
 # Reset the jenkins worker's virtualenv back to the
 # state it was in when the instance was spun up.
 if [ -z ${PYTHON_VERSION+x} ] || [[ ${PYTHON_VERSION} == 'null' ]]; then
+    VENV_PATH="$HOME"
     ARCHIVED_VENV="edx-venv_clean.tar.gz"
 else
+    VENV_PATH="$HOME/edx-venv-$PYTHON_VERSION"
     ARCHIVED_VENV="edx-venv_clean-$PYTHON_VERSION.tar.gz"
 fi
 
 if [ -e $HOME/$ARCHIVED_VENV ]; then
-    rm -rf $HOME/edx-venv
-    tar -C $HOME -xf $HOME/$ARCHIVED_VENV
+    rm -rf $VENV_PATH/edx-venv
+    tar -C $VENV_PATH -xf $HOME/$ARCHIVED_VENV
 fi
 
 # Load the npm packages from the time the worker was built
@@ -41,7 +43,7 @@ if [ -e $HOME/edx-npm-cache_clean.tar.gz ]; then
 fi
 
 # Activate the Python virtualenv
-source $HOME/edx-venv/bin/activate
+source $VENV_PATH/edx-venv/bin/activate
 
 # add the node packages dir to PATH
 PATH=$PATH:node_modules/.bin

scripts/xdist/prepare_xdist_nodes.sh

@@ -23,9 +23,9 @@ do
     worker_reqs_cmd="ssh -o StrictHostKeyChecking=no jenkins@$ip
     'git clone --branch master --depth 1 -q https://github.com/edx/edx-platform.git; cd edx-platform;
     git fetch -fq origin ${XDIST_GIT_REFSPEC}; git checkout -q ${XDIST_GIT_BRANCH};
-    rm -rf /home/jenkins/edx-venv;
-    tar -C /home/jenkins -xf /home/jenkins/edx-venv_clean-${PYTHON_VERSION}.tar.gz;
-    source ../edx-venv/bin/activate;
+    rm -rf /home/jenkins/edx-venv-${PYTHON_VERSION}/edx-venv;
+    tar -C /home/jenkins/edx-venv-${PYTHON_VERSION} -xf /home/jenkins/edx-venv_clean-${PYTHON_VERSION}.tar.gz;
+    source ../edx-venv-${PYTHON_VERSION}/edx-venv/bin/activate;
     pip install -q ${DJANGO_REQUIREMENT} -r requirements/edx/testing.txt; mkdir reports' & "
 
     cmd=$cmd$worker_reqs_cmd