Merge pull request #19296 from edx/youngstrom/update-python-jenkinsfile

Adjust unittest container counts and delete old Jenkinsfile

Jenkinsfile

-def runPythonTests() {
-    sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) {
-        checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
-            doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true,
-            noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
-            refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*',
-            url: 'git@github.com:edx/edx-platform.git']]]
-        console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
-        dir('stdout') {
-            writeFile file: "${TEST_SUITE}-stdout.log", text: console_output
-        }
-        stash includes: 'reports/**/*coverage*', name: "${TEST_SUITE}-reports"
-    }
-}
-
-def pythonTestCleanup() {
-    archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,stdout/*.log,*.log'
-    junit '**/nosetests.xml'
-    sh '''source $HOME/edx-venv/bin/activate
-    bash scripts/xdist/terminate_xdist_nodes.sh'''
-}
-
-pipeline {
-    agent { label "coverage-worker" }
-    options {
-        timestamps()
-        timeout(60)
-    }
-    environment {
-        XDIST_CONTAINER_SUBNET = credentials('XDIST_CONTAINER_SUBNET')
-        XDIST_CONTAINER_SECURITY_GROUP = credentials('XDIST_CONTAINER_SECURITY_GROUP')
-        XDIST_CONTAINER_TASK_NAME = "jenkins-worker-task"
-        XDIST_GIT_BRANCH = "${ghprbActualCommit}"
-    }
-    stages {
-        stage('Run Tests') {
-            parallel {
-                stage("lms-unit") {
-                    agent { label "jenkins-worker" }
-                    environment {
-                        TEST_SUITE = "lms-unit"
-                        XDIST_NUM_TASKS = 10
-                        XDIST_REMOTE_NUM_PROCESSES = 2
-                    }
-                    steps {
-                        script {
-                            runPythonTests()
-                        }
-                    }
-                    post {
-                        always {
-                            script {
-                                pythonTestCleanup()
-                            }
-                        }
-                    }
-                }
-                stage("cms-unit") {
-                    agent { label "jenkins-worker" }
-                    environment {
-                        TEST_SUITE = "cms-unit"
-                        XDIST_NUM_TASKS = 4
-                        XDIST_REMOTE_NUM_PROCESSES = 2
-                    }
-                    steps {
-                        script {
-                            runPythonTests()
-                        }
-                    }
-                    post {
-                        always {
-                            script {
-                                pythonTestCleanup()
-                            }
-                        }
-                    }
-                }
-                stage("commonlib-unit") {
-                    agent { label "jenkins-worker" }
-                    environment {
-                        TEST_SUITE = "commonlib-unit"
-                        XDIST_NUM_TASKS = 3
-                        XDIST_REMOTE_NUM_PROCESSES = 2
-                    }
-                    steps {
-                        script {
-                            runPythonTests()
-                        }
-                    }
-                    post {
-                        always {
-                            script {
-                                pythonTestCleanup()
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        stage('Run coverage') {
-            environment {
-                CODE_COV_TOKEN = credentials('CODE_COV_TOKEN')
-                TARGET_BRANCH = "origin/master"
-                CI_BRANCH = "${ghprbSourceBranch}"
-                SUBSET_JOB = "null" // Keep this variable until we can remove the $SUBSET_JOB path from .coveragerc
-            }
-            steps {
-                sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
-                    checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
-                        doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption',
-                        honorRefspec: true, noTags: true, shallow: true]], submoduleCfg: [],
-                        userRemoteConfigs: [[credentialsId: 'jenkins-worker',
-                        refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*',
-                        url: 'git@github.com:edx/edx-platform.git']]]
-                    unstash 'lms-unit-reports'
-                    unstash 'cms-unit-reports'
-                    unstash 'commonlib-unit-reports'
-                    sh "./scripts/jenkins-report.sh"
-                }
-            }
-            post {
-                always {
-                    publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
-                        reportDir: 'reports', reportFiles: 'diff_coverage_combined.html',
-                        reportName: 'Diff Coverage Report', reportTitles: ''])
-                    publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
-                        reportDir: 'reports/cover', reportFiles: 'index.html',
-                        reportName: 'Coverage.py Report', reportTitles: ''])
-                }
-            }
-        }
-    }
-}

scripts/Jenkinsfiles/python

@@ -21,7 +21,7 @@ def pythonTestCleanup() {
 }
 
 pipeline {
-    agent { label "coverage-worker" }
+    agent { label "jenkins-worker" }
     options {
         timestamps()
         timeout(60)
@@ -40,7 +40,7 @@ pipeline {
                     environment {
                         TEST_SUITE = "lms-unit"
                         XDIST_NUM_TASKS = 10
-                        XDIST_REMOTE_NUM_PROCESSES = 2
+                        XDIST_REMOTE_NUM_PROCESSES = 1
                     }
                     steps {
                         script {
@@ -59,8 +59,8 @@ pipeline {
                     agent { label "jenkins-worker" }
                     environment {
                         TEST_SUITE = "cms-unit"
-                        XDIST_NUM_TASKS = 4
-                        XDIST_REMOTE_NUM_PROCESSES = 2
+                        XDIST_NUM_TASKS = 3
+                        XDIST_REMOTE_NUM_PROCESSES = 1
                     }
                     steps {
                         script {
@@ -80,7 +80,7 @@ pipeline {
                     environment {
                         TEST_SUITE = "commonlib-unit"
                         XDIST_NUM_TASKS = 3
-                        XDIST_REMOTE_NUM_PROCESSES = 2
+                        XDIST_REMOTE_NUM_PROCESSES = 1
                     }
                     steps {
                         script {