Merge pull request #19356 from edx/youngstrom/add-quality-jenkinsfile

Add Jenkinsfiles for quality and bokchoy

scripts/Jenkinsfiles/bokchoy

+def runBokchoyTests() {
+    // Determine git refspec, branch, and clone type
+    git_shallow_clone = true
+    git_extensions = []
+    if (env.ghprbActualCommit) {
+        git_branch = "${ghprbActualCommit}"
+        git_refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+    } else {
+        git_branch = 'master'
+        git_refspec = "+refs/heads/master:refs/remotes/origin/master"
+    }
+    sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+        checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]],
+            doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true,
+            noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+            refspec: git_refspec, url: 'git@github.com:edx/edx-platform.git']]]
+        console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
+        dir('stdout') {
+            writeFile file: "${TEST_SUITE}-${SHARD}-stdout.log", text: console_output
+        }
+    }
+}
+
+def bokchoyTestCleanup() {
+    archiveArtifacts allowEmptyArchive: true, artifacts: 'test_root/log/**/*.log,stdout/*.log'
+    junit '**/reports/bok_choy/**/xunit.xml'
+}
+
+pipeline {
+    agent { label "jenkins-worker" }
+    options {
+        timestamps()
+        timeout(60)
+    }
+    stages {
+        stage('Mark build as pending on Github') {
+            when {
+                 // Only run github-build-status for master builds
+                 expression { env.ghprbActualCommit == null }
+            }
+            steps {
+                script {
+                    commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+                    build job: 'github-build-status',
+                        parameters: [
+                            string(name: 'GIT_SHA', value: commit_sha),
+                            string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
+                            string(name: 'TARGET_URL', value: "${BUILD_URL}"),
+                            string(name: 'DESCRIPTION', value: 'Pending'), string(name: 'CONTEXT', value: 'jenkins/bokchoy-pipeline-master'),
+                            string(name: 'CREATE_DEPLOYMENT', value: 'false'), string(name: 'BUILD_STATUS', value: 'pending')
+                        ],
+                        propagate: false, wait: false
+                }
+            }
+        }
+        stage("Run tests") {
+            steps {
+                script {
+                    def parallel_stages = [:]
+                    for (int i = 1; i <= 22; i++) {
+                        int index = i
+                        parallel_stages["${index}"] = {
+                            node('jenkins-worker') {
+                                withEnv(["SHARD=${index}","TEST_SUITE=bok-choy"]) {
+                                    try {
+                                        stage("Bokchoy shard: ${index}") {
+                                            runBokchoyTests()
+                                        }
+                                    } finally {
+                                        bokchoyTestCleanup()
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    parallel parallel_stages
+                }
+            }
+        }
+    }
+    post {
+        always {
+            script{
+                if (env.ghprbPullId != null) {
+                    // For PR jobs, run the edx-platform-test-notifier for PR reporting
+                    build job: 'edx-platform-test-notifier', parameters: [string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false
+                } else {
+                    // For master jobs run github-build-status and report to slack when necessary
+                    if (currentBuild.currentResult == "SUCCESS") {
+                        create_deployment = "true"
+                        build_status = "success"
+                        build_description = "Build Passed"
+                    }
+                    else {
+                        create_deployment = "false"
+                        build_status = "failure"
+                        build_description = "Build Failed"
+                    }
+
+                    commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+                    build job: 'github-build-status',
+                        parameters: [
+                            string(name: 'GIT_SHA', value: commit_sha),
+                            string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
+                            string(name: 'TARGET_URL', value: "${BUILD_URL}"),
+                            string(name: 'DESCRIPTION', value: build_description), string(name: 'CONTEXT', value: 'jenkins/bokchoy-pipeline-master'),
+                            string(name: 'CREATE_DEPLOYMENT', value: create_deployment), string(name: 'BUILD_STATUS', value: build_status)
+                        ],
+                        propagate: false, wait: false
+
+                    if (currentBuild.currentResult != "SUCCESS"){
+                        slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
+
+                        email_body = "See: <${BUILD_URL}>\n\nChanges:\n"
+                        change_sets = currentBuild.changeSets
+                        for (int j = 0; j < change_sets.size(); j++) {
+                            change_set_items = change_sets[j].items
+                            for (int k = 0; k < change_set_items.length; k++) {
+                                item = change_set_items[k]
+                                email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}"
+                            }
+                        }
+                        emailext body: email_body,
+                            subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
+                    } else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") {
+                        slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
+                        emailext body: "See <${BUILD_URL}>",
+                            subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
+                    }
+                }
+            }
+        }
+    }
+}

scripts/Jenkinsfiles/quality

+def runQualityTests() {
+    sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+        // Determine git refspec, branch, and clone type
+        git_shallow_clone = true
+        git_extensions = []
+        if (env.ghprbActualCommit) {
+            git_branch = "${ghprbActualCommit}"
+            refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+            if (SHARD == "4") {
+                git_shallow_clone = false
+                git_extensions.add([$class: 'WipeWorkspace'])
+                refspec = refspec + " +refs/heads/master:refs/remotes/origin/master"
+            }
+        } else {
+            git_branch = 'master'
+            refspec = "+refs/heads/master:refs/remotes/origin/master"
+        }
+        git_extensions.add([$class: 'CloneOption', honorRefspec: true, noTags: true, shallow: git_shallow_clone ])
+
+        checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]],
+            doGenerateSubmoduleConfigurations: false, extensions: git_extensions, submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+            refspec: refspec,
+            url: 'git@github.com:edx/edx-platform.git']]]
+
+        sh "bash scripts/all-tests.sh"
+        stash includes: '**/reports/**/*', name: "${TEST_SUITE}-${SHARD}-reports"
+    }
+}
+
+def qualityTestCleanup() {
+    archiveArtifacts allowEmptyArchive: true, artifacts: '**/reports/**/*,test_root/log/**/*.log,*.log'
+    junit 'reports/quality_junitxml/*.xml'
+}
+
+pipeline {
+    agent { label "jenkins-worker" }
+    options {
+        timestamps()
+        timeout(60)
+    }
+    stages {
+        stage('Mark build as pending on Github') {
+            when {
+                 // Only run github-build-status for master builds
+                 expression { env.ghprbActualCommit == null }
+            }
+            steps {
+                script {
+                    commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+                    build job: 'github-build-status',
+                        parameters: [
+                            string(name: 'GIT_SHA', value: commit_sha),
+                            string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
+                            string(name: 'TARGET_URL', value: "${BUILD_URL}"),
+                            string(name: 'DESCRIPTION', value: 'Pending'), string(name: 'CONTEXT', value: 'jenkins/quality-pipeline-master'),
+                            string(name: 'CREATE_DEPLOYMENT', value: 'false'), string(name: 'BUILD_STATUS', value: 'pending')
+                        ],
+                        propagate: false, wait: false
+                }
+            }
+        }
+        stage('Run Tests') {
+            parallel {
+                stage("commonlib pylint") {
+                    agent { label "jenkins-worker" }
+                    environment {
+                        TEST_SUITE = "quality"
+                        SHARD = 1
+                    }
+                    steps {
+                        script {
+                            runQualityTests()
+                        }
+                    }
+                    post {
+                        always {
+                            script {
+                                qualityTestCleanup()
+                            }
+                        }
+                    }
+                }
+                stage("lms pylint") {
+                    agent { label "jenkins-worker" }
+                    environment {
+                        TEST_SUITE = "quality"
+                        SHARD = 2
+                    }
+                    steps {
+                        script {
+                            runQualityTests()
+                        }
+                    }
+                    post {
+                        always {
+                            script {
+                                qualityTestCleanup()
+                            }
+                        }
+                    }
+                }
+                stage("cms/openedx/pavelib pylint") {
+                    agent { label "jenkins-worker" }
+                    environment {
+                        TEST_SUITE = "quality"
+                        SHARD = 3
+                    }
+                    steps {
+                        script {
+                            runQualityTests()
+                        }
+                    }
+                    post {
+                        always {
+                            script {
+                                qualityTestCleanup()
+                            }
+                        }
+                    }
+                }
+                stage("Other quality checks") {
+                    agent { label "jenkins-worker" }
+                    environment {
+                        TEST_SUITE = "quality"
+                        SHARD = 4
+                    }
+                    steps {
+                        script {
+                            runQualityTests()
+                        }
+                    }
+                    post {
+                        always {
+                            script {
+                                qualityTestCleanup()
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        stage('Diff quality') {
+            when {
+                // Only run diff quality on PR builds
+                 expression { env.ghprbTargetBranch != null }
+            }
+            environment {
+                TARGET_BRANCH = "origin/${ghprbTargetBranch}"
+            }
+            steps {
+                sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+                    checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: "${ghprbActualCommit}"]],
+                        doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption',
+                        honorRefspec: true, noTags: true, shallow: false], [$class: 'WipeWorkspace']], submoduleCfg: [],
+                        userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+                        refspec: "+refs/heads/${ghprbTargetBranch}:refs/remotes/origin/${ghprbTargetBranch} +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*",
+                        url: 'git@github.com:edx/edx-platform.git']]]
+                    unstash 'quality-1-reports'
+                    unstash 'quality-2-reports'
+                    unstash 'quality-3-reports'
+                    unstash 'quality-4-reports'
+                    sh "./scripts/jenkins-quality-diff.sh"
+                }
+            }
+            post {
+                always {
+                    qualityTestCleanup()
+                    publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'reports/diff_quality',
+                        reportFiles: 'diff_quality_pylint.html, diff_quality_eslint.html', reportName: 'Diff Quality Report', reportTitles: ''])
+                }
+            }
+        }
+    }
+    post {
+        always {
+            script{
+                try {
+                    unstash 'quality-1-reports'
+                    unstash 'quality-2-reports'
+                    unstash 'quality-3-reports'
+                    unstash 'quality-4-reports'
+                    // Check for warnings
+                    warnings canComputeNew: false, canResolveRelativePaths: false, canRunOnFailed: true, categoriesPattern: '',
+                        defaultEncoding: '', excludePattern: '', healthy: '', includePattern: '', messagesPattern: '',
+                        parserConfigurations: [[parserName: 'Pep8', pattern: 'reports/pep8/pep8.report'], [parserName: 'PyLint', pattern: 'reports/**/pylint.report']],
+                        unHealthy: ''
+                    // Publish Quality report
+                    publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true,
+                        reportDir: 'reports/metrics/',
+                        reportFiles: 'pylint/*view*/,pep8/*view*/,python_complexity/*view*/,xsscommitlint/*view*/,xsslint/*view*/,eslint/*view*/',
+                        reportName: 'Quality Report', reportTitles: ''])
+                } finally {
+                    if (env.ghprbPullId != null) {
+                        // For PR jobs, run the edx-platform-test-notifier for PR reporting
+                        build job: 'edx-platform-test-notifier', parameters: [string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false
+                    } else {
+                        // For master jobs run github-build-status and report to slack when necessary
+                        if (currentBuild.currentResult == "SUCCESS") {
+                            create_deployment = "true"
+                            build_status = "success"
+                            build_description = "Build Passed"
+                        }
+                        else {
+                            create_deployment = "false"
+                            build_status = "failure"
+                            build_description = "Build Failed"
+                        }
+
+                        commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+                        build job: 'github-build-status', parameters: [
+                                string(name: 'GIT_SHA', value: commit_sha),
+                                string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
+                                string(name: 'TARGET_URL', value: "${BUILD_URL}"),
+                                string(name: 'DESCRIPTION', value: build_description), string(name: 'CONTEXT', value: 'jenkins/quality-pipeline-master'),
+                                string(name: 'CREATE_DEPLOYMENT', value: create_deployment), string(name: 'BUILD_STATUS', value: build_status)
+                            ],
+                            propagate: false, wait: false
+
+                        if (currentBuild.currentResult != "SUCCESS"){
+                            slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
+
+                            email_body = "See: <${BUILD_URL}>\n\nChanges:\n"
+                            change_sets = currentBuild.changeSets
+                            for (int j = 0; j < change_sets.size(); j++) {
+                                change_set_items = change_sets[j].items
+                                for (int k = 0; k < change_set_items.length; k++) {
+                                    item = change_set_items[k]
+                                    email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}"
+                                }
+                            }
+                            emailext body: email_body,
+                                subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
+                        } else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") {
+                            slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
+                            emailext body: "See <${BUILD_URL}>",
+                                subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
+                        }
+                    }
+                }
+            }
+        }
+    }
+}