diff --git a/scripts/Jenkinsfiles/bokchoy b/scripts/Jenkinsfiles/bokchoy
new file mode 100644
index 0000000000000000000000000000000000000000..77c95ea117ac406b87c32c28326fcf139fe22220
--- /dev/null
+++ b/scripts/Jenkinsfiles/bokchoy
@@ -0,0 +1,134 @@
+def runBokchoyTests() {
+ // Determine git refspec, branch, and clone type
+ git_shallow_clone = true
+ git_extensions = []
+ if (env.ghprbActualCommit) {
+ git_branch = "${ghprbActualCommit}"
+ git_refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+ } else {
+ git_branch = 'master'
+ git_refspec = "+refs/heads/master:refs/remotes/origin/master"
+ }
+ sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+ checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]],
+ doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true,
+ noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+ refspec: git_refspec, url: 'git@github.com:edx/edx-platform.git']]]
+ console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
+ dir('stdout') {
+ writeFile file: "${TEST_SUITE}-${SHARD}-stdout.log", text: console_output
+ }
+ }
+}
+
+def bokchoyTestCleanup() {
+ archiveArtifacts allowEmptyArchive: true, artifacts: 'test_root/log/**/*.log,stdout/*.log'
+ junit '**/reports/bok_choy/**/xunit.xml'
+}
+
+pipeline {
+ agent { label "jenkins-worker" }
+ options {
+ timestamps()
+ timeout(60)
+ }
+ stages {
+ stage('Mark build as pending on Github') {
+ when {
+ // Only run github-build-status for master builds
+ expression { env.ghprbActualCommit == null }
+ }
+ steps {
+ script {
+ commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+ build job: 'github-build-status',
+ parameters: [
+ string(name: 'GIT_SHA', value: commit_sha),
+ string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
+ string(name: 'TARGET_URL', value: "${BUILD_URL}"),
+ string(name: 'DESCRIPTION', value: 'Pending'), string(name: 'CONTEXT', value: 'jenkins/bokchoy-pipeline-master'),
+ string(name: 'CREATE_DEPLOYMENT', value: 'false'), string(name: 'BUILD_STATUS', value: 'pending')
+ ],
+ propagate: false, wait: false
+ }
+ }
+ }
+ stage("Run tests") {
+ steps {
+ script {
+ def parallel_stages = [:]
+ for (int i = 1; i <= 22; i++) {
+ int index = i
+ parallel_stages["${index}"] = {
+ node('jenkins-worker') {
+ withEnv(["SHARD=${index}","TEST_SUITE=bok-choy"]) {
+ try {
+ stage("Bokchoy shard: ${index}") {
+ runBokchoyTests()
+ }
+ } finally {
+ bokchoyTestCleanup()
+ }
+ }
+ }
+ }
+ }
+ parallel parallel_stages
+ }
+ }
+ }
+ }
+ post {
+ always {
+ script{
+ if (env.ghprbPullId != null) {
+ // For PR jobs, run the edx-platform-test-notifier for PR reporting
+ build job: 'edx-platform-test-notifier', parameters: [string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false
+ } else {
+ // For master jobs run github-build-status and report to slack when necessary
+ if (currentBuild.currentResult == "SUCCESS") {
+ create_deployment = "true"
+ build_status = "success"
+ build_description = "Build Passed"
+ }
+ else {
+ create_deployment = "false"
+ build_status = "failure"
+ build_description = "Build Failed"
+ }
+
+ commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+ build job: 'github-build-status',
+ parameters: [
+ string(name: 'GIT_SHA', value: commit_sha),
+ string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
+ string(name: 'TARGET_URL', value: "${BUILD_URL}"),
+ string(name: 'DESCRIPTION', value: build_description), string(name: 'CONTEXT', value: 'jenkins/bokchoy-pipeline-master'),
+ string(name: 'CREATE_DEPLOYMENT', value: create_deployment), string(name: 'BUILD_STATUS', value: build_status)
+ ],
+ propagate: false, wait: false
+
+ if (currentBuild.currentResult != "SUCCESS"){
+ slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
+
+ email_body = "See: <${BUILD_URL}>\n\nChanges:\n"
+ change_sets = currentBuild.changeSets
+ for (int j = 0; j < change_sets.size(); j++) {
+ change_set_items = change_sets[j].items
+ for (int k = 0; k < change_set_items.length; k++) {
+ item = change_set_items[k]
+ email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}"
+ }
+ }
+ emailext body: email_body,
+ subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
+ } else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") {
+ slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
+ emailext body: "See <${BUILD_URL}>",
+ subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/scripts/Jenkinsfiles/quality b/scripts/Jenkinsfiles/quality
new file mode 100644
index 0000000000000000000000000000000000000000..f920858139bd94e1e1900275b2d7a85ceb6fd6e9
--- /dev/null
+++ b/scripts/Jenkinsfiles/quality
@@ -0,0 +1,243 @@
+def runQualityTests() {
+ sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+ // Determine git refspec, branch, and clone type
+ git_shallow_clone = true
+ git_extensions = []
+ if (env.ghprbActualCommit) {
+ git_branch = "${ghprbActualCommit}"
+ refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+ if (SHARD == "4") {
+ git_shallow_clone = false
+ git_extensions.add([$class: 'WipeWorkspace'])
+ refspec = refspec + " +refs/heads/master:refs/remotes/origin/master"
+ }
+ } else {
+ git_branch = 'master'
+ refspec = "+refs/heads/master:refs/remotes/origin/master"
+ }
+ git_extensions.add([$class: 'CloneOption', honorRefspec: true, noTags: true, shallow: git_shallow_clone ])
+
+ checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]],
+ doGenerateSubmoduleConfigurations: false, extensions: git_extensions, submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+ refspec: refspec,
+ url: 'git@github.com:edx/edx-platform.git']]]
+
+ sh "bash scripts/all-tests.sh"
+ stash includes: '**/reports/**/*', name: "${TEST_SUITE}-${SHARD}-reports"
+ }
+}
+
+def qualityTestCleanup() {
+ archiveArtifacts allowEmptyArchive: true, artifacts: '**/reports/**/*,test_root/log/**/*.log,*.log'
+ junit 'reports/quality_junitxml/*.xml'
+}
+
+pipeline {
+ agent { label "jenkins-worker" }
+ options {
+ timestamps()
+ timeout(60)
+ }
+ stages {
+ stage('Mark build as pending on Github') {
+ when {
+ // Only run github-build-status for master builds
+ expression { env.ghprbActualCommit == null }
+ }
+ steps {
+ script {
+ commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+ build job: 'github-build-status',
+ parameters: [
+ string(name: 'GIT_SHA', value: commit_sha),
+ string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
+ string(name: 'TARGET_URL', value: "${BUILD_URL}"),
+ string(name: 'DESCRIPTION', value: 'Pending'), string(name: 'CONTEXT', value: 'jenkins/quality-pipeline-master'),
+ string(name: 'CREATE_DEPLOYMENT', value: 'false'), string(name: 'BUILD_STATUS', value: 'pending')
+ ],
+ propagate: false, wait: false
+ }
+ }
+ }
+ stage('Run Tests') {
+ parallel {
+ stage("commonlib pylint") {
+ agent { label "jenkins-worker" }
+ environment {
+ TEST_SUITE = "quality"
+ SHARD = 1
+ }
+ steps {
+ script {
+ runQualityTests()
+ }
+ }
+ post {
+ always {
+ script {
+ qualityTestCleanup()
+ }
+ }
+ }
+ }
+ stage("lms pylint") {
+ agent { label "jenkins-worker" }
+ environment {
+ TEST_SUITE = "quality"
+ SHARD = 2
+ }
+ steps {
+ script {
+ runQualityTests()
+ }
+ }
+ post {
+ always {
+ script {
+ qualityTestCleanup()
+ }
+ }
+ }
+ }
+ stage("cms/openedx/pavelib pylint") {
+ agent { label "jenkins-worker" }
+ environment {
+ TEST_SUITE = "quality"
+ SHARD = 3
+ }
+ steps {
+ script {
+ runQualityTests()
+ }
+ }
+ post {
+ always {
+ script {
+ qualityTestCleanup()
+ }
+ }
+ }
+ }
+ stage("Other quality checks") {
+ agent { label "jenkins-worker" }
+ environment {
+ TEST_SUITE = "quality"
+ SHARD = 4
+ }
+ steps {
+ script {
+ runQualityTests()
+ }
+ }
+ post {
+ always {
+ script {
+ qualityTestCleanup()
+ }
+ }
+ }
+ }
+ }
+ }
+ stage('Diff quality') {
+ when {
+ // Only run diff quality on PR builds
+ expression { env.ghprbTargetBranch != null }
+ }
+ environment {
+ TARGET_BRANCH = "origin/${ghprbTargetBranch}"
+ }
+ steps {
+ sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+ checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: "${ghprbActualCommit}"]],
+ doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption',
+ honorRefspec: true, noTags: true, shallow: false], [$class: 'WipeWorkspace']], submoduleCfg: [],
+ userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+ refspec: "+refs/heads/${ghprbTargetBranch}:refs/remotes/origin/${ghprbTargetBranch} +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*",
+ url: 'git@github.com:edx/edx-platform.git']]]
+ unstash 'quality-1-reports'
+ unstash 'quality-2-reports'
+ unstash 'quality-3-reports'
+ unstash 'quality-4-reports'
+ sh "./scripts/jenkins-quality-diff.sh"
+ }
+ }
+ post {
+ always {
+ qualityTestCleanup()
+ publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'reports/diff_quality',
+ reportFiles: 'diff_quality_pylint.html, diff_quality_eslint.html', reportName: 'Diff Quality Report', reportTitles: ''])
+ }
+ }
+ }
+ }
+ post {
+ always {
+ script{
+ try {
+ unstash 'quality-1-reports'
+ unstash 'quality-2-reports'
+ unstash 'quality-3-reports'
+ unstash 'quality-4-reports'
+ // Check for warnings
+ warnings canComputeNew: false, canResolveRelativePaths: false, canRunOnFailed: true, categoriesPattern: '',
+ defaultEncoding: '', excludePattern: '', healthy: '', includePattern: '', messagesPattern: '',
+ parserConfigurations: [[parserName: 'Pep8', pattern: 'reports/pep8/pep8.report'], [parserName: 'PyLint', pattern: 'reports/**/pylint.report']],
+ unHealthy: ''
+ // Publish Quality report
+ publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true,
+ reportDir: 'reports/metrics/',
+ reportFiles: 'pylint/*view*/,pep8/*view*/,python_complexity/*view*/,xsscommitlint/*view*/,xsslint/*view*/,eslint/*view*/',
+ reportName: 'Quality Report', reportTitles: ''])
+ } finally {
+ if (env.ghprbPullId != null) {
+ // For PR jobs, run the edx-platform-test-notifier for PR reporting
+ build job: 'edx-platform-test-notifier', parameters: [string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false
+ } else {
+ // For master jobs run github-build-status and report to slack when necessary
+ if (currentBuild.currentResult == "SUCCESS") {
+ create_deployment = "true"
+ build_status = "success"
+ build_description = "Build Passed"
+ }
+ else {
+ create_deployment = "false"
+ build_status = "failure"
+ build_description = "Build Failed"
+ }
+
+ commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+ build job: 'github-build-status', parameters: [
+ string(name: 'GIT_SHA', value: commit_sha),
+ string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
+ string(name: 'TARGET_URL', value: "${BUILD_URL}"),
+ string(name: 'DESCRIPTION', value: build_description), string(name: 'CONTEXT', value: 'jenkins/quality-pipeline-master'),
+ string(name: 'CREATE_DEPLOYMENT', value: create_deployment), string(name: 'BUILD_STATUS', value: build_status)
+ ],
+ propagate: false, wait: false
+
+ if (currentBuild.currentResult != "SUCCESS"){
+ slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
+
+ email_body = "See: <${BUILD_URL}>\n\nChanges:\n"
+ change_sets = currentBuild.changeSets
+ for (int j = 0; j < change_sets.size(); j++) {
+ change_set_items = change_sets[j].items
+ for (int k = 0; k < change_set_items.length; k++) {
+ item = change_set_items[k]
+ email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}"
+ }
+ }
+ emailext body: email_body,
+ subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
+ } else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") {
+ slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
+ emailext body: "See <${BUILD_URL}>",
+ subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
+ }
+ }
+ }
+ }
+ }
+ }
+}