diff --git a/common/test/acceptance/.coveragerc b/common/test/acceptance/.coveragerc
index 975bb71929747115c0fd9a5866e7f9f76db946c3..9d19a5b09ada6857b10b85d103d03183241d0a0c 100644
--- a/common/test/acceptance/.coveragerc
+++ b/common/test/acceptance/.coveragerc
@@ -10,21 +10,32 @@ source =
omit =
lms/envs/*
cms/envs/*
+ cms/manage.py
+ cms/djangoapps/contentstore/views/dev.py
common/djangoapps/terrain/*
common/djangoapps/*/migrations/*
+ openedx/core/djangoapps/debug/*
openedx/core/djangoapps/*/migrations/*
*/test*
*/management/*
*/urls*
*/wsgi*
+ lms/debug/*
+ lms/djangoapps/*/features/*
lms/djangoapps/*/migrations/*
+ cms/djangoapps/*/features/*
cms/djangoapps/*/migrations/*
+concurrency = multiprocessing
parallel = True
[report]
ignore_errors = True
+exclude_lines =
+ pragma: no cover
+ raise NotImplementedError
+
[html]
title = Bok Choy Test Coverage Report
directory = reports/bok_choy/cover
diff --git a/common/test/acceptance/tests/discussion/test_discussion.py b/common/test/acceptance/tests/discussion/test_discussion.py
index cd8648a74df7b43cfb9de573e4471851ee71a304..cda7583a94660cc9938603d93c2022a612f56e26 100644
--- a/common/test/acceptance/tests/discussion/test_discussion.py
+++ b/common/test/acceptance/tests/discussion/test_discussion.py
@@ -1226,6 +1226,7 @@ class DiscussionUserProfileTest(UniqueCourseTest):
self.profiled_user_id = self.setup_user(username=self.PROFILED_USERNAME)
# now create a second user who will view the profile.
self.user_id = self.setup_user()
+ UserProfileViewFixture([]).push()
def setup_course(self):
"""
diff --git a/docs/testing.rst b/docs/testing.rst
index f03ac03e29ac8a3d2fc159a8b394194ad4864f52..a97c42d960182315dff8207a4b157e4097e3d674 100644
--- a/docs/testing.rst
+++ b/docs/testing.rst
@@ -437,14 +437,14 @@ To test only a certain feature, specify the file and the testcase class.
::
- paver test_bokchoy -t studio/test_studio_bad_data.py:BadComponentTest
+ paver test_bokchoy -t studio/test_studio_bad_data.py::BadComponentTest
To execute only a certain test case, specify the file name, class, and
test case method.
::
- paver test_bokchoy -t lms/test_lms.py:RegistrationTest.test_register
+ paver test_bokchoy -t lms/test_lms.py::RegistrationTest::test_register
During acceptance test execution, log files and also screenshots of
failed tests are captured in test\_root/log.
@@ -454,7 +454,7 @@ If you check this in, your tests will hang on jenkins.
::
- from nose.tools import set_trace; set_trace()
+ import pdb; pdb.set_trace()
By default, all bokchoy tests are run with the 'split' ModuleStore. To
override the modulestore that is used, use the default\_store option.
@@ -506,7 +506,7 @@ relative to the ``common/test/acceptance/tests`` directory. This is an example f
::
- paver test_a11y -t lms/test_lms_dashboard.py:LmsDashboardA11yTest.test_dashboard_course_listings_a11y
+ paver test_a11y -t lms/test_lms_dashboard.py::LmsDashboardA11yTest::test_dashboard_course_listings_a11y
**Coverage**:
@@ -644,7 +644,7 @@ Running Tests on Paver Scripts
To run tests on the scripts that power the various Paver commands, use the following command::
- nosetests paver
+ nosetests pavelib
Testing internationalization with dummy translations
diff --git a/pavelib/paver_tests/test_paver_bok_choy_cmds.py b/pavelib/paver_tests/test_paver_bok_choy_cmds.py
index fdd2a4263e2830ab32c2c836f8d27bd429546a15..c04534ad0a36635f1a35d3ccb0855d7f8248e847 100644
--- a/pavelib/paver_tests/test_paver_bok_choy_cmds.py
+++ b/pavelib/paver_tests/test_paver_bok_choy_cmds.py
@@ -11,6 +11,7 @@ import ddt
from mock import Mock, call, patch
from paver.easy import BuildFailure, call_task, environment
+from pavelib.utils.envs import Env
from pavelib.utils.test.suites import BokChoyTestSuite, Pa11yCrawler
from pavelib.utils.test.suites.bokchoy_suite import DEMO_COURSE_IMPORT_DIR, DEMO_COURSE_TAR_GZ
@@ -40,10 +41,15 @@ class TestPaverBokChoyCmd(unittest.TestCase):
),
"SELENIUM_DRIVER_LOG_DIR='{}/test_root/log{}'".format(REPO_DIR, shard_str),
"VERIFY_XSS='{}'".format(verify_xss),
- "nosetests",
+ "coverage",
+ "run",
+ "--rcfile={}".format(Env.BOK_CHOY_COVERAGERC),
+ "-m",
+ "pytest",
"{}/common/test/acceptance/{}".format(REPO_DIR, name),
- "--xunit-file={}/reports/bok_choy{}/xunit.xml".format(REPO_DIR, shard_str),
- "--verbosity=2",
+ "--durations=20",
+ "--junitxml={}/reports/bok_choy{}/xunit.xml".format(REPO_DIR, shard_str),
+ "--verbose",
]
return expected_statement
@@ -122,11 +128,11 @@ class TestPaverBokChoyCmd(unittest.TestCase):
Using 1 process means paver should ask for the traditional xunit plugin for plugin results
"""
expected_verbosity_command = [
- "--xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+ "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else ''
),
- "--verbosity=2",
+ "--verbose",
]
suite = BokChoyTestSuite('', num_processes=1)
self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)
@@ -138,13 +144,13 @@ class TestPaverBokChoyCmd(unittest.TestCase):
"""
process_count = 2
expected_verbosity_command = [
- "--xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+ "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else '',
),
- "--processes={}".format(process_count),
- "--no-color",
- "--process-timeout=1200",
+ "-n {}".format(process_count),
+ "--color=no",
+ "--verbose",
]
suite = BokChoyTestSuite('', num_processes=process_count)
self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)
@@ -155,27 +161,17 @@ class TestPaverBokChoyCmd(unittest.TestCase):
"""
process_count = 3
expected_verbosity_command = [
- "--xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+ "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else '',
),
- "--processes={}".format(process_count),
- "--no-color",
- "--process-timeout=1200",
+ "-n {}".format(process_count),
+ "--color=no",
+ "--verbose",
]
suite = BokChoyTestSuite('', num_processes=process_count)
self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)
- def test_invalid_verbosity_and_processes(self):
- """
- If an invalid combination of verbosity and number of processors is passed in, a
- BuildFailure should be raised
- """
- suite = BokChoyTestSuite('', num_processes=2, verbosity=3)
- with self.assertRaises(BuildFailure):
- # pylint: disable=pointless-statement
- suite.verbosity_processes_command
-
@ddt.ddt
class TestPaverPa11yCrawlerCmd(unittest.TestCase):
diff --git a/pavelib/paver_tests/test_utils.py b/pavelib/paver_tests/test_utils.py
index 65bdc966a651af578e86315312111ee77c62eaff..cccd922f74cba283949184e476e71fca0e99f51b 100644
--- a/pavelib/paver_tests/test_utils.py
+++ b/pavelib/paver_tests/test_utils.py
@@ -6,9 +6,11 @@ import unittest
from mock import patch
+from pavelib.utils.envs import Env
from pavelib.utils.test.utils import MINIMUM_FIREFOX_VERSION, check_firefox_version
+@unittest.skipIf(Env.USING_DOCKER, 'Firefox version check works differently under Docker Devstack')
class TestUtils(unittest.TestCase):
"""
Test utils.py under pavelib/utils/test
diff --git a/pavelib/utils/test/bokchoy_options.py b/pavelib/utils/test/bokchoy_options.py
index 8631da945f792e66e04e3af0e9e5e41e0ebff328..48ec319a9434615c3f16d12a59a9e6fa996d33c9 100644
--- a/pavelib/utils/test/bokchoy_options.py
+++ b/pavelib/utils/test/bokchoy_options.py
@@ -19,8 +19,16 @@ BOKCHOY_DEFAULT_STORE_DEPR = make_option(
default=os.environ.get('DEFAULT_STORE', 'split'),
help='deprecated in favor of default-store'
)
-BOKCHOY_FASTTEST = make_option('-a', '--fasttest', action='store_true', help='Skip some setup')
-BOKCHOY_COVERAGERC = make_option('--coveragerc', help='coveragerc file to use during this test')
+BOKCHOY_EVAL_ATTR = make_option(
+ "-a", "--eval-attr",
+ dest="eval_attr", help="Only run tests matching given attribute expression."
+)
+BOKCHOY_FASTTEST = make_option('--fasttest', action='store_true', help='Skip some setup')
+BOKCHOY_COVERAGERC = make_option(
+ '--coveragerc',
+ default=Env.BOK_CHOY_COVERAGERC,
+ help='coveragerc file to use during this test'
+)
BOKCHOY_OPTS = [
('test-spec=', 't', 'Specific test to run'),
@@ -28,7 +36,9 @@ BOKCHOY_OPTS = [
('skip-clean', 'C', 'Skip cleaning repository before running tests'),
make_option('-r', '--serversonly', action='store_true', help='Prepare suite and leave servers running'),
make_option('-o', '--testsonly', action='store_true', help='Assume servers are running and execute tests only'),
+ BOKCHOY_COVERAGERC,
BOKCHOY_DEFAULT_STORE,
+ BOKCHOY_EVAL_ATTR,
make_option(
'-d', '--test-dir',
default='tests',
diff --git a/pavelib/utils/test/suites/bokchoy_suite.py b/pavelib/utils/test/suites/bokchoy_suite.py
index 6c05afe93c141ba77fd6637fd10f2516ebf69f7e..453e4dc19f93d193cb24d17c7e1858acc6ccacc6 100644
--- a/pavelib/utils/test/suites/bokchoy_suite.py
+++ b/pavelib/utils/test/suites/bokchoy_suite.py
@@ -179,10 +179,11 @@ class BokChoyTestSuite(TestSuite):
testsonly - assume servers are running (as per above) and run tests with no setup or cleaning of environment
test_spec - when set, specifies test files, classes, cases, etc. See platform doc.
default_store - modulestore to use when running tests (split or draft)
+ eval_attr - only run tests matching given attribute expression
num_processes - number of processes or threads to use in tests. Recommendation is that this
is less than or equal to the number of available processors.
verify_xss - when set, check for XSS vulnerabilities in the page HTML.
- See nosetest documentation: http://nose.readthedocs.org/en/latest/usage.html
+ See pytest documentation: https://docs.pytest.org/en/latest/
"""
def __init__(self, *args, **kwargs):
super(BokChoyTestSuite, self).__init__(*args, **kwargs)
@@ -196,6 +197,7 @@ class BokChoyTestSuite(TestSuite):
self.testsonly = kwargs.get('testsonly', False)
self.test_spec = kwargs.get('test_spec', None)
self.default_store = kwargs.get('default_store', None)
+ self.eval_attr = kwargs.get('eval_attr', None)
self.verbosity = kwargs.get('verbosity', DEFAULT_VERBOSITY)
self.num_processes = kwargs.get('num_processes', DEFAULT_NUM_PROCESSES)
self.verify_xss = kwargs.get('verify_xss', os.environ.get('VERIFY_XSS', True))
@@ -203,7 +205,7 @@ class BokChoyTestSuite(TestSuite):
self.har_dir = self.log_dir / 'hars'
self.a11y_file = Env.BOK_CHOY_A11Y_CUSTOM_RULES_FILE
self.imports_dir = kwargs.get('imports_dir', None)
- self.coveragerc = kwargs.get('coveragerc', None)
+ self.coveragerc = kwargs.get('coveragerc', Env.BOK_CHOY_COVERAGERC)
self.save_screenshots = kwargs.get('save_screenshots', False)
def __enter__(self):
@@ -269,29 +271,22 @@ class BokChoyTestSuite(TestSuite):
@property
def verbosity_processes_command(self):
"""
- Multiprocessing, xunit, color, and verbosity do not work well together. We need to construct
- the proper combination for use with nosetests.
+ Construct the proper combination of multiprocessing, XUnit XML file, color, and verbosity for use with pytest.
"""
- command = []
-
- if self.verbosity != DEFAULT_VERBOSITY and self.num_processes != DEFAULT_NUM_PROCESSES:
- msg = 'Cannot pass in both num_processors and verbosity. Quitting'
- raise BuildFailure(msg)
+ command = ["--junitxml={}".format(self.xunit_report)]
if self.num_processes != 1:
- # Construct "multiprocess" nosetest command
- command = [
- "--xunitmp-file={}".format(self.xunit_report),
- "--processes={}".format(self.num_processes),
- "--no-color",
- "--process-timeout=1200",
- ]
-
- else:
- command = [
- "--xunit-file={}".format(self.xunit_report),
- "--verbosity={}".format(self.verbosity),
+ # Construct "multiprocess" pytest command
+ command += [
+ "-n {}".format(self.num_processes),
+ "--color=no",
]
+ if self.verbosity < 1:
+ command.append("--quiet")
+ elif self.verbosity > 1:
+ command.append("--verbose")
+ if self.eval_attr:
+ command.append("-a '{}'".format(self.eval_attr))
return command
@@ -300,7 +295,7 @@ class BokChoyTestSuite(TestSuite):
Infinite loop. Servers will continue to run in the current session unless interrupted.
"""
print 'Bok-choy servers running. Press Ctrl-C to exit...\n'
- print 'Note: pressing Ctrl-C multiple times can corrupt noseid files and system state. Just press it once.\n'
+ print 'Note: pressing Ctrl-C multiple times can corrupt system state. Just press it once.\n'
while True:
try:
@@ -312,7 +307,7 @@ class BokChoyTestSuite(TestSuite):
@property
def cmd(self):
"""
- This method composes the nosetests command to send to the terminal. If nosetests aren't being run,
+ This method composes the pytest command to send to the terminal. If pytest isn't being run,
the command returns None.
"""
# Default to running all tests if no specific test is specified
@@ -321,12 +316,12 @@ class BokChoyTestSuite(TestSuite):
else:
test_spec = self.test_dir / self.test_spec
- # Skip any additional commands (such as nosetests) if running in
+ # Skip any additional commands (such as pytest) if running in
# servers only mode
if self.serversonly:
return None
- # Construct the nosetests command, specifying where to save
+ # Construct the pytest command, specifying where to save
# screenshots and XUnit XML reports
cmd = [
"DEFAULT_STORE={}".format(self.default_store),
@@ -335,11 +330,21 @@ class BokChoyTestSuite(TestSuite):
"BOKCHOY_A11Y_CUSTOM_RULES_FILE='{}'".format(self.a11y_file),
"SELENIUM_DRIVER_LOG_DIR='{}'".format(self.log_dir),
"VERIFY_XSS='{}'".format(self.verify_xss),
- "nosetests",
+ ]
+ if self.save_screenshots:
+ cmd.append("NEEDLE_SAVE_BASELINE=True")
+ cmd += [
+ "coverage",
+ "run",
+ ]
+ if self.coveragerc:
+ cmd.append("--rcfile={}".format(self.coveragerc))
+ cmd += [
+ "-m",
+ "pytest",
test_spec,
+ "--durations=20",
] + self.verbosity_processes_command
- if self.save_screenshots:
- cmd.append("--with-save-baseline")
if self.extra_args:
cmd.append(self.extra_args)
cmd.extend(self.passthrough_options)
diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt
index 248bdd8c261f67edff5d7fc346e6825b7cc658a6..767d019b8bda47109e646b84f4966f3a2a5e24cd 100644
--- a/requirements/edx/testing.txt
+++ b/requirements/edx/testing.txt
@@ -9,4 +9,11 @@
# * @edx/testeng - to discuss it's impact on test infrastructure
# * @edx/devops - to check system requirements
+execnet==1.4.1
+py==1.4.34
pysqlite==2.8.3
+pytest==3.1.3
+pytest-attrib==0.1.3
+pytest-catchlog==1.2.2
+pytest-django==3.1.2
+pytest-xdist==1.18.1
diff --git a/scripts/accessibility-tests.sh b/scripts/accessibility-tests.sh
index 6597cb21fc49ae90aff5356d77f87be33656c9cb..41dbb8e0ca3cdde4b3297311878f53f7f55f6531 100755
--- a/scripts/accessibility-tests.sh
+++ b/scripts/accessibility-tests.sh
@@ -6,7 +6,7 @@ echo "Setting up for accessibility tests..."
source scripts/jenkins-common.sh
echo "Running explicit accessibility tests..."
-SELENIUM_BROWSER=phantomjs paver test_a11y --with-xunitmp
+SELENIUM_BROWSER=phantomjs paver test_a11y
echo "Generating coverage report..."
paver a11y_coverage
diff --git a/scripts/generic-ci-tests.sh b/scripts/generic-ci-tests.sh
index 392d8374926fb397ee0f71177ab5a59da52ea22a..ea87a18ebd2b12034d83f429afc88523e939969a 100755
--- a/scripts/generic-ci-tests.sh
+++ b/scripts/generic-ci-tests.sh
@@ -162,7 +162,7 @@ case "$TEST_SUITE" in
"bok-choy")
- PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS --with-flaky --with-xunit"
+ PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS"
case "$SHARD" in
@@ -171,11 +171,11 @@ case "$TEST_SUITE" in
;;
[1-9]|10)
- paver test_bokchoy --attr="shard=$SHARD" $PAVER_ARGS
+ paver test_bokchoy --eval-attr="shard==$SHARD" $PAVER_ARGS
;;
11|"noshard")
- paver test_bokchoy --attr='!shard,a11y=False' $PAVER_ARGS
+ paver test_bokchoy --eval-attr='not shard and not a11y' $PAVER_ARGS
;;
# Default case because if we later define another bok-choy shard on Jenkins
@@ -190,7 +190,7 @@ case "$TEST_SUITE" in
# May be unnecessary if we changed the "Skip if there are no test files"
# option to True in the jenkins job definitions.
mkdir -p reports/bok_choy
- emptyxunit "bok_choy/nosetests"
+ emptyxunit "bok_choy/xunit"
;;
esac
;;
diff --git a/setup.cfg b/setup.cfg
index 673bf03b017eacfb82302ab3ccafde56dd5fdad6..9123e9f69a64066c5aede8c584058f66631c8902 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -14,6 +14,9 @@ process-timeout=300
#nocapture=1
#pdb=1
+[tool:pytest]
+norecursedirs = .git conf node_modules test_root cms/envs lms/envs
+
[pep8]
# error codes: http://pep8.readthedocs.org/en/latest/intro.html#error-codes
# E501: line too long