diff --git a/requirements/edx/development.in b/requirements/edx/development.in index 1b5cdae5f9ec6aa2a6ab396ebd61adea9a4a9683..7cbf550c8dabe5c2b42bf75d70cccac8f8293f05 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -17,7 +17,6 @@ click # Used for perf_tests utilities in modulesto django-debug-toolbar==1.8 # A set of panels that display debug information about the current request/response edx-sphinx-theme # Documentation theme pyinotify # More efficient checking for runserver reload trigger events -snakefood # Lists dependencies between Python modules, used in scripts/dependencies/* sphinx==1.8.5 # Pinned because 2.0.0 release requires Python '>=3.5' but current Python is 2.7.12 vulture # Detects possible dead/unused code, used in scripts/find-dead-code.sh modernize # Used to make Python 2 code more modern with the intention of eventually porting it over to Python 3. diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 22abbc87f9879c80b0fda9752f5096f034a1a04e..ef279b18220d0d18d73d8bc7821abf095be25ae6 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -303,7 +303,6 @@ simplejson==3.16.0 singledispatch==3.4.0.3 six==1.12.0 slumber==0.7.1 -snakefood==1.4 snowballstemmer==1.9.1 # via sphinx social-auth-app-django==2.1.0 social-auth-core==1.7.0 diff --git a/scripts/dependencies/development.py b/scripts/dependencies/development.py deleted file mode 100755 index 506d7c34d81dad0d72d5f0a5e5bf7706835c4924..0000000000000000000000000000000000000000 --- a/scripts/dependencies/development.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -""" -List any dependencies on development utilities in edx-platform from -non-development modules. Generally, there shouldn't be any; such a dependency -could result in ImportErrors in production or tests where development packages -aren't installed. - -This script counts on scripts/dependencies/enumerate.sh having already -been run in order to generate a dependency data file to work from. -""" -from __future__ import absolute_import, print_function - -import os -import re -import sys - -# Enumerate all the Python modules that should only be imported from development utilities -pattern_fragments = [ - # Development utility modules within edx-platform - r'^xmodule/modulestore/perf_tests' # modulestore performance tests - - # Development-only package dependencies - r'^debug_toolbar', # django-debug-toolbar -] - -dev_pattern = re.compile('|'.join(pattern_fragments)) - -data_path = 'reports/dependencies/dependencies.txt' -if not os.path.exists(data_path): - print('The dependencies data file is unavailable; run scripts/dependencies/enumerate.sh first.') - sys.exit(1) -exit_status = 0 -with open(data_path, 'r') as f: - for dep in map(eval, f): - (from_root, from_name), (to_root, to_name) = dep - if to_name is None: - continue - if dev_pattern.search(to_name) and not dev_pattern.search(from_name): - # We usually don't care about dependencies between modules in site-packages - if from_root.endswith(u'site-packages') and to_root.endswith(u'site-packages'): - continue - # The django-debug-toolbar URL imports are safely behind conditions on INSTALLED_APPS - if from_name in {u'cms/urls.py', u'lms/urls.py'} and to_name == u'debug_toolbar': - continue - print(dep) - exit_status = 1 -sys.exit(exit_status) diff --git a/scripts/dependencies/enumerate.sh b/scripts/dependencies/enumerate.sh deleted file mode 100755 index 1ea2ae970e8fe6c4ade8ae633588e6940f8934af..0000000000000000000000000000000000000000 --- a/scripts/dependencies/enumerate.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -e - -############################################################################ -# -# enumerate.sh -# -# Enumerates all dependencies (imports) from Python modules in -# edx-platform. The resulting data file generated at -# reports/dependencies/dependencies.txt can then be used by other scripts -# to detect inappropriate imports, such as: -# -# * Imports of test modules or testing packages from core application code -# * Imports of development-only packages from core or test code -# * Imports from a package we want to stop using as a dependency -# * Imports of other edx-platform modules from a module we want to move to -# a separate package in its own repository -# -# This script can take a while to run (a few minutes), so it should be run -# independently of the other scripts which use this data. -# -# While running, a number of warnings such as "Could not import module -# 'assert_equal'" may be generated. This is normal; the snakefood utility -# can't really distinguish between the import of a module and the import of -# an object within a module, so it prints a warning on all instances of the -# latter just in case it actually was an attempt to import a module which -# it couldn't find in the current PYTHONPATH. If you do see some modules -# listed which you think should be findable, you may need to run -# "make requirements" or update the ROOTS variable in this script. -# -############################################################################ - -OUTPUT_DIR="reports/dependencies" -mkdir -p ${OUTPUT_DIR} -DEPENDENCIES=${OUTPUT_DIR}/dependencies.txt -ROOTS=cms/djangoapps:common/djangoapps:lms/djangoapps:scripts/xsslint -PYTHONPATH=${ROOTS} sfood cms common lms openedx pavelib scripts manage.py pavement.py > ${DEPENDENCIES} diff --git a/scripts/dependencies/from_package.py b/scripts/dependencies/from_package.py deleted file mode 100755 index 614fa49bd9cd02c6b60118efdde6d9bee38ae3d2..0000000000000000000000000000000000000000 --- a/scripts/dependencies/from_package.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -""" -List any modules that are imported from the given package. This can be used -to determine what needs to be refactored to allow a package to be broken out -into a separately installed package. The package argument to the script -should be formatted as shown in these examples: - -* scripts/dependencies/from_package.py xmodule -* scripts/dependencies/from_package.py openedx/features/course_experience -* scripts/dependencies/from_package.py cms/djangoapps/verify_student - -This script counts on scripts/dependencies/enumerate.sh having already -been run in order to generate a dependency data file to work from. -""" -from __future__ import absolute_import, print_function - -import os -import re -import sys - -pattern = re.compile(u'^{}'.format(sys.argv[1])) - -data_path = 'reports/dependencies/dependencies.txt' -if not os.path.exists(data_path): - print('The dependencies data file is unavailable; run scripts/dependencies/enumerate.sh first.') -with open(data_path, 'r') as f: - for dep in map(eval, f): - (from_root, from_name), (to_root, to_name) = dep - if to_name is None: - continue - if pattern.search(from_name) and not pattern.search(to_name): - # We usually don't care about dependencies between modules in site-packages - if from_root.endswith(u'site-packages') and to_root.endswith(u'site-packages'): - continue - # We don't really care about dependencies on the standard library - if to_root.startswith('/usr/lib/python') or to_root.endswith('lib/python2.7'): - continue - print(dep) diff --git a/scripts/dependencies/on_package.py b/scripts/dependencies/on_package.py deleted file mode 100755 index 77ad4e9c1f998993226a9bb1390690427596fa08..0000000000000000000000000000000000000000 --- a/scripts/dependencies/on_package.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -""" -List any modules that import code from the given package. This can be used -to determine if the package can be safely removed, or just to understand -what context it's used in. The package argument to the script should be -formatted as shown in these examples: - -* scripts/dependencies/on_package.py nose -* scripts/dependencies/on_package.py third_parth_auth -* scripts/dependencies/on_package.py cms/djangoapps/verify_student - -This script counts on scripts/dependencies/enumerate.sh having already -been run in order to generate a dependency data file to work from. -""" -from __future__ import absolute_import, print_function - -import os -import re -import sys - -pattern = re.compile(u'^{}'.format(sys.argv[1])) - -data_path = 'reports/dependencies/dependencies.txt' -if not os.path.exists(data_path): - print('The dependencies data file is unavailable; run scripts/dependencies/enumerate.sh first.') -with open(data_path, 'r') as f: - for dep in map(eval, f): - (from_root, from_name), (to_root, to_name) = dep - if to_name is None: - continue - if pattern.search(to_name) and not pattern.search(from_name): - # We usually don't care about dependencies between modules in site-packages - if from_root.endswith(u'site-packages') and to_root.endswith(u'site-packages'): - continue - print(dep) diff --git a/scripts/dependencies/testing.py b/scripts/dependencies/testing.py deleted file mode 100755 index dfd4409d928ec911e020b4f62a29cfa24da5f358..0000000000000000000000000000000000000000 --- a/scripts/dependencies/testing.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -""" -List any dependencies on test modules in edx-platform from non-test modules. -Generally, there shouldn't be any; such a dependency could result in -ImportErrors in production where testing packages aren't installed. - -This script counts on scripts/dependencies/enumerate.sh having already -been run in order to generate a dependency data file to work from. -""" -from __future__ import absolute_import, print_function - -import os -import re -import sys - -# Enumerate all the Python modules that should only be imported during test runs -pattern_fragments = [ - # Test modules within edx-platform - r'/tests?\.py', # test.py, tests.py - r'/tests?_[^/]*\.py', # test_*.py, tests_*.py - r'/[^/]*_tests\.py', # *_tests.py - r'/tests?/', # */test/*, */tests/* - r'[cl]ms/.*/features/', # cms/*/features/*, lms/*/features/* - r'/testing\.py', # testing.py - r'/testutils\.py', # testutils.py - r'/tests$', # tests/__init__.py - r'conftest\.py', # conftest.py - r'/factories\.py', # factories.py - r'^terrain', # terrain/* - r'/setup_models_to_send_test_emails\.py', # setup_models_to_send_test_emails management command - - # Testing-only package dependencies - r'^bs4', # beautifulsoup4 - r'^before_after$', # before_after - r'^bok_choy', # bok-choy - r'^cssselect', # cssselect - r'^factory', # factory_boy - r'^freezegun', # freezegun - r'^httpretty', # httpretty - r'^moto', # moto - r'^nose', # nose - r'^pyquery', # pyquery - r'^pytest.py$', # pytest - r'^selenium', # selenium - r'^singledispatch', # singledispatch - r'^testfixtures', # testfixtures -] - -test_pattern = re.compile('|'.join(pattern_fragments)) - -data_path = 'reports/dependencies/dependencies.txt' -if not os.path.exists(data_path): - print('The dependencies data file is unavailable; run scripts/dependencies/enumerate.sh first.') - sys.exit(1) -exit_status = 0 -with open(data_path, 'r') as f: - for dep in map(eval, f): - (from_root, from_name), (to_root, to_name) = dep - if to_name is None: - continue - if test_pattern.search(to_name) and not test_pattern.search(from_name): - # snakefood sometimes picks a weird place to split the root path and filename - if from_root.endswith('/tests'): - continue - # We usually don't care about dependencies between modules in site-packages - if from_root.endswith(u'site-packages') and to_root.endswith(u'site-packages'): - continue - # Dependencies on django.test and waffle.testutils are ok - if to_name.startswith(u'django/test') or to_name == u'waffle/testutils.py': - continue - # Dependencies within pavelib are ok - if from_name.startswith(u'pavelib') and to_name.startswith(u'pavelib'): - continue - print(dep) - exit_status = 1 -sys.exit(exit_status)