diff --git a/cms/envs/production.py b/cms/envs/production.py index 018ecca4e82b3882e2fba219e9a497efc21f9112..b4604bf34eff909a9a277f0dd78c9c0e9f360a34 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -208,6 +208,12 @@ if 'loc_cache' not in CACHES: if 'staticfiles' in CACHES: CACHES['staticfiles']['KEY_PREFIX'] = EDX_PLATFORM_REVISION +# In order to transition from local disk asset storage to S3 backed asset storage, +# we need to run asset collection twice, once for local disk and once for S3. +# Once we have migrated to service assets off S3, then we can convert this back to +# managed by the yaml file contents +STATICFILES_STORAGE = os.environ.get('STATICFILES_STORAGE', ENV_TOKENS.get('STATICFILES_STORAGE', STATICFILES_STORAGE)) + SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN') SESSION_COOKIE_HTTPONLY = ENV_TOKENS.get('SESSION_COOKIE_HTTPONLY', True) SESSION_ENGINE = ENV_TOKENS.get('SESSION_ENGINE', SESSION_ENGINE) diff --git a/lms/djangoapps/instructor/tests/test_api.py b/lms/djangoapps/instructor/tests/test_api.py index 3408f52b4a138bae9030d936ad44477a38224c68..b1d56fd4230490850c12e9f90b97929b7cce4d21 100644 --- a/lms/djangoapps/instructor/tests/test_api.py +++ b/lms/djangoapps/instructor/tests/test_api.py @@ -3297,7 +3297,7 @@ class TestInstructorAPILevelsDataDump(SharedModuleStoreTestCase, LoginEnrollment self.assertIn("attachment; filename=org", response['Content-Disposition']) @patch('lms.djangoapps.instructor_task.models.logger.error') - @patch.dict(settings.GRADES_DOWNLOAD, {'STORAGE_TYPE': 's3'}) + @patch.dict(settings.GRADES_DOWNLOAD, {'STORAGE_TYPE': 's3', 'ROOT_PATH': 'tmp/edx-s3/grades'}) def test_list_report_downloads_error(self, mock_error): """ Tests the Rate-Limit exceeded is handled and does not raise 500 error. diff --git a/lms/djangoapps/instructor_task/tests/test_models.py b/lms/djangoapps/instructor_task/tests/test_models.py index 5eab482b9f2026d1b817a4b4e594ad10454b1b17..770b234c6ae86fd0b90d355ea40761eb19fb5e80 100644 --- a/lms/djangoapps/instructor_task/tests/test_models.py +++ b/lms/djangoapps/instructor_task/tests/test_models.py @@ -84,7 +84,11 @@ class LocalFSReportStoreTestCase(ReportStoreTestMixin, TestReportMixin, SimpleTe return ReportStore.from_config(config_name='GRADES_DOWNLOAD') -@patch.dict(settings.GRADES_DOWNLOAD, {'STORAGE_TYPE': 's3'}) +@patch.dict(settings.GRADES_DOWNLOAD, { + 'STORAGE_TYPE': 's3', + # Strip the leading `/`, because boto doesn't want it + 'ROOT_PATH': settings.GRADES_DOWNLOAD['ROOT_PATH'].lstrip('/') +}) class S3ReportStoreTestCase(MockS3Mixin, ReportStoreTestMixin, TestReportMixin, SimpleTestCase): """ Test the old S3ReportStore configuration. diff --git a/lms/envs/production.py b/lms/envs/production.py index 7cd00b77b9a5efe2ead3a2f503938ef34e2fda02..8292f1a66e77babf2bc7f965cc8e5b3dd696b66a 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -244,6 +244,12 @@ if 'loc_cache' not in CACHES: if 'staticfiles' in CACHES: CACHES['staticfiles']['KEY_PREFIX'] = EDX_PLATFORM_REVISION +# In order to transition from local disk asset storage to S3 backed asset storage, +# we need to run asset collection twice, once for local disk and once for S3. +# Once we have migrated to service assets off S3, then we can convert this back to +# managed by the yaml file contents +STATICFILES_STORAGE = os.environ.get('STATICFILES_STORAGE', ENV_TOKENS.get('STATICFILES_STORAGE', STATICFILES_STORAGE)) + # Email overrides DEFAULT_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_FROM_EMAIL', DEFAULT_FROM_EMAIL) DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBACK_EMAIL) diff --git a/openedx/core/djangoapps/theming/finders.py b/openedx/core/djangoapps/theming/finders.py index ff3c1bc91f236e387dca44cfc5831f7b204af353..c16082efc36949db1ab0423cb925c0f68c595fe7 100644 --- a/openedx/core/djangoapps/theming/finders.py +++ b/openedx/core/djangoapps/theming/finders.py @@ -47,7 +47,7 @@ class ThemeFilesFinder(BaseFinder): themes = get_themes() for theme in themes: theme_storage = self.storage_class( - os.path.join(theme.path, self.source_dir), + location=os.path.join(theme.path, self.source_dir), prefix=theme.theme_dir_name, ) diff --git a/openedx/core/djangoapps/theming/storage.py b/openedx/core/djangoapps/theming/storage.py index a831e3cbbce2e34814dcb23f0da693167ee44f2f..7d560819489a4614ad8a3cc7d5dc99b904b66d5d 100644 --- a/openedx/core/djangoapps/theming/storage.py +++ b/openedx/core/djangoapps/theming/storage.py @@ -28,7 +28,7 @@ from openedx.core.djangoapps.theming.helpers import ( ) -class ThemeStorage(StaticFilesStorage): +class ThemeMixin(object): """ Comprehensive theme aware Static files storage. """ @@ -38,16 +38,10 @@ class ThemeStorage(StaticFilesStorage): # instead of "images/logo.png" prefix = None - def __init__(self, location=None, base_url=None, file_permissions_mode=None, - directory_permissions_mode=None, prefix=None): + def __init__(self, **kwargs): - self.prefix = prefix - super(ThemeStorage, self).__init__( - location=location, - base_url=base_url, - file_permissions_mode=file_permissions_mode, - directory_permissions_mode=directory_permissions_mode, - ) + self.prefix = kwargs.pop('prefix', None) + super(ThemeMixin, self).__init__(**kwargs) def url(self, name): """ @@ -76,7 +70,7 @@ class ThemeStorage(StaticFilesStorage): if prefix and self.themed(name, prefix): name = os.path.join(prefix, name) - return super(ThemeStorage, self).url(name) + return super(ThemeMixin, self).url(name) def themed(self, name, theme): """ @@ -112,6 +106,10 @@ class ThemeStorage(StaticFilesStorage): return self.exists(os.path.join(theme, name)) +class ThemeStorage(ThemeMixin, StaticFilesStorage): + pass + + class ThemeCachedFilesMixin(CachedFilesMixin): """ Comprehensive theme aware CachedFilesMixin. diff --git a/openedx/core/lib/django_require/staticstorage.py b/openedx/core/lib/django_require/staticstorage.py index ee2321e894a55bc77088bea6c59a68b89d17d1b0..df0c543f50921c9e489918905281e4396507875a 100644 --- a/openedx/core/lib/django_require/staticstorage.py +++ b/openedx/core/lib/django_require/staticstorage.py @@ -2,12 +2,12 @@ :class:`~django_require.staticstorage.OptimizedCachedRequireJsStorage` """ - -from openedx.core.storage import PipelineForgivingStorage +from openedx.core.storage import PipelineForgivingMixin +from pipeline.storage import PipelineCachedStorage from require.storage import OptimizedFilesMixin -class OptimizedCachedRequireJsStorage(OptimizedFilesMixin, PipelineForgivingStorage): +class OptimizedCachedRequireJsStorage(OptimizedFilesMixin, PipelineForgivingMixin, PipelineCachedStorage): """ Custom storage backend that is used by Django-require. """ diff --git a/openedx/core/storage.py b/openedx/core/storage.py index 262ff47580653bc4300bc70f693c5d192ca517fd..885e8a62ce0585019fc9cf5fc51f70fcb710ce11 100644 --- a/openedx/core/storage.py +++ b/openedx/core/storage.py @@ -7,19 +7,20 @@ from django.contrib.staticfiles.storage import StaticFilesStorage from django.core.files.storage import get_storage_class, FileSystemStorage from django.utils.deconstruct import deconstructible from django.utils.lru_cache import lru_cache -from pipeline.storage import NonPackagingMixin, PipelineCachedStorage +from pipeline.storage import NonPackagingMixin from require.storage import OptimizedFilesMixin +from storages.backends.s3boto3 import S3Boto3Storage -from openedx.core.djangoapps.theming.storage import ThemeCachedFilesMixin, ThemePipelineMixin, ThemeStorage +from openedx.core.djangoapps.theming.storage import ThemeCachedFilesMixin, ThemePipelineMixin, ThemeMixin -class PipelineForgivingStorage(PipelineCachedStorage): +class PipelineForgivingMixin(object): """ An extension of the django-pipeline storage backend which forgives missing files. """ def hashed_name(self, name, content=None, **kwargs): try: - out = super(PipelineForgivingStorage, self).hashed_name(name, content, **kwargs) + out = super(PipelineForgivingMixin, self).hashed_name(name, content, **kwargs) except ValueError: # This means that a file could not be found, and normally this would # cause a fatal error, which seems rather excessive given that @@ -29,7 +30,7 @@ class PipelineForgivingStorage(PipelineCachedStorage): def stored_name(self, name): try: - out = super(PipelineForgivingStorage, self).stored_name(name) + out = super(PipelineForgivingMixin, self).stored_name(name) except ValueError: # This means that a file could not be found, and normally this would # cause a fatal error, which seems rather excessive given that @@ -38,25 +39,33 @@ class PipelineForgivingStorage(PipelineCachedStorage): return out -class ProductionStorage( - PipelineForgivingStorage, +class ProductionMixin( + PipelineForgivingMixin, OptimizedFilesMixin, ThemePipelineMixin, ThemeCachedFilesMixin, - ThemeStorage, - StaticFilesStorage + ThemeMixin, ): """ - This class combines Django's StaticFilesStorage class with several mixins - that provide additional functionality. We use this version on production. + This class combines several mixins that provide additional functionality, and + can be applied over an existing Storage. + We use this version on production. """ pass +class ProductionStorage(ProductionMixin, StaticFilesStorage): + pass + + +class ProductionS3Storage(ProductionMixin, S3Boto3Storage): # pylint: disable=abstract-method + pass + + class DevelopmentStorage( NonPackagingMixin, ThemePipelineMixin, - ThemeStorage, + ThemeMixin, StaticFilesStorage ): """ diff --git a/requirements/edx/base.in b/requirements/edx/base.in index 97d5192bfa67e2899e4199a872ce5e7c9c750622..035f468ececf3bfa61f92a26a1e0fd43fcf507bc 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -60,7 +60,7 @@ django-ses django-simple-history django-splash django-statici18n==1.4.0 -django-storages==1.4.1 +django-storages django-user-tasks django-waffle==0.12.0 django-webpack-loader # Used to wire webpack bundles into the django asset pipeline diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 6ed1d427d2bdcac48e3dfc9ad00704e3366b03e1..b6b24add20daf072f2d1504f8d91b7668d9f7825 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -81,7 +81,7 @@ django-ses==0.8.13 django-simple-history==2.8.0 django-splash==0.2.5 django-statici18n==1.4.0 -django-storages==1.4.1 +django-storages==1.8 django-user-tasks==0.3.0 django-waffle==0.12.0 django-webpack-loader==0.6.0 @@ -104,7 +104,7 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.36 +edx-enterprise==2.0.37 edx-i18n-tools==0.5.0 edx-milestones==0.2.6 edx-oauth2-provider==1.3.1 @@ -175,7 +175,7 @@ paver==1.3.4 pbr==5.4.4 pdfminer.six==20191110 piexif==1.0.2 -pillow==6.2.1 +pillow==7.0.0 pkgconfig==1.5.1 # via xmlsec polib==1.1.0 # via edx-i18n-tools psutil==1.2.1 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 0a2e64153fa74bbe6577da40d04886f7d4d0926d..b942a06c4850e42eafb11cbdce4e497332c9988c 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -94,7 +94,7 @@ django-ses==0.8.13 django-simple-history==2.8.0 django-splash==0.2.5 django-statici18n==1.4.0 -django-storages==1.4.1 +django-storages==1.8 django-user-tasks==0.3.0 django-waffle==0.12.0 django-webpack-loader==0.6.0 @@ -118,7 +118,7 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.36 +edx-enterprise==2.0.37 edx-i18n-tools==0.5.0 edx-lint==1.3.0 edx-milestones==0.2.6 @@ -217,7 +217,7 @@ paver==1.3.4 pbr==5.4.4 pdfminer.six==20191110 piexif==1.0.2 -pillow==6.2.1 +pillow==7.0.0 pip-tools==4.3.0 pkgconfig==1.5.1 pluggy==0.13.1 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index c5531c39f389a4ef06f214ee3e08889e80ce4bc7..2e5bf1f60e362af2cfc4e3985ded600feb3227e1 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -92,7 +92,7 @@ django-ses==0.8.13 django-simple-history==2.8.0 django-splash==0.2.5 django-statici18n==1.4.0 -django-storages==1.4.1 +django-storages==1.8 django-user-tasks==0.3.0 django-waffle==0.12.0 django-webpack-loader==0.6.0 @@ -115,7 +115,7 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.36 +edx-enterprise==2.0.37 edx-i18n-tools==0.5.0 edx-lint==1.3.0 edx-milestones==0.2.6 @@ -209,7 +209,7 @@ paver==1.3.4 pbr==5.4.4 pdfminer.six==20191110 piexif==1.0.2 -pillow==6.2.1 +pillow==7.0.0 pkgconfig==1.5.1 pluggy==0.13.1 polib==1.1.0