WIP: xblock pipeline work (#10176)

[PERF-303] Integer XBlocks/XModules into the static asset pipeline.

This PR, based on hackathon work from Christina/Andy, implements a way to discover all installed XBlocks and XModules and to enumerate their public assets, then pulling them in during the collectstatic phase and hashing them.  In turn, the methods for generating URLs to resources will then returned the hashed name for assets, allowing them to be served from nginx/CDNs, and cached heavily.

.jshintrc

@@ -148,6 +148,7 @@
         "JSON",
 
         // edX globals
-        "edx"
+        "edx",
+        "XBlock"
     ]
 }

cms/djangoapps/contentstore/views/preview.py

@@ -9,8 +9,9 @@ from django.http import Http404, HttpResponseBadRequest
 from django.contrib.auth.decorators import login_required
 from edxmako.shortcuts import render_to_string
 
-from openedx.core.lib.xblock_utils import replace_static_urls, wrap_xblock, wrap_fragment, wrap_xblock_aside,\
-    request_token
+from openedx.core.lib.xblock_utils import (
+    replace_static_urls, wrap_xblock, wrap_fragment, wrap_xblock_aside, request_token, xblock_local_resource_url,
+)
 from xmodule.x_module import PREVIEW_VIEWS, STUDENT_VIEW, AUTHOR_VIEW
 from xmodule.contentstore.django import contentstore
 from xmodule.error_module import ErrorDescriptor
@@ -31,7 +32,6 @@ from xblock_django.user_service import DjangoXBlockUserService
 
 from lms.djangoapps.lms_xblock.field_data import LmsFieldData
 from cms.lib.xblock.field_data import CmsFieldData
-from cms.lib.xblock.runtime import local_resource_url
 
 from util.sandboxing import can_execute_unsafe_code, get_python_lib_zip
 
@@ -115,7 +115,7 @@ class PreviewModuleSystem(ModuleSystem):  # pylint: disable=abstract-method
         }) + '?' + query
 
     def local_resource_url(self, block, uri):
-        return local_resource_url(block, uri)
+        return xblock_local_resource_url(block, uri)
 
     def applicable_aside_types(self, block):
         """

cms/envs/common.py

@@ -524,6 +524,7 @@ STATICFILES_FINDERS = [
     'openedx.core.djangoapps.theming.finders.ComprehensiveThemeFinder',
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
     'pipeline.finders.PipelineFinder',
 ]
 

cms/envs/devstack_optimized.py

@@ -32,6 +32,9 @@ DEBUG = True
 # Set REQUIRE_DEBUG to false so that it behaves like production
 REQUIRE_DEBUG = False
 
+# Fetch static files out of the pipeline's static root
+STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
+
 #  Serve static files at /static directly from the staticfiles directory under test root.
 # Note: optimized files for testing are generated with settings from test_static_optimized
 STATIC_URL = "/static/"

cms/envs/test_static_optimized.py

@@ -31,6 +31,7 @@ STATICFILES_STORAGE = 'openedx.core.lib.django_require.staticstorage.OptimizedCa
 STATICFILES_FINDERS = [
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
 ]
 
 # Redirect to the test_root folder within the repo

cms/lib/xblock/runtime.py

@@ -23,13 +23,3 @@ def handler_url(block, handler_name, suffix='', query='', thirdparty=False):
         url += '?' + query
 
     return url
-
-
-def local_resource_url(block, uri):
-    """
-    local_resource_url for Studio
-    """
-    return reverse('xblock_resource_url', kwargs={
-        'block_type': block.scope_ids.block_type,
-        'uri': uri,
-    })

cms/startup.py

@@ -13,6 +13,7 @@ from monkey_patch import (
     third_party_auth,
     django_db_models_options
 )
+from openedx.core.lib.xblock_utils import xblock_local_resource_url
 
 import xmodule.x_module
 import cms.lib.xblock.runtime
@@ -46,7 +47,7 @@ def run():
     # TODO: Remove this code when Runtimes are no longer created by modulestores
     # https://openedx.atlassian.net/wiki/display/PLAT/Convert+from+Storage-centric+runtimes+to+Application-centric+runtimes
     xmodule.x_module.descriptor_global_handler_url = cms.lib.xblock.runtime.handler_url
-    xmodule.x_module.descriptor_global_local_resource_url = cms.lib.xblock.runtime.local_resource_url
+    xmodule.x_module.descriptor_global_local_resource_url = xblock_local_resource_url
 
 
 def add_mimetypes():

common/lib/xmodule/xmodule/annotatable_module.py

@@ -162,3 +162,4 @@ class AnnotatableModule(AnnotatableFields, XModule):
 class AnnotatableDescriptor(AnnotatableFields, RawDescriptor):
     module_class = AnnotatableModule
     mako_template = "widgets/raw-edit.html"
+    resources_dir = None

common/lib/xmodule/xmodule/public/js/split_test_author_view.js → common/lib/xmodule/xmodule/assets/split_test/public/js/split_test_author_view.js

 /* JavaScript for editing operations that can be done on the split test author view. */
 window.SplitTestAuthorView = function (runtime, element) {
+    "use strict";
     var $element = $(element);
     var splitTestLocator = $element.closest('.studio-xblock-wrapper').data('locator');
 

common/lib/xmodule/xmodule/public/js/split_test_staff.js → common/lib/xmodule/xmodule/assets/split_test/public/js/split_test_staff.js

-
-/**
- * Creates a new selector for managing toggling which child to show
- * @constructor
- */
-
-function ABTestSelector(runtime, elem) {
-    var _this = this;
+/* Creates a new selector for managing toggling which child to show. */
+window.ABTestSelector = function (runtime, elem) {
+    "use strict";
+    var _this = {};
     _this.elem = $(elem);
     _this.children = _this.elem.find('.split-test-child');
     _this.content_container = _this.elem.find('.split-test-child-container');
@@ -23,17 +19,13 @@ function ABTestSelector(runtime, elem) {
         });
     }
 
-    select = _this.elem.find('.split-test-select');
-    cur_group_id = select.val();
+    var select = _this.elem.find('.split-test-select');
+    var cur_group_id = select.val();
     select_child(cur_group_id);
 
     // bind the change event to the dropdown
     select.change(function() {
-        group_id = $(this).val()
+        var group_id = $(this).val();
         select_child(group_id);
     });
-
-}
-
-
-
+};

common/lib/xmodule/xmodule/public/js/split_test_student.js → common/lib/xmodule/xmodule/assets/split_test/public/js/split_test_student.js

 /* Javascript for the Split Test XBlock. */
-function SplitTestStudentView(runtime, element) {
+window.SplitTestStudentView = function (runtime, element) {
+    "use strict";
     $.post(runtime.handlerUrl(element, 'log_child_render'));
     return {};
-}
+};

common/lib/xmodule/xmodule/public/js/vertical_student_view.js → common/lib/xmodule/xmodule/assets/vertical/public/js/vertical_student_view.js

 /* JavaScript for Vertical Student View. */
 window.VerticalStudentView = function (runtime, element) {
-
-    'use strict';
+    "use strict";
     RequireJS.require(['js/bookmarks/views/bookmark_button'], function (BookmarkButton) {
         var $element = $(element);
         var $bookmarkButtonElement = $element.find('.bookmark-button');

common/lib/xmodule/xmodule/backcompat_module.py

@@ -60,6 +60,8 @@ def process_includes(fn):
 
 
 class SemanticSectionDescriptor(XModuleDescriptor):
+    resources_dir = None
+
     @classmethod
     @process_includes
     def from_xml(cls, xml_data, system, id_generator):
@@ -82,6 +84,8 @@ class SemanticSectionDescriptor(XModuleDescriptor):
 
 
 class TranslateCustomTagDescriptor(XModuleDescriptor):
+    resources_dir = None
+
     @classmethod
     def from_xml(cls, xml_data, system, id_generator):
         """

common/lib/xmodule/xmodule/capa_module.py

@@ -131,6 +131,7 @@ class CapaDescriptor(CapaFields, RawDescriptor):
     INDEX_CONTENT_TYPE = 'CAPA'
 
     module_class = CapaModule
+    resources_dir = None
 
     has_score = True
     show_in_read_only_mode = True

common/lib/xmodule/xmodule/conditional_module.py

@@ -184,6 +184,8 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
 
     module_class = ConditionalModule
 
+    resources_dir = None
+
     filename_extension = "xml"
 
     has_score = False

common/lib/xmodule/xmodule/course_module.py

@@ -789,6 +789,8 @@ class CourseDescriptor(CourseFields, SequenceDescriptor, LicenseMixin):
     """
     module_class = CourseModule
 
+    resources_dir = None
+
     def __init__(self, *args, **kwargs):
         """
         Expects the same arguments as XModuleDescriptor.__init__

common/lib/xmodule/xmodule/discussion_module.py

@@ -108,8 +108,9 @@ class DiscussionModule(DiscussionFields, XModule):
 
 
 class DiscussionDescriptor(DiscussionFields, MetadataOnlyEditingDescriptor, RawDescriptor):
-
     module_class = DiscussionModule
+    resources_dir = None
+
     # The discussion XML format uses `id` and `for` attributes,
     # but these would overload other module attributes, so we prefix them
     # for actual use in the code

common/lib/xmodule/xmodule/editing_module.py

@@ -20,6 +20,8 @@ class EditingDescriptor(EditingFields, MakoModuleDescriptor):
 
     This class is intended to be used as a mixin.
     """
+    resources_dir = None
+
     mako_template = "widgets/raw-edit.html"
 
     @property

common/lib/xmodule/xmodule/error_module.py

@@ -75,6 +75,7 @@ class ErrorDescriptor(ErrorFields, XModuleDescriptor):
     Module that provides a raw editing view of broken xml.
     """
     module_class = ErrorModule
+    resources_dir = None
 
     def get_html(self):
         return u''