From d6547988566751688952f2fb780070cb6f944ba1 Mon Sep 17 00:00:00 2001
From: Calen Pennington <cale@edx.org>
Date: Wed, 30 Apr 2014 10:17:45 -0400
Subject: [PATCH] Make course ids and usage ids opaque to LMS and Studio
[partial commit]
This commit updates common/lib/xmodule.
These keys are now objects with a limited interface, and the particular
internal representation is managed by the data storage layer (the
modulestore).
For the LMS, there should be no outward-facing changes to the system.
The keys are, for now, a change to internal representation only. For
Studio, the new serialized form of the keys is used in urls, to allow
for further migration in the future.
Co-Author: Andy Armstrong <andya@edx.org>
Co-Author: Christina Roberts <christina@edx.org>
Co-Author: David Baumgold <db@edx.org>
Co-Author: Diana Huang <dkh@edx.org>
Co-Author: Don Mitchell <dmitchell@edx.org>
Co-Author: Julia Hansbrough <julia@edx.org>
Co-Author: Nimisha Asthagiri <nasthagiri@edx.org>
Co-Author: Sarina Canelake <sarina@edx.org>
[LMS-2370]
---
common/lib/xmodule/setup.py | 12 +
common/lib/xmodule/xmodule/abtest_module.py | 2 +-
common/lib/xmodule/xmodule/capa_base.py | 24 +-
.../lib/xmodule/xmodule/conditional_module.py | 28 +-
.../xmodule/xmodule/contentstore/content.py | 105 +--
.../lib/xmodule/xmodule/contentstore/mongo.py | 103 ++-
.../lib/xmodule/xmodule/contentstore/utils.py | 13 +-
common/lib/xmodule/xmodule/course_module.py | 26 +-
common/lib/xmodule/xmodule/error_module.py | 2 -
common/lib/xmodule/xmodule/foldit_module.py | 2 +-
common/lib/xmodule/xmodule/html_module.py | 2 +-
common/lib/xmodule/xmodule/lti_module.py | 6 +-
.../xmodule/xmodule/modulestore/__init__.py | 542 +++++----------
.../lib/xmodule/xmodule/modulestore/django.py | 1 -
.../xmodule/xmodule/modulestore/exceptions.py | 7 +
.../xmodule/modulestore/loc_mapper_store.py | 505 ++++++++------
.../xmodule/xmodule/modulestore/locator.py | 642 ++++++++----------
.../lib/xmodule/xmodule/modulestore/mixed.py | 299 +++-----
.../xmodule/xmodule/modulestore/mongo/base.py | 539 +++++++++------
.../xmodule/modulestore/mongo/draft.py | 134 ++--
.../xmodule/xmodule/modulestore/parsers.py | 79 +--
.../lib/xmodule/xmodule/modulestore/search.py | 65 +-
.../xmodule/modulestore/split_migrator.py | 119 ++--
.../split_mongo/caching_descriptor_system.py | 55 +-
.../split_mongo/mongo_connection.py | 24 +-
.../xmodule/modulestore/split_mongo/split.py | 337 ++++-----
.../xmodule/modulestore/store_utilities.py | 172 ++---
.../xmodule/modulestore/tests/django_utils.py | 2 +-
.../xmodule/modulestore/tests/factories.py | 22 +-
.../modulestore/tests/test_location.py | 210 +++---
.../modulestore/tests/test_location_mapper.py | 289 ++++----
.../modulestore/tests/test_locators.py | 353 ++++------
.../tests/test_mixed_modulestore.py | 175 ++---
.../modulestore/tests/test_modulestore.py | 62 +-
.../xmodule/modulestore/tests/test_mongo.py | 240 ++++---
.../xmodule/modulestore/tests/test_orphan.py | 156 +----
.../xmodule/modulestore/tests/test_publish.py | 141 +---
.../modulestore/tests/test_split_migrator.py | 271 +++-----
.../tests/test_split_modulestore.py | 445 ++++++------
.../tests/test_split_w_old_mongo.py | 137 ++++
.../xmodule/modulestore/tests/test_xml.py | 35 +-
.../modulestore/tests/test_xml_importer.py | 153 ++++-
common/lib/xmodule/xmodule/modulestore/xml.py | 225 +++---
.../xmodule/modulestore/xml_exporter.py | 39 +-
.../xmodule/modulestore/xml_importer.py | 409 +++++------
.../combined_open_ended_modulev1.py | 4 +-
.../combined_open_ended_rubric.py | 2 +-
.../open_ended_module.py | 4 +-
.../openendedchild.py | 2 +-
.../peer_grading_service.py | 5 +
.../xmodule/xmodule/peer_grading_module.py | 10 +-
common/lib/xmodule/xmodule/seq_module.py | 4 +-
.../lib/xmodule/xmodule/split_test_module.py | 10 +-
common/lib/xmodule/xmodule/tabs.py | 10 +-
common/lib/xmodule/xmodule/tests/__init__.py | 25 +-
.../xmodule/tests/test_annotatable_module.py | 2 +-
.../xmodule/xmodule/tests/test_capa_module.py | 10 +-
.../xmodule/tests/test_combined_open_ended.py | 52 +-
.../xmodule/xmodule/tests/test_conditional.py | 27 +-
.../lib/xmodule/xmodule/tests/test_content.py | 22 +-
.../xmodule/tests/test_course_module.py | 6 +-
.../tests/test_delay_between_attempts.py | 3 +-
.../xmodule/tests/test_editing_module.py | 3 +-
.../xmodule/tests/test_error_module.py | 15 +-
.../lib/xmodule/xmodule/tests/test_export.py | 10 +-
.../lib/xmodule/xmodule/tests/test_import.py | 78 +--
.../xmodule/tests/test_import_static.py | 6 +-
.../xmodule/xmodule/tests/test_lti_unit.py | 46 +-
.../xmodule/tests/test_peer_grading.py | 65 +-
.../xmodule/tests/test_self_assessment.py | 9 +-
common/lib/xmodule/xmodule/tests/test_tabs.py | 24 +-
.../xmodule/tests/test_util_open_ended.py | 7 +-
.../lib/xmodule/xmodule/tests/test_video.py | 21 +-
.../xmodule/tests/test_xblock_wrappers.py | 2 +-
.../lib/xmodule/xmodule/tests/xml/__init__.py | 14 +-
.../xmodule/xmodule/tests/xml/factories.py | 12 +-
common/lib/xmodule/xmodule/vertical_module.py | 2 +-
.../xmodule/video_module/transcripts_utils.py | 7 +-
common/lib/xmodule/xmodule/x_module.py | 53 +-
79 files changed, 3759 insertions(+), 4022 deletions(-)
create mode 100644 common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
diff --git a/common/lib/xmodule/setup.py b/common/lib/xmodule/setup.py
index 58d81709674..55d12c74f1b 100644
--- a/common/lib/xmodule/setup.py
+++ b/common/lib/xmodule/setup.py
@@ -67,5 +67,17 @@ setup(
'console_scripts': [
'xmodule_assets = xmodule.static_content:main',
],
+ 'course_key': [
+ 'slashes = xmodule.modulestore.locations:SlashSeparatedCourseKey',
+ 'course-locator = xmodule.modulestore.locator:CourseLocator',
+ ],
+ 'usage_key': [
+ 'location = xmodule.modulestore.locations:Location',
+ 'edx = xmodule.modulestore.locator:BlockUsageLocator',
+ ],
+ 'asset_key': [
+ 'asset-location = xmodule.modulestore.locations:AssetLocation',
+ 'edx = xmodule.modulestore.locator:BlockUsageLocator',
+ ],
},
)
diff --git a/common/lib/xmodule/xmodule/abtest_module.py b/common/lib/xmodule/xmodule/abtest_module.py
index 325d98cc5f1..2a7bc5f0cbb 100644
--- a/common/lib/xmodule/xmodule/abtest_module.py
+++ b/common/lib/xmodule/xmodule/abtest_module.py
@@ -67,7 +67,7 @@ class ABTestModule(ABTestFields, XModule):
def get_child_descriptors(self):
active_locations = set(self.group_content[self.group])
- return [desc for desc in self.descriptor.get_children() if desc.location.url() in active_locations]
+ return [desc for desc in self.descriptor.get_children() if desc.location.to_deprecated_string() in active_locations]
def displayable_items(self):
# Most modules return "self" as the displayable_item. We never display ourself
diff --git a/common/lib/xmodule/xmodule/capa_base.py b/common/lib/xmodule/xmodule/capa_base.py
index dd22a5f3b5d..9f9a8c4e516 100644
--- a/common/lib/xmodule/xmodule/capa_base.py
+++ b/common/lib/xmodule/xmodule/capa_base.py
@@ -207,7 +207,7 @@ class CapaMixin(CapaFields):
# Need the problem location in openendedresponse to send out. Adding
# it to the system here seems like the least clunky way to get it
# there.
- self.runtime.set('location', self.location.url())
+ self.runtime.set('location', self.location.to_deprecated_string())
try:
# TODO (vshnayder): move as much as possible of this work and error
@@ -225,7 +225,7 @@ class CapaMixin(CapaFields):
except Exception as err: # pylint: disable=broad-except
msg = u'cannot create LoncapaProblem {loc}: {err}'.format(
- loc=self.location.url(), err=err)
+ loc=self.location.to_deprecated_string(), err=err)
# TODO (vshnayder): do modules need error handlers too?
# We shouldn't be switching on DEBUG.
if self.runtime.DEBUG:
@@ -239,7 +239,7 @@ class CapaMixin(CapaFields):
# create a dummy problem with error message instead of failing
problem_text = (u'<problem><text><span class="inline-error">'
u'Problem {url} has an error:</span>{msg}</text></problem>'.format(
- url=self.location.url(),
+ url=self.location.to_deprecated_string(),
msg=msg)
)
self.lcp = self.new_lcp(self.get_state_for_lcp(), text=problem_text)
@@ -259,7 +259,7 @@ class CapaMixin(CapaFields):
self.seed = 1
elif self.rerandomize == "per_student" and hasattr(self.runtime, 'seed'):
# see comment on randomization_bin
- self.seed = randomization_bin(self.runtime.seed, self.location.url)
+ self.seed = randomization_bin(self.runtime.seed, unicode(self.location).encode('utf-8'))
else:
self.seed = struct.unpack('i', os.urandom(4))[0]
@@ -370,7 +370,7 @@ class CapaMixin(CapaFields):
progress = self.get_progress()
return self.runtime.render_template('problem_ajax.html', {
'element_id': self.location.html_id(),
- 'id': self.id,
+ 'id': self.location.to_deprecated_string(),
'ajax_url': self.runtime.ajax_url,
'progress_status': Progress.to_js_status_str(progress),
'progress_detail': Progress.to_js_detail_str(progress),
@@ -510,7 +510,7 @@ class CapaMixin(CapaFields):
msg = (
u'[courseware.capa.capa_module] <font size="+1" color="red">'
u'Failed to generate HTML for problem {url}</font>'.format(
- url=cgi.escape(self.location.url()))
+ url=cgi.escape(self.location.to_deprecated_string()))
)
msg += u'<p>Error:</p><p><pre>{msg}</pre></p>'.format(msg=cgi.escape(err.message))
msg += u'<p><pre>{tb}</pre></p>'.format(tb=cgi.escape(traceback.format_exc()))
@@ -598,7 +598,7 @@ class CapaMixin(CapaFields):
context = {
'problem': content,
- 'id': self.id,
+ 'id': self.location.to_deprecated_string(),
'check_button': check_button,
'check_button_checking': check_button_checking,
'reset_button': self.should_show_reset_button(),
@@ -763,7 +763,7 @@ class CapaMixin(CapaFields):
Returns the answers: {'answers' : answers}
"""
event_info = dict()
- event_info['problem_id'] = self.location.url()
+ event_info['problem_id'] = self.location.to_deprecated_string()
self.track_function_unmask('showanswer', event_info)
if not self.answer_available():
raise NotFoundError('Answer is not available')
@@ -906,7 +906,7 @@ class CapaMixin(CapaFields):
"""
event_info = dict()
event_info['state'] = self.lcp.get_state()
- event_info['problem_id'] = self.location.url()
+ event_info['problem_id'] = self.location.to_deprecated_string()
answers = self.make_dict_of_responses(data)
answers_without_files = convert_files_to_filenames(answers)
@@ -1218,7 +1218,7 @@ class CapaMixin(CapaFields):
Returns the error messages for exceptions occurring while performing
the rescoring, rather than throwing them.
"""
- event_info = {'state': self.lcp.get_state(), 'problem_id': self.location.url()}
+ event_info = {'state': self.lcp.get_state(), 'problem_id': self.location.to_deprecated_string()}
_ = self.runtime.service(self, "i18n").ugettext
@@ -1293,7 +1293,7 @@ class CapaMixin(CapaFields):
"""
event_info = dict()
event_info['state'] = self.lcp.get_state()
- event_info['problem_id'] = self.location.url()
+ event_info['problem_id'] = self.location.to_deprecated_string()
answers = self.make_dict_of_responses(data)
event_info['answers'] = answers
@@ -1346,7 +1346,7 @@ class CapaMixin(CapaFields):
"""
event_info = dict()
event_info['old_state'] = self.lcp.get_state()
- event_info['problem_id'] = self.location.url()
+ event_info['problem_id'] = self.location.to_deprecated_string()
_ = self.runtime.service(self, "i18n").ugettext
if self.closed():
diff --git a/common/lib/xmodule/xmodule/conditional_module.py b/common/lib/xmodule/xmodule/conditional_module.py
index 4c05737d0cf..809c6ef54dc 100644
--- a/common/lib/xmodule/xmodule/conditional_module.py
+++ b/common/lib/xmodule/xmodule/conditional_module.py
@@ -9,7 +9,6 @@ from lxml import etree
from pkg_resources import resource_string
from xmodule.x_module import XModule
-from xmodule.modulestore import Location
from xmodule.seq_module import SequenceDescriptor
from xblock.fields import Scope, ReferenceList
from xmodule.modulestore.exceptions import ItemNotFoundError
@@ -144,7 +143,6 @@ class ConditionalModule(ConditionalFields, XModule):
return self.system.render_template('conditional_ajax.html', {
'element_id': self.location.html_id(),
- 'id': self.id,
'ajax_url': self.system.ajax_url,
'depends': ';'.join(self.required_html_ids)
})
@@ -199,20 +197,14 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
# substitution can be done.
if not self.sources_list:
if 'sources' in self.xml_attributes and isinstance(self.xml_attributes['sources'], basestring):
- sources = ConditionalDescriptor.parse_sources(self.xml_attributes)
- self.sources_list = sources
+ self.sources_list = ConditionalDescriptor.parse_sources(self.xml_attributes)
@staticmethod
def parse_sources(xml_element):
""" Parse xml_element 'sources' attr and return a list of location strings. """
- result = []
sources = xml_element.get('sources')
if sources:
- locations = [location.strip() for location in sources.split(';')]
- for location in locations:
- if Location.is_valid(location): # Check valid location url.
- result.append(location)
- return result
+ return [location.strip() for location in sources.split(';')]
def get_required_module_descriptors(self):
"""Returns a list of XModuleDescriptor instances upon
@@ -221,7 +213,7 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
descriptors = []
for location in self.sources_list:
try:
- descriptor = self.system.load_item(Location(location))
+ descriptor = self.system.load_item(location)
descriptors.append(descriptor)
except ItemNotFoundError:
msg = "Invalid module by location."
@@ -238,7 +230,7 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
if child.tag == 'show':
locations = ConditionalDescriptor.parse_sources(child)
for location in locations:
- children.append(Location(location))
+ children.append(location)
show_tag_list.append(location)
else:
try:
@@ -251,22 +243,18 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
return {'show_tag_list': show_tag_list}, children
def definition_to_xml(self, resource_fs):
- def to_string(string_list):
- """ Convert List of strings to a single string with "; " as the separator. """
- return "; ".join(string_list)
-
xml_object = etree.Element(self._tag_name)
for child in self.get_children():
- location = str(child.location)
- if location not in self.show_tag_list:
+ if child.location not in self.show_tag_list:
self.runtime.add_block_as_child_node(child, xml_object)
if self.show_tag_list:
show_str = u'<{tag_name} sources="{sources}" />'.format(
- tag_name='show', sources=to_string(self.show_tag_list))
+ tag_name='show', sources=';'.join(location.to_deprecated_string() for location in self.show_tag_list))
xml_object.append(etree.fromstring(show_str))
# Overwrite the original sources attribute with the value from sources_list, as
# Locations may have been changed to Locators.
- self.xml_attributes['sources'] = to_string(self.sources_list)
+ stringified_sources_list = map(lambda loc: loc.to_deprecated_string(), self.sources_list)
+ self.xml_attributes['sources'] = ';'.join(stringified_sources_list)
return xml_object
diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py
index 536fe994e2a..8782116acc0 100644
--- a/common/lib/xmodule/xmodule/contentstore/content.py
+++ b/common/lib/xmodule/xmodule/contentstore/content.py
@@ -1,3 +1,5 @@
+import bson.son
+import re
XASSET_LOCATION_TAG = 'c4x'
XASSET_SRCREF_PREFIX = 'xasset:'
@@ -8,7 +10,7 @@ import logging
import StringIO
from urlparse import urlparse, urlunparse
-from xmodule.modulestore import Location
+from xmodule.modulestore.locations import AssetLocation, SlashSeparatedCourseKey
from .django import contentstore
from PIL import Image
@@ -22,7 +24,7 @@ class StaticContent(object):
self._data = data
self.length = length
self.last_modified_at = last_modified_at
- self.thumbnail_location = Location(thumbnail_location) if thumbnail_location is not None else None
+ self.thumbnail_location = thumbnail_location
# optional information about where this file was imported from. This is needed to support import/export
# cycles
self.import_path = import_path
@@ -39,44 +41,48 @@ class StaticContent(object):
extension=XASSET_THUMBNAIL_TAIL_NAME,)
@staticmethod
- def compute_location(org, course, name, revision=None, is_thumbnail=False):
- name = name.replace('/', '_')
- return Location([XASSET_LOCATION_TAG, org, course, 'asset' if not is_thumbnail else 'thumbnail',
- Location.clean_keeping_underscores(name), revision])
+ def compute_location(course_key, path, revision=None, is_thumbnail=False):
+ """
+ Constructs a location object for static content.
+
+ - course_key: the course that this asset belongs to
+ - path: is the name of the static asset
+ - revision: is the object's revision information
+ - is_tumbnail: is whether or not we want the thumbnail version of this
+ asset
+ """
+ path = path.replace('/', '_')
+ return AssetLocation(
+ course_key.org, course_key.course, course_key.run,
+ 'asset' if not is_thumbnail else 'thumbnail',
+ AssetLocation.clean_keeping_underscores(path),
+ revision
+ )
def get_id(self):
return StaticContent.get_id_from_location(self.location)
def get_url_path(self):
- return StaticContent.get_url_path_from_location(self.location)
+ return self.location.to_deprecated_string()
@property
def data(self):
return self._data
- @staticmethod
- def get_url_path_from_location(location):
- if location is not None:
- return u"/{tag}/{org}/{course}/{category}/{name}".format(**location.dict())
- else:
- return None
+ ASSET_URL_RE = re.compile(r"""
+ /?c4x/
+ (?P<org>[^/]+)/
+ (?P<course>[^/]+)/
+ (?P<category>[^/]+)/
+ (?P<name>[^/]+)
+ """, re.VERBOSE | re.IGNORECASE)
@staticmethod
def is_c4x_path(path_string):
"""
Returns a boolean if a path is believed to be a c4x link based on the leading element
"""
- return path_string.startswith(u'/{0}/'.format(XASSET_LOCATION_TAG))
-
- @staticmethod
- def renamespace_c4x_path(path_string, target_location):
- """
- Returns an updated string which incorporates a new org/course in order to remap an asset path
- to a new namespace
- """
- location = StaticContent.get_location_from_path(path_string)
- location = location.replace(org=target_location.org, course=target_location.course)
- return StaticContent.get_url_path_from_location(location)
+ return StaticContent.ASSET_URL_RE.match(path_string) is not None
@staticmethod
def get_static_path_from_location(location):
@@ -88,28 +94,35 @@ class StaticContent(object):
the actual /c4x/... path which the client needs to reference static content
"""
if location is not None:
- return u"/static/{name}".format(**location.dict())
+ return u"/static/{name}".format(name=location.name)
else:
return None
@staticmethod
- def get_base_url_path_for_course_assets(loc):
- if loc is not None:
- return u"/c4x/{org}/{course}/asset".format(**loc.dict())
+ def get_base_url_path_for_course_assets(course_key):
+ if course_key is None:
+ return None
+
+ assert(isinstance(course_key, SlashSeparatedCourseKey))
+ return course_key.make_asset_key('asset', '').to_deprecated_string()
@staticmethod
def get_id_from_location(location):
- return {'tag': location.tag, 'org': location.org, 'course': location.course,
- 'category': location.category, 'name': location.name,
- 'revision': location.revision}
+ """
+ Get the doc store's primary key repr for this location
+ """
+ return bson.son.SON([
+ ('tag', 'c4x'), ('org', location.org), ('course', location.course),
+ ('category', location.category), ('name', location.name),
+ ('revision', location.revision),
+ ])
@staticmethod
def get_location_from_path(path):
- # remove leading / character if it is there one
- if path.startswith('/'):
- path = path[1:]
-
- return Location(path.split('/'))
+ """
+ Generate an AssetKey for the given path (old c4x/org/course/asset/name syntax)
+ """
+ return AssetLocation.from_deprecated_string(path)
@staticmethod
def convert_legacy_static_url_with_course_id(path, course_id):
@@ -117,12 +130,10 @@ class StaticContent(object):
Returns a path to a piece of static content when we are provided with a filepath and
a course_id
"""
-
# Generate url of urlparse.path component
scheme, netloc, orig_path, params, query, fragment = urlparse(path)
- course_id_dict = Location.parse_course_id(course_id)
- loc = StaticContent.compute_location(course_id_dict['org'], course_id_dict['course'], orig_path)
- loc_url = StaticContent.get_url_path_from_location(loc)
+ loc = StaticContent.compute_location(course_id, orig_path)
+ loc_url = loc.to_deprecated_string()
# Reconstruct with new path
return urlunparse((scheme, netloc, loc_url, params, query, fragment))
@@ -167,7 +178,7 @@ class ContentStore(object):
def find(self, filename):
raise NotImplementedError
- def get_all_content_for_course(self, location, start=0, maxresults=-1, sort=None):
+ def get_all_content_for_course(self, course_key, start=0, maxresults=-1, sort=None):
'''
Returns a list of static assets for a course, followed by the total number of assets.
By default all assets are returned, but start and maxresults can be provided to limit the query.
@@ -192,13 +203,21 @@ class ContentStore(object):
'''
raise NotImplementedError
+ def delete_all_course_assets(self, course_key):
+ """
+ Delete all of the assets which use this course_key as an identifier
+ :param course_key:
+ """
+ raise NotImplementedError
+
def generate_thumbnail(self, content, tempfile_path=None):
thumbnail_content = None
# use a naming convention to associate originals with the thumbnail
thumbnail_name = StaticContent.generate_thumbnail_name(content.location.name)
- thumbnail_file_location = StaticContent.compute_location(content.location.org, content.location.course,
- thumbnail_name, is_thumbnail=True)
+ thumbnail_file_location = StaticContent.compute_location(
+ content.location.course_key, thumbnail_name, is_thumbnail=True
+ )
# if we're uploading an image, then let's generate a thumbnail so that we can
# serve it up when needed without having to rescale on the fly
diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py
index b20ba7a5983..76fca881038 100644
--- a/common/lib/xmodule/xmodule/contentstore/mongo.py
+++ b/common/lib/xmodule/xmodule/contentstore/mongo.py
@@ -2,8 +2,7 @@ import pymongo
import gridfs
from gridfs.errors import NoFile
-from xmodule.modulestore import Location
-from xmodule.modulestore.mongo.base import location_to_query
+from xmodule.modulestore.mongo.base import location_to_query, MongoModuleStore, location_to_son
from xmodule.contentstore.content import XASSET_LOCATION_TAG
import logging
@@ -13,6 +12,8 @@ from xmodule.exceptions import NotFoundError
from fs.osfs import OSFS
import os
import json
+import bson.son
+from xmodule.modulestore.locations import AssetLocation
class MongoContentStore(ContentStore):
@@ -28,6 +29,7 @@ class MongoContentStore(ContentStore):
pymongo.MongoClient(
host=host,
port=port,
+ document_class=bson.son.SON,
**kwargs
),
db
@@ -46,8 +48,10 @@ class MongoContentStore(ContentStore):
# Seems like with the GridFS we can't update existing ID's we have to do a delete/add pair
self.delete(content_id)
+ thumbnail_location = content.thumbnail_location.to_deprecated_list_repr() if content.thumbnail_location else None
with self.fs.new_file(_id=content_id, filename=content.get_url_path(), content_type=content.content_type,
- displayname=content.name, thumbnail_location=content.thumbnail_location,
+ displayname=content.name,
+ thumbnail_location=thumbnail_location,
import_path=content.import_path,
# getattr b/c caching may mean some pickled instances don't have attr
locked=getattr(content, 'locked', False)) as fp:
@@ -62,23 +66,40 @@ class MongoContentStore(ContentStore):
def delete(self, content_id):
if self.fs.exists({"_id": content_id}):
self.fs.delete(content_id)
+ assert not self.fs.exists({"_id": content_id})
def find(self, location, throw_on_not_found=True, as_stream=False):
content_id = StaticContent.get_id_from_location(location)
+ # Use slow attr based lookup b/c we weren't careful to control the key order in _id before
+ content_id = {u'_id.{}'.format(key): value for key, value in content_id.iteritems()}
+ fs_pointer = self.fs_files.find_one(content_id, fields={'_id': 1})
+ if fs_pointer is None:
+ if throw_on_not_found:
+ raise NotFoundError()
+ else:
+ return None
+ content_id = fs_pointer['_id']
+
try:
if as_stream:
fp = self.fs.get(content_id)
+ thumbnail_location = getattr(fp, 'thumbnail_location', None)
+ if thumbnail_location:
+ thumbnail_location = location.course_key.make_asset_key('thumbnail', thumbnail_location[4])
return StaticContentStream(
location, fp.displayname, fp.content_type, fp, last_modified_at=fp.uploadDate,
- thumbnail_location=getattr(fp, 'thumbnail_location', None),
+ thumbnail_location=thumbnail_location,
import_path=getattr(fp, 'import_path', None),
length=fp.length, locked=getattr(fp, 'locked', False)
)
else:
with self.fs.get(content_id) as fp:
+ thumbnail_location = getattr(fp, 'thumbnail_location', None)
+ if thumbnail_location:
+ thumbnail_location = location.course_key.make_asset_key('thumbnail', thumbnail_location[4])
return StaticContent(
location, fp.displayname, fp.content_type, fp.read(), last_modified_at=fp.uploadDate,
- thumbnail_location=getattr(fp, 'thumbnail_location', None),
+ thumbnail_location=thumbnail_location,
import_path=getattr(fp, 'import_path', None),
length=fp.length, locked=getattr(fp, 'locked', False)
)
@@ -90,8 +111,12 @@ class MongoContentStore(ContentStore):
def get_stream(self, location):
content_id = StaticContent.get_id_from_location(location)
+ # use slow attr based lookup because we weren't careful to control the key order in _id before
+ content_id = {u'_id.{}'.format(key): value for key, value in content_id.iteritems()}
+ fs_pointer = self.fs_files.find_one(content_id, fields={'_id': 1})
+
try:
- handle = self.fs.get(content_id)
+ handle = self.fs.get(fs_pointer['_id'])
except NoFile:
raise NotFoundError()
@@ -100,7 +125,7 @@ class MongoContentStore(ContentStore):
def close_stream(self, handle):
try:
handle.close()
- except Exception:
+ except Exception: # pylint: disable=broad-except
pass
def export(self, location, output_directory):
@@ -117,21 +142,22 @@ class MongoContentStore(ContentStore):
with disk_fs.open(content.name, 'wb') as asset_file:
asset_file.write(content.data)
- def export_all_for_course(self, course_location, output_directory, assets_policy_file):
+ def export_all_for_course(self, course_key, output_directory, assets_policy_file):
"""
Export all of this course's assets to the output_directory. Export all of the assets'
attributes to the policy file.
- :param course_location: the Location of type 'course'
- :param output_directory: the directory under which to put all the asset files
- :param assets_policy_file: the filename for the policy file which should be in the same
- directory as the other policy files.
+ Args:
+ course_key (CourseKey): the :class:`CourseKey` identifying the course
+ output_directory: the directory under which to put all the asset files
+ assets_policy_file: the filename for the policy file which should be in the same
+ directory as the other policy files.
"""
policy = {}
- assets, __ = self.get_all_content_for_course(course_location)
+ assets, __ = self.get_all_content_for_course(course_key)
for asset in assets:
- asset_location = Location(asset['_id'])
+ asset_location = AssetLocation._from_deprecated_son(asset['_id'], course_key.run) # pylint: disable=protected-access
self.export(asset_location, output_directory)
for attr, value in asset.iteritems():
if attr not in ['_id', 'md5', 'uploadDate', 'length', 'chunkSize']:
@@ -140,15 +166,15 @@ class MongoContentStore(ContentStore):
with open(assets_policy_file, 'w') as f:
json.dump(policy, f)
- def get_all_content_thumbnails_for_course(self, location):
- return self._get_all_content_for_course(location, get_thumbnails=True)[0]
+ def get_all_content_thumbnails_for_course(self, course_key):
+ return self._get_all_content_for_course(course_key, get_thumbnails=True)[0]
- def get_all_content_for_course(self, location, start=0, maxresults=-1, sort=None):
+ def get_all_content_for_course(self, course_key, start=0, maxresults=-1, sort=None):
return self._get_all_content_for_course(
- location, start=start, maxresults=maxresults, get_thumbnails=False, sort=sort
+ course_key, start=start, maxresults=maxresults, get_thumbnails=False, sort=sort
)
- def _get_all_content_for_course(self, location, get_thumbnails=False, start=0, maxresults=-1, sort=None):
+ def _get_all_content_for_course(self, course_key, get_thumbnails=False, start=0, maxresults=-1, sort=None):
'''
Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
@@ -168,20 +194,22 @@ class MongoContentStore(ContentStore):
]
'''
- course_filter = Location(XASSET_LOCATION_TAG, category="asset" if not get_thumbnails else "thumbnail",
- course=location.course, org=location.org)
+ course_filter = course_key.make_asset_key(
+ "asset" if not get_thumbnails else "thumbnail",
+ None
+ )
# 'borrow' the function 'location_to_query' from the Mongo modulestore implementation
if maxresults > 0:
items = self.fs_files.find(
- location_to_query(course_filter),
+ location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG),
skip=start, limit=maxresults, sort=sort
)
else:
- items = self.fs_files.find(location_to_query(course_filter), sort=sort)
+ items = self.fs_files.find(location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG), sort=sort)
count = items.count()
return list(items), count
- def set_attr(self, location, attr, value=True):
+ def set_attr(self, asset_key, attr, value=True):
"""
Add/set the given attr on the asset at the given location. Does not allow overwriting gridFS built in
attrs such as _id, md5, uploadDate, length. Value can be any type which pymongo accepts.
@@ -191,11 +219,11 @@ class MongoContentStore(ContentStore):
Raises NotFoundError if no such item exists
Raises AttributeError is attr is one of the build in attrs.
- :param location: a c4x asset location
+ :param asset_key: an AssetKey
:param attr: which attribute to set
:param value: the value to set it to (any type pymongo accepts such as datetime, number, string)
"""
- self.set_attrs(location, {attr: value})
+ self.set_attrs(asset_key, {attr: value})
def get_attr(self, location, attr, default=None):
"""
@@ -216,15 +244,13 @@ class MongoContentStore(ContentStore):
:param location: a c4x asset location
"""
- # raises exception if location is not fully specified
- Location.ensure_fully_specified(location)
for attr in attr_dict.iterkeys():
if attr in ['_id', 'md5', 'uploadDate', 'length']:
raise AttributeError("{} is a protected attribute.".format(attr))
- item = self.fs_files.find_one(location_to_query(location))
- if item is None:
- raise NotFoundError()
- self.fs_files.update({"_id": item["_id"]}, {"$set": attr_dict})
+ asset_db_key = {'_id': location_to_son(location, tag=XASSET_LOCATION_TAG)}
+ if self.fs_files.find(asset_db_key).count() == 0:
+ raise NotFoundError(asset_db_key)
+ self.fs_files.update(asset_db_key, {"$set": attr_dict})
def get_attrs(self, location):
"""
@@ -236,7 +262,18 @@ class MongoContentStore(ContentStore):
:param location: a c4x asset location
"""
- item = self.fs_files.find_one(location_to_query(location))
+ item = self.fs_files.find_one({'_id': location_to_son(location, tag=XASSET_LOCATION_TAG)})
if item is None:
raise NotFoundError()
return item
+
+ def delete_all_course_assets(self, course_key):
+ """
+ Delete all assets identified via this course_key. Dangerous operation which may remove assets
+ referenced by other runs or other courses.
+ :param course_key:
+ """
+ course_query = MongoModuleStore._course_key_to_son(course_key, tag=XASSET_LOCATION_TAG) # pylint: disable=protected-access
+ matching_assets = self.fs_files.find(course_query)
+ for asset in matching_assets:
+ self.fs.delete(asset['_id'])
diff --git a/common/lib/xmodule/xmodule/contentstore/utils.py b/common/lib/xmodule/xmodule/contentstore/utils.py
index f354dbf4209..c9873e3bc2d 100644
--- a/common/lib/xmodule/xmodule/contentstore/utils.py
+++ b/common/lib/xmodule/xmodule/contentstore/utils.py
@@ -1,4 +1,3 @@
-from xmodule.modulestore import Location
from xmodule.contentstore.content import StaticContent
from .django import contentstore
@@ -13,18 +12,14 @@ def empty_asset_trashcan(course_locs):
# first delete all of the thumbnails
thumbs = store.get_all_content_thumbnails_for_course(course_loc)
for thumb in thumbs:
- thumb_loc = Location(thumb["_id"])
- id = StaticContent.get_id_from_location(thumb_loc)
- print "Deleting {0}...".format(id)
- store.delete(id)
+ print "Deleting {0}...".format(thumb)
+ store.delete(thumb['_id'])
# then delete all of the assets
assets, __ = store.get_all_content_for_course(course_loc)
for asset in assets:
- asset_loc = Location(asset["_id"])
- id = StaticContent.get_id_from_location(asset_loc)
- print "Deleting {0}...".format(id)
- store.delete(id)
+ print "Deleting {0}...".format(asset)
+ store.delete(asset['_id'])
def restore_asset_from_trashcan(location):
diff --git a/common/lib/xmodule/xmodule/course_module.py b/common/lib/xmodule/xmodule/course_module.py
index 3afc44b85b9..ee159a96776 100644
--- a/common/lib/xmodule/xmodule/course_module.py
+++ b/common/lib/xmodule/xmodule/course_module.py
@@ -438,7 +438,7 @@ class CourseDescriptor(CourseFields, SequenceDescriptor):
if isinstance(self.location, Location):
self.wiki_slug = self.location.course
elif isinstance(self.location, CourseLocator):
- self.wiki_slug = self.location.package_id or self.display_name
+ self.wiki_slug = self.id.offering or self.display_name
if self.due_date_display_format is None and self.show_timezone is False:
# For existing courses with show_timezone set to False (and no due_date_display_format specified),
@@ -810,32 +810,10 @@ class CourseDescriptor(CourseFields, SequenceDescriptor):
def make_id(org, course, url_name):
return '/'.join([org, course, url_name])
- @staticmethod
- def id_to_location(course_id):
- '''Convert the given course_id (org/course/name) to a location object.
- Throws ValueError if course_id is of the wrong format.
- '''
- course_id_dict = Location.parse_course_id(course_id)
- course_id_dict['tag'] = 'i4x'
- course_id_dict['category'] = 'course'
- return Location(course_id_dict)
-
- @staticmethod
- def location_to_id(location):
- '''Convert a location of a course to a course_id. If location category
- is not "course", raise a ValueError.
-
- location: something that can be passed to Location
- '''
- loc = Location(location)
- if loc.category != "course":
- raise ValueError("{0} is not a course location".format(loc))
- return "/".join([loc.org, loc.course, loc.name])
-
@property
def id(self):
"""Return the course_id for this course"""
- return self.location_to_id(self.location)
+ return self.location.course_key
@property
def start_date_text(self):
diff --git a/common/lib/xmodule/xmodule/error_module.py b/common/lib/xmodule/xmodule/error_module.py
index 68735351fa9..9ce938fea68 100644
--- a/common/lib/xmodule/xmodule/error_module.py
+++ b/common/lib/xmodule/xmodule/error_module.py
@@ -11,7 +11,6 @@ import sys
from lxml import etree
from xmodule.x_module import XModule, XModuleDescriptor
from xmodule.errortracker import exc_info_to_str
-from xmodule.modulestore import Location
from xblock.fields import String, Scope, ScopeIds
from xblock.field_data import DictFieldData
@@ -81,7 +80,6 @@ class ErrorDescriptor(ErrorFields, XModuleDescriptor):
@classmethod
def _construct(cls, system, contents, error_msg, location):
- location = Location(location)
if error_msg is None:
# this string is not marked for translation because we don't have
diff --git a/common/lib/xmodule/xmodule/foldit_module.py b/common/lib/xmodule/xmodule/foldit_module.py
index e24a71527b4..666d247bfd6 100644
--- a/common/lib/xmodule/xmodule/foldit_module.py
+++ b/common/lib/xmodule/xmodule/foldit_module.py
@@ -108,7 +108,7 @@ class FolditModule(FolditFields, XModule):
from foldit.models import Score
if courses is None:
- courses = [self.location.course_id]
+ courses = [self.location.course_key]
leaders = [(leader['username'], leader['score']) for leader in Score.get_tops_n(10, course_list=courses)]
leaders.sort(key=lambda x: -x[1])
diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py
index 2b58eaf4d36..88e3528cfe7 100644
--- a/common/lib/xmodule/xmodule/html_module.py
+++ b/common/lib/xmodule/xmodule/html_module.py
@@ -121,7 +121,7 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
# Add some specific HTML rendering context when editing HTML modules where we pass
# the root /c4x/ url for assets. This allows client-side substitutions to occur.
_context.update({
- 'base_asset_url': StaticContent.get_base_url_path_for_course_assets(self.location) + '/',
+ 'base_asset_url': StaticContent.get_base_url_path_for_course_assets(self.location.course_key),
'enable_latex_compiler': self.use_latex_compiler,
'editor': self.editor
})
diff --git a/common/lib/xmodule/xmodule/lti_module.py b/common/lib/xmodule/xmodule/lti_module.py
index bd23579fe08..91e851ec360 100644
--- a/common/lib/xmodule/xmodule/lti_module.py
+++ b/common/lib/xmodule/xmodule/lti_module.py
@@ -347,9 +347,7 @@ class LTIModule(LTIFields, XModule):
"""
Return course by course id.
"""
- course_location = CourseDescriptor.id_to_location(self.course_id)
- course = self.descriptor.runtime.modulestore.get_item(course_location)
- return course
+ return self.descriptor.runtime.modulestore.get_course(self.course_id)
@property
def context_id(self):
@@ -359,7 +357,7 @@ class LTIModule(LTIFields, XModule):
context_id is an opaque identifier that uniquely identifies the context (e.g., a course)
that contains the link being launched.
"""
- return self.course_id
+ return self.course_id.to_deprecated_string()
@property
def role(self):
diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py
index e010948e055..10816823f87 100644
--- a/common/lib/xmodule/xmodule/modulestore/__init__.py
+++ b/common/lib/xmodule/xmodule/modulestore/__init__.py
@@ -6,7 +6,7 @@ that are stored in a database an accessible using their Location as an identifie
import logging
import re
-from collections import namedtuple
+from collections import namedtuple, defaultdict
import collections
from abc import ABCMeta, abstractmethod
@@ -14,8 +14,13 @@ from xblock.plugin import default_select
from .exceptions import InvalidLocationError, InsufficientSpecificationError
from xmodule.errortracker import make_error_tracker
+from xmodule.modulestore.keys import CourseKey, UsageKey
+from xmodule.modulestore.locations import Location # For import backwards compatibility
+from opaque_keys import InvalidKeyError
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xblock.runtime import Mixologist
from xblock.core import XBlock
+import datetime
log = logging.getLogger('edx.modulestore')
@@ -23,286 +28,6 @@ SPLIT_MONGO_MODULESTORE_TYPE = 'split'
MONGO_MODULESTORE_TYPE = 'mongo'
XML_MODULESTORE_TYPE = 'xml'
-URL_RE = re.compile("""
- (?P<tag>[^:]+)://?
- (?P<org>[^/]+)/
- (?P<course>[^/]+)/
- (?P<category>[^/]+)/
- (?P<name>[^@]+)
- (@(?P<revision>[^/]+))?
- """, re.VERBOSE)
-
-# TODO (cpennington): We should decide whether we want to expand the
-# list of valid characters in a location
-INVALID_CHARS = re.compile(r"[^\w.%-]", re.UNICODE)
-# Names are allowed to have colons.
-INVALID_CHARS_NAME = re.compile(r"[^\w.:%-]", re.UNICODE)
-
-# html ids can contain word chars and dashes
-INVALID_HTML_CHARS = re.compile(r"[^\w-]", re.UNICODE)
-
-_LocationBase = namedtuple('LocationBase', 'tag org course category name revision')
-
-
-def _check_location_part(val, regexp):
- """
- Check that `regexp` doesn't match inside `val`. If it does, raise an exception
-
- Args:
- val (string): The value to check
- regexp (re.RegexObject): The regular expression specifying invalid characters
-
- Raises:
- InvalidLocationError: Raised if any invalid character is found in `val`
- """
- if val is not None and regexp.search(val) is not None:
- raise InvalidLocationError("Invalid characters in {!r}.".format(val))
-
-
-class Location(_LocationBase):
- '''
- Encodes a location.
-
- Locations representations of URLs of the
- form {tag}://{org}/{course}/{category}/{name}[@{revision}]
-
- However, they can also be represented as dictionaries (specifying each component),
- tuples or lists (specified in order), or as strings of the url
- '''
- __slots__ = ()
-
- @staticmethod
- def _clean(value, invalid):
- """
- invalid should be a compiled regexp of chars to replace with '_'
- """
- return re.sub('_+', '_', invalid.sub('_', value))
-
- @staticmethod
- def clean(value):
- """
- Return value, made into a form legal for locations
- """
- return Location._clean(value, INVALID_CHARS)
-
- @staticmethod
- def clean_keeping_underscores(value):
- """
- Return value, replacing INVALID_CHARS, but not collapsing multiple '_' chars.
- This for cleaning asset names, as the YouTube ID's may have underscores in them, and we need the
- transcript asset name to match. In the future we may want to change the behavior of _clean.
- """
- return INVALID_CHARS.sub('_', value)
-
- @staticmethod
- def clean_for_url_name(value):
- """
- Convert value into a format valid for location names (allows colons).
- """
- return Location._clean(value, INVALID_CHARS_NAME)
-
- @staticmethod
- def clean_for_html(value):
- """
- Convert a string into a form that's safe for use in html ids, classes, urls, etc.
- Replaces all INVALID_HTML_CHARS with '_', collapses multiple '_' chars
- """
- return Location._clean(value, INVALID_HTML_CHARS)
-
- @staticmethod
- def is_valid(value):
- '''
- Check if the value is a valid location, in any acceptable format.
- '''
- try:
- Location(value)
- except InvalidLocationError:
- return False
- return True
-
- @staticmethod
- def ensure_fully_specified(location):
- '''Make sure location is valid, and fully specified. Raises
- InvalidLocationError or InsufficientSpecificationError if not.
-
- returns a Location object corresponding to location.
- '''
- loc = Location(location)
- for key, val in loc.dict().iteritems():
- if key != 'revision' and val is None:
- raise InsufficientSpecificationError(location)
- return loc
-
- def __new__(_cls, loc_or_tag=None, org=None, course=None, category=None,
- name=None, revision=None):
- """
- Create a new location that is a clone of the specifed one.
-
- location - Can be any of the following types:
- string: should be of the form
- {tag}://{org}/{course}/{category}/{name}[@{revision}]
-
- list: should be of the form [tag, org, course, category, name, revision]
-
- dict: should be of the form {
- 'tag': tag,
- 'org': org,
- 'course': course,
- 'category': category,
- 'name': name,
- 'revision': revision,
- }
- Location: another Location object
-
- In both the dict and list forms, the revision is optional, and can be
- ommitted.
-
- Components must be composed of alphanumeric characters, or the
- characters '_', '-', and '.'. The name component is additionally allowed to have ':',
- which is interpreted specially for xml storage.
-
- Components may be set to None, which may be interpreted in some contexts
- to mean wildcard selection.
- """
- if (org is None and course is None and category is None and name is None and revision is None):
- location = loc_or_tag
- else:
- location = (loc_or_tag, org, course, category, name, revision)
-
- if location is None:
- return _LocationBase.__new__(_cls, *([None] * 6))
-
- def check_dict(dict_):
- # Order matters, so flatten out into a list
- keys = ['tag', 'org', 'course', 'category', 'name', 'revision']
- list_ = [dict_[k] for k in keys]
- check_list(list_)
-
- def check_list(list_):
- list_ = list(list_)
- for val in list_[:4] + [list_[5]]:
- _check_location_part(val, INVALID_CHARS)
- # names allow colons
- _check_location_part(list_[4], INVALID_CHARS_NAME)
-
- if isinstance(location, Location):
- return location
- elif isinstance(location, basestring):
- match = URL_RE.match(location)
- if match is None:
- log.debug(u"location %r doesn't match URL", location)
- raise InvalidLocationError(location)
- groups = match.groupdict()
- check_dict(groups)
- return _LocationBase.__new__(_cls, **groups)
- elif isinstance(location, (list, tuple)):
- if len(location) not in (5, 6):
- log.debug(u'location has wrong length')
- raise InvalidLocationError(location)
-
- if len(location) == 5:
- args = tuple(location) + (None,)
- else:
- args = tuple(location)
-
- check_list(args)
- return _LocationBase.__new__(_cls, *args)
- elif isinstance(location, dict):
- kwargs = dict(location)
- kwargs.setdefault('revision', None)
-
- check_dict(kwargs)
- return _LocationBase.__new__(_cls, **kwargs)
- else:
- raise InvalidLocationError(location)
-
- def url(self):
- """
- Return a string containing the URL for this location
- """
- url = u"{0.tag}://{0.org}/{0.course}/{0.category}/{0.name}".format(self)
- if self.revision:
- url += u"@{rev}".format(rev=self.revision) # pylint: disable=E1101
- return url
-
- def html_id(self):
- """
- Return a string with a version of the location that is safe for use in
- html id attributes
- """
- id_string = u"-".join(v for v in self.list() if v is not None)
- return Location.clean_for_html(id_string)
-
- def dict(self):
- """
- Return an OrderedDict of this locations keys and values. The order is
- tag, org, course, category, name, revision
- """
- return self._asdict()
-
- def list(self):
- return list(self)
-
- def __str__(self):
- return str(self.url().encode("utf-8"))
-
- def __unicode__(self):
- return self.url()
-
- def __repr__(self):
- return "Location%s" % repr(tuple(self))
-
- @property
- def course_id(self):
- """
- Return the ID of the Course that this item belongs to by looking
- at the location URL hierachy.
-
- Throws an InvalidLocationError is this location does not represent a course.
- """
- if self.category != 'course':
- raise InvalidLocationError(u'Cannot call course_id for {0} because it is not of category course'.format(self))
-
- return "/".join([self.org, self.course, self.name])
-
- COURSE_ID_RE = re.compile("""
- (?P<org>[^/]+)/
- (?P<course>[^/]+)/
- (?P<name>.*)
- """, re.VERBOSE)
-
- @staticmethod
- def parse_course_id(course_id):
- """
- Given a org/course/name course_id, return a dict of {"org": org, "course": course, "name": name}
-
- If the course_id is not of the right format, raise ValueError
- """
- match = Location.COURSE_ID_RE.match(course_id)
- if match is None:
- raise ValueError("{} is not of form ORG/COURSE/NAME".format(course_id))
- return match.groupdict()
-
- def _replace(self, **kwargs):
- """
- Return a new :class:`Location` with values replaced
- by the values specified in `**kwargs`
- """
- for name, value in kwargs.iteritems():
- if name == 'name':
- _check_location_part(value, INVALID_CHARS_NAME)
- else:
- _check_location_part(value, INVALID_CHARS)
-
- # namedtuple is an old-style class, so don't use super
- return _LocationBase._replace(self, **kwargs)
-
- def replace(self, **kwargs):
- '''
- Expose a public method for replacing location elements
- '''
- return self._replace(**kwargs)
-
class ModuleStoreRead(object):
"""
@@ -313,14 +38,14 @@ class ModuleStoreRead(object):
__metaclass__ = ABCMeta
@abstractmethod
- def has_item(self, course_id, location):
+ def has_item(self, usage_key):
"""
- Returns True if location exists in this ModuleStore.
+ Returns True if usage_key exists in this ModuleStore.
"""
pass
@abstractmethod
- def get_item(self, location, depth=0):
+ def get_item(self, usage_key, depth=0):
"""
Returns an XModuleDescriptor instance for the item at location.
@@ -330,7 +55,7 @@ class ModuleStoreRead(object):
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
- location: Something that can be passed to Location
+ usage_key: A :class:`.UsageKey` subclass instance
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
@@ -340,23 +65,16 @@ class ModuleStoreRead(object):
pass
@abstractmethod
- def get_instance(self, course_id, location, depth=0):
- """
- Get an instance of this location, with policy for course_id applied.
- TODO (vshnayder): this may want to live outside the modulestore eventually
- """
- pass
-
- @abstractmethod
- def get_item_errors(self, location):
+ def get_course_errors(self, course_key):
"""
Return a list of (msg, exception-or-None) errors that the modulestore
- encountered when loading the item at location.
-
- location : something that can be passed to Location
+ encountered when loading the course at course_id.
Raises the same exceptions as get_item if the location isn't found or
isn't fully specified.
+
+ Args:
+ course_key (:class:`.CourseKey`): The course to check for errors
"""
pass
@@ -376,6 +94,68 @@ class ModuleStoreRead(object):
"""
pass
+ def _block_matches(self, fields_or_xblock, qualifiers):
+ '''
+ Return True or False depending on whether the field value (block contents)
+ matches the qualifiers as per get_items. Note, only finds directly set not
+ inherited nor default value matches.
+ For substring matching pass a regex object.
+ for arbitrary function comparison such as date time comparison, pass
+ the function as in start=lambda x: x < datetime.datetime(2014, 1, 1, 0, tzinfo=pytz.UTC)
+
+ Args:
+ fields_or_xblock (dict or XBlock): either the json blob (from the db or get_explicitly_set_fields)
+ or the xblock.fields() value or the XBlock from which to get those values
+ qualifiers (dict): field: searchvalue pairs.
+ '''
+ if isinstance(fields_or_xblock, XBlock):
+ fields = fields_or_xblock.fields
+ xblock = fields_or_xblock
+ is_xblock = True
+ else:
+ fields = fields_or_xblock
+ is_xblock = False
+
+ def _is_set_on(key):
+ """
+ Is this key set in fields? (return tuple of boolean and value). A helper which can
+ handle fields either being the json doc or xblock fields. Is inner function to restrict
+ use and to access local vars.
+ """
+ if key not in fields:
+ return False, None
+ field = fields[key]
+ if is_xblock:
+ return field.is_set_on(fields_or_xblock), getattr(xblock, key)
+ else:
+ return True, field
+
+ for key, criteria in qualifiers.iteritems():
+ is_set, value = _is_set_on(key)
+ if not is_set:
+ return False
+ if not self._value_matches(value, criteria):
+ return False
+ return True
+
+ def _value_matches(self, target, criteria):
+ '''
+ helper for _block_matches: does the target (field value) match the criteria?
+
+ If target is a list, do any of the list elements meet the criteria
+ If the criteria is a regex, does the target match it?
+ If the criteria is a function, does invoking it on the target yield something truthy?
+ Otherwise, is the target == criteria
+ '''
+ if isinstance(target, list):
+ return any(self._value_matches(ele, criteria) for ele in target)
+ elif isinstance(criteria, re._pattern_type):
+ return criteria.search(target) is not None
+ elif callable(criteria):
+ return criteria(target)
+ else:
+ return criteria == target
+
@abstractmethod
def get_courses(self):
'''
@@ -385,14 +165,26 @@ class ModuleStoreRead(object):
pass
@abstractmethod
- def get_course(self, course_id):
+ def get_course(self, course_id, depth=None):
'''
- Look for a specific course id. Returns the course descriptor, or None if not found.
+ Look for a specific course by its id (:class:`CourseKey`).
+ Returns the course descriptor, or None if not found.
'''
pass
@abstractmethod
- def get_parent_locations(self, location, course_id):
+ def has_course(self, course_id, ignore_case=False):
+ '''
+ Look for a specific course id. Returns whether it exists.
+ Args:
+ course_id (CourseKey):
+ ignore_case (boolean): some modulestores are case-insensitive. Use this flag
+ to search for whether a potentially conflicting course exists in that case.
+ '''
+ pass
+
+ @abstractmethod
+ def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
course. Needed for path_to_location().
@@ -401,7 +193,7 @@ class ModuleStoreRead(object):
pass
@abstractmethod
- def get_orphans(self, course_location, branch):
+ def get_orphans(self, course_key):
"""
Get all of the xblocks in the given course which have no parents and are not of types which are
usually orphaned. NOTE: may include xblocks which still have references via xblocks which don't
@@ -445,7 +237,7 @@ class ModuleStoreWrite(ModuleStoreRead):
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
- :raises VersionConflictError: if package_id and version_guid given and the current
+ :raises VersionConflictError: if org, offering, and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
"""
pass
@@ -461,11 +253,39 @@ class ModuleStoreWrite(ModuleStoreRead):
:param force: fork the structure and don't update the course draftVersion if there's a version
conflict (only applicable to version tracking and conflict detecting persistence stores)
- :raises VersionConflictError: if package_id and version_guid given and the current
+ :raises VersionConflictError: if org, offering, and version_guid given and the current
version head != version_guid and force is not True. (only applicable to version tracking stores)
"""
pass
+ @abstractmethod
+ def create_course(self, org, offering, user_id=None, fields=None, **kwargs):
+ """
+ Creates and returns the course.
+
+ Args:
+ org (str): the organization that owns the course
+ offering (str): the name of the course offering
+ user_id: id of the user creating the course
+ fields (dict): Fields to set on the course at initialization
+ kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
+
+ Returns: a CourseDescriptor
+ """
+ pass
+
+ @abstractmethod
+ def delete_course(self, course_key, user_id=None):
+ """
+ Deletes the course. It may be a soft or hard delete. It may or may not remove the xblock definitions
+ depending on the persistence layer and how tightly bound the xblocks are to the course.
+
+ Args:
+ course_key (CourseKey): which course to delete
+ user_id: id of the user deleting the course
+ """
+ pass
+
class ModuleStoreReadBase(ModuleStoreRead):
'''
@@ -477,7 +297,7 @@ class ModuleStoreReadBase(ModuleStoreRead):
self,
doc_store_config=None, # ignore if passed up
metadata_inheritance_cache_subsystem=None, request_cache=None,
- modulestore_update_signal=None, xblock_mixins=(), xblock_select=None,
+ xblock_mixins=(), xblock_select=None,
# temporary parms to enable backward compatibility. remove once all envs migrated
db=None, collection=None, host=None, port=None, tz_aware=True, user=None, password=None,
# allow lower level init args to pass harmlessly
@@ -486,38 +306,22 @@ class ModuleStoreReadBase(ModuleStoreRead):
'''
Set up the error-tracking logic.
'''
- self._location_errors = {} # location -> ErrorLog
+ self._course_errors = defaultdict(make_error_tracker) # location -> ErrorLog
self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem
- self.modulestore_update_signal = modulestore_update_signal
self.request_cache = request_cache
self.xblock_mixins = xblock_mixins
self.xblock_select = xblock_select
- def _get_errorlog(self, location):
- """
- If we already have an errorlog for this location, return it. Otherwise,
- create one.
+ def get_course_errors(self, course_key):
"""
- location = Location(location)
- if location not in self._location_errors:
- self._location_errors[location] = make_error_tracker()
- return self._location_errors[location]
-
- def get_item_errors(self, location):
- """
- Return list of errors for this location, if any. Raise the same
- errors as get_item if location isn't present.
-
- NOTE: For now, the only items that track errors are CourseDescriptors in
- the xml datastore. This will return an empty list for all other items
- and datastores.
+ Return list of errors for this :class:`.CourseKey`, if any. Raise the same
+ errors as get_item if course_key isn't present.
"""
# check that item is present and raise the promised exceptions if needed
# TODO (vshnayder): post-launch, make errors properties of items
# self.get_item(location)
-
- errorlog = self._get_errorlog(location)
- return errorlog.errors
+ assert(isinstance(course_key, CourseKey))
+ return self._course_errors[course_key].errors
def get_errored_courses(self):
"""
@@ -528,42 +332,36 @@ class ModuleStoreReadBase(ModuleStoreRead):
"""
return {}
- def get_course(self, course_id):
- """Default impl--linear search through course list"""
- for c in self.get_courses():
- if c.id == course_id:
- return c
- return None
-
- def update_item(self, xblock, user_id=None, allow_not_found=False, force=False):
+ def get_course(self, course_id, depth=None):
"""
- Update the given xblock's persisted repr. Pass the user's unique id which the persistent store
- should save with the update if it has that ability.
-
- :param allow_not_found: whether this method should raise an exception if the given xblock
- has not been persisted before.
- :param force: fork the structure and don't update the course draftVersion if there's a version
- conflict (only applicable to version tracking and conflict detecting persistence stores)
+ See ModuleStoreRead.get_course
- :raises VersionConflictError: if package_id and version_guid given and the current
- version head != version_guid and force is not True. (only applicable to version tracking stores)
- """
- raise NotImplementedError
-
- def delete_item(self, location, user_id=None, delete_all_versions=False, delete_children=False, force=False):
+ Default impl--linear search through course list
"""
- Delete an item from persistence. Pass the user's unique id which the persistent store
- should save with the update if it has that ability.
+ assert(isinstance(course_id, CourseKey))
+ for course in self.get_courses():
+ if course.id == course_id:
+ return course
+ return None
- :param delete_all_versions: removes both the draft and published version of this item from
- the course if using draft and old mongo. Split may or may not implement this.
- :param force: fork the structure and don't update the course draftVersion if there's a version
- conflict (only applicable to version tracking and conflict detecting persistence stores)
+ def has_course(self, course_id, ignore_case=False):
+ """
+ Look for a specific course id. Returns whether it exists.
+ Args:
+ course_id (CourseKey):
+ ignore_case (boolean): some modulestores are case-insensitive. Use this flag
+ to search for whether a potentially conflicting course exists in that case.
+ """
+ # linear search through list
+ assert(isinstance(course_id, CourseKey))
+ if ignore_case:
+ return any(
+ (c.id.org.lower() == course_id.org.lower() and c.id.offering.lower() == course_id.offering.lower())
+ for c in self.get_courses()
+ )
+ else:
+ return any(c.id == course_id for c in self.get_courses())
- :raises VersionConflictError: if package_id and version_guid given and the current
- version head != version_guid and force is not True. (only applicable to version tracking stores)
- """
- raise NotImplementedError
class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
'''
@@ -592,6 +390,36 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
result[field.scope][field_name] = value
return result
+ def update_item(self, xblock, user_id=None, allow_not_found=False, force=False):
+ """
+ Update the given xblock's persisted repr. Pass the user's unique id which the persistent store
+ should save with the update if it has that ability.
+
+ :param allow_not_found: whether this method should raise an exception if the given xblock
+ has not been persisted before.
+ :param force: fork the structure and don't update the course draftVersion if there's a version
+ conflict (only applicable to version tracking and conflict detecting persistence stores)
+
+ :raises VersionConflictError: if org, offering, and version_guid given and the current
+ version head != version_guid and force is not True. (only applicable to version tracking stores)
+ """
+ raise NotImplementedError
+
+ def delete_item(self, location, user_id=None, delete_all_versions=False, delete_children=False, force=False):
+ """
+ Delete an item from persistence. Pass the user's unique id which the persistent store
+ should save with the update if it has that ability.
+
+ :param delete_all_versions: removes both the draft and published version of this item from
+ the course if using draft and old mongo. Split may or may not implement this.
+ :param force: fork the structure and don't update the course draftVersion if there's a version
+ conflict (only applicable to version tracking and conflict detecting persistence stores)
+
+ :raises VersionConflictError: if org, offering, and version_guid given and the current
+ version head != version_guid and force is not True. (only applicable to version tracking stores)
+ """
+ raise NotImplementedError
+
def only_xmodules(identifier, entry_points):
"""Only use entry_points that are supplied by the xmodule package"""
diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py
index abefeae84fc..c0c67d06f60 100644
--- a/common/lib/xmodule/xmodule/modulestore/django.py
+++ b/common/lib/xmodule/xmodule/modulestore/django.py
@@ -66,7 +66,6 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
return class_(
metadata_inheritance_cache_subsystem=metadata_inheritance_cache,
request_cache=request_cache,
- modulestore_update_signal=Signal(providing_args=['modulestore', 'course_id', 'location']),
xblock_mixins=getattr(settings, 'XBLOCK_MIXINS', ()),
xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None),
doc_store_config=doc_store_config,
diff --git a/common/lib/xmodule/xmodule/modulestore/exceptions.py b/common/lib/xmodule/xmodule/modulestore/exceptions.py
index 47d810f06d5..bf85cf50688 100644
--- a/common/lib/xmodule/xmodule/modulestore/exceptions.py
+++ b/common/lib/xmodule/xmodule/modulestore/exceptions.py
@@ -37,6 +37,13 @@ class DuplicateItemError(Exception):
self.store = store
self.collection = collection
+ def __str__(self, *args, **kwargs):
+ """
+ Print info about what's duplicated
+ """
+ return '{0.store}[{0.collection}] already has {0.element_id}'.format(
+ self, Exception.__str__(self, *args, **kwargs)
+ )
class VersionConflictError(Exception):
"""
diff --git a/common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py b/common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
index eedb9333366..929e0e9bb04 100644
--- a/common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
+++ b/common/lib/xmodule/xmodule/modulestore/loc_mapper_store.py
@@ -5,11 +5,12 @@ from random import randint
import re
import pymongo
import bson.son
+import urllib
from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError
from xmodule.modulestore.locator import BlockUsageLocator, CourseLocator
-from xmodule.modulestore import Location
-import urllib
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
+from xmodule.modulestore.keys import UsageKey
class LocMapperStore(object):
@@ -27,6 +28,7 @@ class LocMapperStore(object):
or dominant store, but that's not a requirement. This store creates its own connection.
'''
+ SCHEMA_VERSION = 1
def __init__(
self, cache, host, db, collection, port=27017, user=None, password=None,
**kwargs
@@ -39,6 +41,7 @@ class LocMapperStore(object):
host=host,
port=port,
tz_aware=True,
+ document_class=bson.son.SON,
**kwargs
),
db
@@ -51,31 +54,21 @@ class LocMapperStore(object):
self.cache = cache
# location_map functions
- def create_map_entry(self, course_location, package_id=None, draft_branch='draft', prod_branch='published',
+ def create_map_entry(self, course_key, org=None, offering=None, draft_branch='draft', prod_branch='published',
block_map=None):
"""
- Add a new entry to map this course_location to the new style CourseLocator.package_id. If package_id is not
- provided, it creates the default map of using org.course.name from the location if
- the location.category = 'course'; otherwise, it uses org.course.
+ Add a new entry to map this SlashSeparatedCourseKey to the new style CourseLocator.org & offering. If
+ org and offering are not provided, it defaults them based on course_key.
- You can create more than one mapping to the
- same package_id target. In that case, the reverse translate will be arbitrary (no guarantee of which wins).
- The use
- case for more than one mapping is to map both org/course/run and org/course to the same new package_id thus
- making a default for org/course. When querying for just org/course, the translator will prefer any entry
- which does not have a name in the _id; otherwise, it will return an arbitrary match.
+ WARNING: Exactly 1 CourseLocator key should index a given SlashSeparatedCourseKey.
+ We provide no mechanism to enforce this assertion.
- Note: the opposite is not true. That is, it never makes sense to use 2 different CourseLocator.package_id
- keys to index the same old Locator org/course/.. pattern. There's no checking to ensure you don't do this.
-
- NOTE: if there's already an entry w the given course_location, this may either overwrite that entry or
+ NOTE: if there's already an entry w the given course_key, this may either overwrite that entry or
throw an error depending on how mongo is configured.
- :param course_location: a Location preferably whose category is 'course'. Unlike the other
- map methods, this one doesn't take the old-style course_id. It should be called with
- a course location not a block location; however, if called w/ a non-course Location, it creates
- a "default" map for the org/course pair to a new package_id.
- :param package_id: the CourseLocator style package_id
+ :param course_key (SlashSeparatedCourseKey): a SlashSeparatedCourseKey
+ :param org (string): the CourseLocator style org
+ :param offering (string): the CourseLocator offering
:param draft_branch: the branch name to assign for drafts. This is hardcoded because old mongo had
a fixed notion that there was 2 and only 2 versions for modules: draft and production. The old mongo
did not, however, require that a draft version exist. The new one, however, does require a draft to
@@ -85,54 +78,49 @@ class LocMapperStore(object):
to publish).
:param block_map: an optional map to specify preferred names for blocks where the keys are the
Location block names and the values are the BlockUsageLocator.block_id.
- """
- if package_id is None:
- if course_location.category == 'course':
- package_id = u"{0.org}.{0.course}.{0.name}".format(course_location)
- else:
- package_id = u"{0.org}.{0.course}".format(course_location)
- # very like _interpret_location_id but w/o the _id
- location_id = self._construct_location_son(
- course_location.org, course_location.course,
- course_location.name if course_location.category == 'course' else None
- )
- # create location id with lower case
- location_id_lower = self._construct_lower_location_son(
- course_location.org, course_location.course,
- course_location.name if course_location.category == 'course' else None
- )
- try:
- self.location_map.insert({
- '_id': location_id,
- 'lower_id': location_id_lower,
- 'course_id': package_id,
- 'lower_course_id': package_id.lower(),
- 'draft_branch': draft_branch,
- 'prod_branch': prod_branch,
- 'block_map': block_map or {},
- })
- except pymongo.errors.DuplicateKeyError:
- # update old entry with 'lower_id' and 'lower_course_id'
- location_update = {'lower_id': location_id_lower, 'lower_course_id': package_id.lower()}
- self.location_map.update({'_id': location_id}, {'$set': location_update})
-
- return package_id
-
- def translate_location(self, old_style_course_id, location, published=True,
+ Returns:
+ :class:`CourseLocator` representing the new id for the course
+
+ Raises:
+ ValueError if one and only one of org and offering is provided. Provide either both or neither.
+ """
+ if org is None and offering is None:
+ assert(isinstance(course_key, SlashSeparatedCourseKey))
+ org = course_key.org
+ offering = u"{0.course}.{0.run}".format(course_key)
+ elif org is None or offering is None:
+ raise ValueError(
+ u"Either supply both org and offering or neither. Not just one: {}, {}".format(org, offering)
+ )
+
+ # very like _interpret_location_id but using mongo subdoc lookup (more performant)
+ course_son = self._construct_course_son(course_key)
+
+ self.location_map.insert({
+ '_id': course_son,
+ 'org': org,
+ 'lower_org': org.lower(),
+ 'offering': offering,
+ 'lower_offering': offering.lower(),
+ 'draft_branch': draft_branch,
+ 'prod_branch': prod_branch,
+ 'block_map': block_map or {},
+ 'schema': self.SCHEMA_VERSION,
+ })
+
+ return CourseLocator(org, offering)
+
+ def translate_location(self, location, published=True,
add_entry_if_missing=True, passed_block_id=None):
"""
- Translate the given module location to a Locator. If the mapping has the run id in it, then you
- should provide old_style_course_id with that run id in it to disambiguate the mapping if there exists more
- than one entry in the mapping table for the org.course.
+ Translate the given module location to a Locator.
The rationale for auto adding entries was that there should be a reasonable default translation
- if the code just trips into this w/o creating translations. The downfall is that ambiguous course
- locations may generate conflicting block_ids.
+ if the code just trips into this w/o creating translations.
Will raise ItemNotFoundError if there's no mapping and add_entry_if_missing is False.
- :param old_style_course_id: the course_id used in old mongo not the new one (optional, will use location)
:param location: a Location pointing to a module
:param published: a boolean to indicate whether the caller wants the draft or published branch.
:param add_entry_if_missing: a boolean as to whether to raise ItemNotFoundError or to create an entry if
@@ -144,43 +132,32 @@ class LocMapperStore(object):
NOTE: unlike old mongo, draft branches contain the whole course; so, it applies to all category
of locations including course.
"""
- location_id = self._interpret_location_course_id(old_style_course_id, location)
- if old_style_course_id is None:
- old_style_course_id = self._generate_location_course_id(location_id)
+ course_son = self._interpret_location_course_id(location.course_key)
- cached_value = self._get_locator_from_cache(old_style_course_id, location, published)
+ cached_value = self._get_locator_from_cache(location, published)
if cached_value:
return cached_value
- maps = self.location_map.find(location_id)
- maps = list(maps)
- if len(maps) == 0:
+ entry = self.location_map.find_one(course_son)
+ if entry is None:
if add_entry_if_missing:
# create a new map
- course_location = location.replace(category='course', name=location_id['_id']['name'])
- self.create_map_entry(course_location)
- entry = self.location_map.find_one(location_id)
+ self.create_map_entry(location.course_key)
+ entry = self.location_map.find_one(course_son)
else:
raise ItemNotFoundError(location)
- elif len(maps) == 1:
- entry = maps[0]
else:
- # find entry w/o name, if any; otherwise, pick arbitrary
- entry = maps[0]
- for item in maps:
- if 'name' not in item['_id']:
- entry = item
- break
+ entry = self._migrate_if_necessary([entry])[0]
block_id = entry['block_map'].get(self.encode_key_for_mongo(location.name))
if block_id is None:
if add_entry_if_missing:
block_id = self._add_to_block_map(
- location, location_id, entry['block_map'], passed_block_id
+ location, course_son, entry['block_map'], passed_block_id
)
else:
raise ItemNotFoundError(location)
- elif isinstance(block_id, dict):
+ else:
# jump_to_id uses a None category.
if location.category is None:
if len(block_id) == 1:
@@ -191,22 +168,29 @@ class LocMapperStore(object):
elif location.category in block_id:
block_id = block_id[location.category]
elif add_entry_if_missing:
- block_id = self._add_to_block_map(location, location_id, entry['block_map'])
+ block_id = self._add_to_block_map(location, course_son, entry['block_map'])
else:
raise ItemNotFoundError(location)
- else:
- raise InvalidLocationError()
+ prod_course_locator = CourseLocator(
+ org=entry['org'],
+ offering=entry['offering'],
+ branch=entry['prod_branch']
+ )
published_usage = BlockUsageLocator(
- package_id=entry['course_id'], branch=entry['prod_branch'], block_id=block_id)
+ prod_course_locator,
+ block_id=block_id
+ )
draft_usage = BlockUsageLocator(
- package_id=entry['course_id'], branch=entry['draft_branch'], block_id=block_id)
+ prod_course_locator.for_branch(entry['draft_branch']),
+ block_id=block_id
+ )
if published:
result = published_usage
else:
result = draft_usage
- self._cache_location_map_entry(old_style_course_id, location, published_usage, draft_usage)
+ self._cache_location_map_entry(location, published_usage, draft_usage)
return result
def translate_locator_to_location(self, locator, get_course=False, lower_only=False):
@@ -217,18 +201,22 @@ class LocMapperStore(object):
the block's block_id was previously stored in the
map (a side effect of translate_location or via add|update_block_location).
- If get_course, then rather than finding the map for this locator, it finds the 'course' root
- for the mapped course.
-
If there are no matches, it returns None.
- If there's more than one location to locator mapping to the same package_id, it looks for the first
- one with a mapping for the block block_id and picks that arbitrary course location.
-
- :param locator: a BlockUsageLocator
+ Args:
+ locator: a BlockUsageLocator to translate
+ get_course: rather than finding the map for this locator, returns the CourseKey
+ for the mapped course.
+ lower_only: (obsolete?) the locator's fields are lowercased and not the actual case
+ for the identifier (e.g., came from a sql db which lowercases all ids). Find the actual
+ case Location for the desired object
"""
if get_course:
- cached_value = self._get_course_location_from_cache(locator.package_id, lower_only)
+ cached_value = self._get_course_location_from_cache(
+ # if locator is already a course_key it won't have course_key attr
+ getattr(locator, 'course_key', locator),
+ lower_only
+ )
else:
cached_value = self._get_location_from_cache(locator)
if cached_value:
@@ -237,90 +225,118 @@ class LocMapperStore(object):
# This does not require that the course exist in any modulestore
# only that it has a mapping entry.
if lower_only:
- maps = self.location_map.find({'lower_course_id': locator.package_id.lower()})
+ # migrate any records which don't have the lower_org and lower_offering fields as
+ # this won't be able to find what it wants. (only needs to be run once ever per db,
+ # I'm not sure how to control that, but I'm putting some check here for once per launch)
+ if not getattr(self, 'lower_offering_migrated', False):
+ obsolete = self.location_map.find(
+ {'lower_org': {"$exists": False}, "lower_offering": {"$exists": False}, }
+ )
+ self._migrate_if_necessary(obsolete)
+ setattr(self, 'lower_offering_migrated', True)
+
+ entry = self.location_map.find_one(bson.son.SON([
+ ('lower_org', locator.org.lower()),
+ ('lower_offering', locator.offering.lower()),
+ ]))
else:
- maps = self.location_map.find({'course_id': locator.package_id})
+ # migrate any records which don't have the lower_org and lower_offering fields as
+ # this won't be able to find what it wants. (only needs to be run once ever per db,
+ # I'm not sure how to control that, but I'm putting some check here for once per launch)
+ if not getattr(self, 'offering_migrated', False):
+ obsolete = self.location_map.find(
+ {'org': {"$exists": False}, "offering": {"$exists": False}, }
+ )
+ self._migrate_if_necessary(obsolete)
+ setattr(self, 'offering_migrated', True)
+
+ entry = self.location_map.find_one(bson.son.SON([
+ ('org', locator.org),
+ ('lower_offering', locator.offering),
+ ]))
+
# look for one which maps to this block block_id
- if maps.count() == 0:
+ if entry is None:
return None
- result = None
- for candidate in maps:
- if get_course and 'name' in candidate['_id']:
- candidate_id = candidate['_id']
- return Location(
- 'i4x', candidate_id['org'], candidate_id['course'], 'course', candidate_id['name']
+ old_course_id = self._generate_location_course_id(entry['_id'])
+ if get_course:
+ return old_course_id
+
+ for old_name, cat_to_usage in entry['block_map'].iteritems():
+ for category, block_id in cat_to_usage.iteritems():
+ # cache all entries and then figure out if we have the one we want
+ # Always return revision=None because the
+ # old draft module store wraps locations as draft before
+ # trying to access things.
+ location = old_course_id.make_usage_key(
+ category,
+ self.decode_key_from_mongo(old_name)
+ )
+
+ if lower_only:
+ entry_org = "lower_org"
+ entry_offering = "lower_offering"
+ else:
+ entry_org = "org"
+ entry_offering = "offering"
+
+ published_locator = BlockUsageLocator(
+ CourseLocator(
+ org=entry[entry_org], offering=entry[entry_offering],
+ branch=entry['prod_branch']
+ ),
+ block_id=block_id
)
- old_course_id = self._generate_location_course_id(candidate['_id'])
- for old_name, cat_to_usage in candidate['block_map'].iteritems():
- for category, block_id in cat_to_usage.iteritems():
- # cache all entries and then figure out if we have the one we want
- # Always return revision=None because the
- # old draft module store wraps locations as draft before
- # trying to access things.
- location = Location(
- 'i4x',
- candidate['_id']['org'],
- candidate['_id']['course'],
- category,
- self.decode_key_from_mongo(old_name),
- None)
-
- if lower_only:
- candidate_key = "lower_course_id"
- else:
- candidate_key = "course_id"
-
- published_locator = BlockUsageLocator(
- candidate[candidate_key], branch=candidate['prod_branch'], block_id=block_id
- )
- draft_locator = BlockUsageLocator(
- candidate[candidate_key], branch=candidate['draft_branch'], block_id=block_id
- )
- self._cache_location_map_entry(old_course_id, location, published_locator, draft_locator)
-
- if get_course and category == 'course':
- result = location
- elif not get_course and block_id == locator.block_id:
- result = location
- if result is not None:
- return result
+ draft_locator = BlockUsageLocator(
+ CourseLocator(
+ org=entry[entry_org], offering=entry[entry_offering],
+ branch=entry['draft_branch']
+ ),
+ block_id=block_id
+ )
+ self._cache_location_map_entry(location, published_locator, draft_locator)
+
+ if block_id == locator.block_id:
+ return location
+
return None
- def translate_location_to_course_locator(self, old_style_course_id, location, published=True, lower_only=False):
+ def translate_location_to_course_locator(self, course_key, published=True):
"""
Used when you only need the CourseLocator and not a full BlockUsageLocator. Probably only
useful for get_items which wildcards name or category.
- :param course_id: old style course id
+ :param course_key: a CourseKey or a UsageKey
+ :param published: a boolean representing whether or not we should return the published or draft version
+
+ Returns a Courselocator
"""
- cached = self._get_course_locator_from_cache(old_style_course_id, published)
+ if isinstance(course_key, UsageKey):
+ course_key = course_key.course_key
+
+ cached = self._get_course_locator_from_cache(course_key, published)
if cached:
return cached
- location_id = self._interpret_location_course_id(old_style_course_id, location, lower_only)
+ course_son = self._interpret_location_course_id(course_key)
- maps = self.location_map.find(location_id)
- maps = list(maps)
- if len(maps) == 0:
- raise ItemNotFoundError(location)
- elif len(maps) == 1:
- entry = maps[0]
- else:
- # find entry w/o name, if any; otherwise, pick arbitrary
- entry = maps[0]
- for item in maps:
- if 'name' not in item['_id']:
- entry = item
- break
- published_course_locator = CourseLocator(package_id=entry['course_id'], branch=entry['prod_branch'])
- draft_course_locator = CourseLocator(package_id=entry['course_id'], branch=entry['draft_branch'])
- self._cache_course_locator(old_style_course_id, published_course_locator, draft_course_locator)
+ entry = self.location_map.find_one(course_son)
+ if entry is None:
+ raise ItemNotFoundError(course_key)
+
+ published_course_locator = CourseLocator(
+ org=entry['org'], offering=entry['offering'], branch=entry['prod_branch']
+ )
+ draft_course_locator = CourseLocator(
+ org=entry['org'], offering=entry['offering'], branch=entry['draft_branch']
+ )
+ self._cache_course_locator(course_key, published_course_locator, draft_course_locator)
if published:
return published_course_locator
else:
return draft_course_locator
- def _add_to_block_map(self, location, location_id, block_map, block_id=None):
+ def _add_to_block_map(self, location, course_son, block_map, block_id=None):
'''add the given location to the block_map and persist it'''
if block_id is None:
if self._block_id_is_guid(location.name):
@@ -335,63 +351,33 @@ class LocMapperStore(object):
block_id = self._verify_uniqueness(location.name, block_map)
encoded_location_name = self.encode_key_for_mongo(location.name)
block_map.setdefault(encoded_location_name, {})[location.category] = block_id
- self.location_map.update(location_id, {'$set': {'block_map': block_map}})
+ self.location_map.update(course_son, {'$set': {'block_map': block_map}})
return block_id
- def _interpret_location_course_id(self, course_id, location, lower_only=False):
+ def _interpret_location_course_id(self, course_key):
"""
- Take the old style course id (org/course/run) and return a dict w/ a SON for querying the mapping table.
- If the course_id is empty, it uses location, but this may result in an inadequate id.
+ Take a CourseKey and return a SON for querying the mapping table.
- :param course_id: old style 'org/course/run' id from Location.course_id where Location.category = 'course'
- :param location: a Location object which may be to a module or a course. Provides partial info
- if course_id is omitted.
+ :param course_key: a CourseKey object for a course.
"""
- if course_id:
- # re doesn't allow ?P<_id.org> and ilk
- matched = re.match(r'([^/]+)/([^/]+)/([^/]+)', course_id)
- if lower_only:
- return {'lower_id': self._construct_lower_location_son(*matched.groups())}
- return {'_id': self._construct_location_son(*matched.groups())}
-
- if location.category == 'course':
- if lower_only:
- return {'lower_id': self._construct_lower_location_son(location.org, location.course, location.name)}
- return {'_id': self._construct_location_son(location.org, location.course, location.name)}
- else:
- return bson.son.SON([('_id.org', location.org), ('_id.course', location.course)])
+ return {'_id': self._construct_course_son(course_key)}
def _generate_location_course_id(self, entry_id):
"""
- Generate a Location course_id for the given entry's id.
+ Generate a CourseKey for the given entry's id.
"""
- # strip id envelope if any
- entry_id = entry_id.get('_id', entry_id)
- if entry_id.get('name', False):
- return u'{0[org]}/{0[course]}/{0[name]}'.format(entry_id)
- elif entry_id.get('_id.org', False):
- # the odd format one
- return u'{0[_id.org]}/{0[_id.course]}'.format(entry_id)
- else:
- return u'{0[org]}/{0[course]}'.format(entry_id)
+ return SlashSeparatedCourseKey(entry_id['org'], entry_id['course'], entry_id['name'])
- def _construct_location_son(self, org, course, name=None):
+ def _construct_course_son(self, course_key):
"""
- Construct the SON needed to repr the location for either a query or an insertion
+ Construct the SON needed to repr the course_key for either a query or an insertion
"""
- if name:
- return bson.son.SON([('org', org), ('course', course), ('name', name)])
- else:
- return bson.son.SON([('org', org), ('course', course)])
-
- def _construct_lower_location_son(self, org, course, name=None):
- """
- Construct the SON needed to represent the location with lower case
- """
- if name is not None:
- name = name.lower()
-
- return self._construct_location_son(org.lower(), course.lower(), name)
+ assert(isinstance(course_key, SlashSeparatedCourseKey))
+ return bson.son.SON([
+ ('org', course_key.org),
+ ('course', course_key.course),
+ ('name', course_key.run)
+ ])
def _block_id_is_guid(self, name):
"""
@@ -434,11 +420,11 @@ class LocMapperStore(object):
"""
return urllib.unquote(fieldname)
- def _get_locator_from_cache(self, old_course_id, location, published):
+ def _get_locator_from_cache(self, location, published):
"""
See if the location x published pair is in the cache. If so, return the mapped locator.
"""
- entry = self.cache.get(u'{}+{}'.format(old_course_id, location.url()))
+ entry = self.cache.get(u'{}+{}'.format(location.course_key, location))
if entry is not None:
if published:
return entry[0]
@@ -452,12 +438,12 @@ class LocMapperStore(object):
"""
if not old_course_id:
return None
- entry = self.cache.get(old_course_id)
+ entry = self.cache.get(unicode(old_course_id))
if entry is not None:
if published:
- return entry[0].as_course_locator()
+ return entry[0].course_key
else:
- return entry[1].as_course_locator()
+ return entry[1].course_key
def _get_location_from_cache(self, locator):
"""
@@ -483,9 +469,9 @@ class LocMapperStore(object):
"""
if not old_course_id:
return
- self.cache.set(old_course_id, (published_course_locator, draft_course_locator))
+ self.cache.set(unicode(old_course_id), (published_course_locator, draft_course_locator))
- def _cache_location_map_entry(self, old_course_id, location, published_usage, draft_usage):
+ def _cache_location_map_entry(self, location, published_usage, draft_usage):
"""
Cache the mapping from location to the draft and published Locators in entry.
Also caches the inverse. If the location is category=='course', it caches it for
@@ -497,25 +483,48 @@ class LocMapperStore(object):
setmany[u'courseIdLower+{}'.format(published_usage.package_id.lower())] = location
setmany[unicode(published_usage)] = location
setmany[unicode(draft_usage)] = location
- setmany[u'{}+{}'.format(old_course_id, location.url())] = (published_usage, draft_usage)
- setmany[old_course_id] = (published_usage, draft_usage)
+ setmany[unicode(location)] = (published_usage, draft_usage)
+ setmany[unicode(location.course_key)] = (published_usage, draft_usage)
self.cache.set_many(setmany)
- def delete_course_mapping(self, course_location):
+ def delete_course_mapping(self, course_key):
"""
Remove provided course location from loc_mapper and cache.
- :param course_location: a Location whose category is 'course'.
- """
- course_locator = self.translate_location(course_location.course_id, course_location)
- course_locator_draft = self.translate_location(
- course_location.course_id, course_location, published=False
- )
-
- self.location_map.remove({'course_id': course_locator.package_id})
- self._delete_cache_location_map_entry(
- course_location.course_id, course_location, course_locator, course_locator_draft
- )
+ :param course_key: a CourseKey for the course we wish to delete
+ """
+ self.location_map.remove(self._interpret_location_course_id(course_key))
+
+ # Remove the location of course (draft and published) from cache
+ cached_key = self.cache.get(unicode(course_key))
+ if cached_key:
+ delete_keys = []
+ published_locator = unicode(cached_key[0].course_key)
+ course_location = self._course_location_from_cache(published_locator)
+ delete_keys.append(u'courseId+{}'.format(published_locator))
+ delete_keys.append(u'courseIdLower+{}'.format(unicode(cached_key[0].course_key).lower()))
+ delete_keys.append(published_locator)
+ delete_keys.append(unicode(cached_key[1].course_key))
+ delete_keys.append(unicode(course_location))
+ delete_keys.append(unicode(course_key))
+ self.cache.delete_many(delete_keys)
+
+ def _migrate_if_necessary(self, entries):
+ """
+ Run the entries through any applicable schema updates and return the updated entries
+ """
+ entries = [
+ self._migrate[entry.get('schema', 0)](self, entry)
+ for entry in entries
+ ]
+ return entries
+
+ def _entry_id_to_son(self, entry_id):
+ return bson.son.SON([
+ ('org', entry_id['org']),
+ ('course', entry_id['course']),
+ ('name', entry_id['name'])
+ ])
def _delete_cache_location_map_entry(self, old_course_id, location, published_usage, draft_usage):
"""
@@ -528,6 +537,50 @@ class LocMapperStore(object):
delete_keys.append(unicode(published_usage))
delete_keys.append(unicode(draft_usage))
- delete_keys.append(u'{}+{}'.format(old_course_id, location.url()))
+ delete_keys.append(u'{}+{}'.format(old_course_id, location.to_deprecated_string()))
delete_keys.append(old_course_id)
self.cache.delete_many(delete_keys)
+
+ def _migrate_top(self, entry, updated=False):
+ """
+ Current version, so a no data change until next update. But since it's the top
+ it's responsible for persisting the record if it changed.
+ """
+ if updated:
+ entry['schema'] = self.SCHEMA_VERSION
+ entry_id = self._entry_id_to_son(entry['_id'])
+ self.location_map.update({'_id': entry_id}, entry)
+
+ return entry
+
+ def _migrate_0(self, entry):
+ """
+ If entry had an '_id' without a run, remove the whole record.
+
+ Add fields: schema, org, offering, lower_org, and lower_offering
+ Remove: course_id, lower_course_id
+ :param entry:
+ """
+ if 'name' not in entry['_id']:
+ entry_id = entry['_id']
+ entry_id = bson.son.SON([
+ ('org', entry_id['org']),
+ ('course', entry_id['course']),
+ ])
+ self.location_map.remove({'_id': entry_id})
+ return None
+
+ # add schema, org, offering, etc, remove old fields
+ entry['schema'] = 0
+ entry.pop('course_id', None)
+ entry.pop('lower_course_id', None)
+ old_course_id = SlashSeparatedCourseKey(entry['_id']['org'], entry['_id']['course'], entry['_id']['name'])
+ entry['org'] = old_course_id.org
+ entry['lower_org'] = old_course_id.org.lower()
+ entry['offering'] = old_course_id.offering.replace('/', '+')
+ entry['lower_offering'] = entry['offering'].lower()
+ return self._migrate_1(entry, True)
+
+ # insert new migrations just before _migrate_top. _migrate_top sets the schema version and
+ # saves the record
+ _migrate = [_migrate_0, _migrate_top]
diff --git a/common/lib/xmodule/xmodule/modulestore/locator.py b/common/lib/xmodule/xmodule/modulestore/locator.py
index a51ac765e19..deb11057293 100644
--- a/common/lib/xmodule/xmodule/modulestore/locator.py
+++ b/common/lib/xmodule/xmodule/modulestore/locator.py
@@ -5,17 +5,23 @@ Identifier for course resources.
from __future__ import absolute_import
import logging
import inspect
-from abc import ABCMeta, abstractmethod
+import re
+from abc import abstractmethod
from bson.objectid import ObjectId
from bson.errors import InvalidId
-from xmodule.modulestore.exceptions import InsufficientSpecificationError, OverSpecificationError
+from opaque_keys import OpaqueKey, InvalidKeyError
-from .parsers import parse_url, parse_package_id, parse_block_ref
-from .parsers import BRANCH_PREFIX, BLOCK_PREFIX, VERSION_PREFIX
-import re
-from xmodule.modulestore import Location
+from xmodule.modulestore.keys import CourseKey, UsageKey
+
+from xmodule.modulestore.parsers import (
+ parse_url,
+ parse_block_ref,
+ BRANCH_PREFIX,
+ BLOCK_PREFIX,
+ VERSION_PREFIX,
+ ALLOWED_ID_RE)
log = logging.getLogger(__name__)
@@ -32,41 +38,21 @@ class LocalId(object):
return "localid_{}".format(self.block_id or id(self))
-class Locator(object):
+class Locator(OpaqueKey):
"""
A locator is like a URL, it refers to a course resource.
Locator is an abstract base class: do not instantiate
"""
- __metaclass__ = ABCMeta
-
@abstractmethod
def url(self):
"""
Return a string containing the URL for this location. Raises
- InsufficientSpecificationError if the instance doesn't have a
+ InvalidKeyError if the instance doesn't have a
complete enough specification to generate a url
"""
- raise InsufficientSpecificationError()
-
- def __eq__(self, other):
- return self.__dict__ == other.__dict__
-
- def __hash__(self):
- """
- Hash on contents.
- """
- return hash(unicode(self))
-
- def __repr__(self):
- '''
- repr(self) returns something like this: CourseLocator("mit.eecs.6002x")
- '''
- classname = self.__class__.__name__
- if classname.find('.') != -1:
- classname = classname.split['.'][-1]
- return '%s("%s")' % (classname, unicode(self))
+ raise NotImplementedError()
def __str__(self):
'''
@@ -74,73 +60,14 @@ class Locator(object):
'''
return unicode(self).encode('utf-8')
- def __unicode__(self):
- '''
- unicode(self) returns something like this: "mit.eecs.6002x"
- '''
- return unicode(self).encode('utf-8')
-
@abstractmethod
def version(self):
"""
Returns the ObjectId referencing this specific location.
- Raises InsufficientSpecificationError if the instance
+ Raises InvalidKeyError if the instance
doesn't have a complete enough specification.
"""
- raise InsufficientSpecificationError()
-
- def set_property(self, property_name, new):
- """
- Initialize property to new value.
- If property has already been initialized to a different value, raise an exception.
- """
- current = getattr(self, property_name)
- if current and current != new:
- raise OverSpecificationError('%s cannot be both %s and %s' %
- (property_name, current, new))
- setattr(self, property_name, new)
-
- @staticmethod
- def to_locator_or_location(location):
- """
- Convert the given locator like thing to the appropriate type of object, or, if already
- that type, just return it. Returns an old Location, BlockUsageLocator,
- or DefinitionLocator.
-
- :param location: can be a Location, Locator, string, tuple, list, or dict.
- """
- if isinstance(location, (Location, Locator)):
- return location
- if isinstance(location, basestring):
- return Locator.parse_url(location)
- if isinstance(location, (list, tuple)):
- return Location(location)
- if isinstance(location, dict) and 'name' in location:
- return Location(location)
- if isinstance(location, dict):
- return BlockUsageLocator(**location)
- raise ValueError(location)
-
- URL_TAG_RE = re.compile(r'^(\w+)://')
- @staticmethod
- def parse_url(url):
- """
- Parse the url into one of the Locator types (must have a tag indicating type)
- Return the new instance. Supports i4x, cvx, edx, defx
-
- :param url: the url to parse
- """
- parsed = Locator.URL_TAG_RE.match(url)
- if parsed is None:
- raise ValueError(parsed)
- parsed = parsed.group(1)
- if parsed in ['i4x', 'c4x']:
- return Location(url)
- elif parsed == 'edx':
- return BlockUsageLocator(url)
- elif parsed == 'defx':
- return DefinitionLocator(url)
- return None
+ raise NotImplementedError()
@classmethod
def as_object_id(cls, value):
@@ -154,229 +81,212 @@ class Locator(object):
raise ValueError('"%s" is not a valid version_guid' % value)
-class CourseLocator(Locator):
+class BlockLocatorBase(Locator):
+
+ # Token separating org from offering
+ ORG_SEPARATOR = '+'
+
+ def version(self):
+ """
+ Returns the ObjectId referencing this specific location.
+ """
+ return self.version_guid
+
+ def url(self):
+ """
+ Return a string containing the URL for this location.
+ """
+ return self.NAMESPACE_SEPARATOR.join([self.CANONICAL_NAMESPACE, self._to_string()])
+
+ @classmethod
+ def _parse_url(cls, url):
+ """
+ url must be a string beginning with 'edx:' and containing
+ either a valid version_guid or org & offering (with optional branch), or both.
+ """
+ if not isinstance(url, basestring):
+ raise TypeError('%s is not an instance of basestring' % url)
+
+ parse = parse_url(url)
+ if not parse:
+ raise InvalidKeyError(cls, url)
+
+ if parse['version_guid']:
+ parse['version_guid'] = cls.as_object_id(parse['version_guid'])
+
+ return parse
+
+ @property
+ def package_id(self):
+ if self.org and self.offering:
+ return u'{}{}{}'.format(self.org, self.ORG_SEPARATOR, self.offering)
+ else:
+ return None
+
+
+class CourseLocator(BlockLocatorBase, CourseKey):
"""
Examples of valid CourseLocator specifications:
CourseLocator(version_guid=ObjectId('519665f6223ebd6980884f2b'))
- CourseLocator(package_id='mit.eecs.6002x')
- CourseLocator(package_id='mit.eecs.6002x/branch/published')
- CourseLocator(package_id='mit.eecs.6002x', branch='published')
- CourseLocator(url='edx://version/519665f6223ebd6980884f2b')
- CourseLocator(url='edx://mit.eecs.6002x')
- CourseLocator(url='edx://mit.eecs.6002x/branch/published')
- CourseLocator(url='edx://mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b')
-
- Should have at lease a specific package_id (id for the course as if it were a project w/
+ CourseLocator(org='mit.eecs', offering='6.002x')
+ CourseLocator(org='mit.eecs', offering='6002x', branch = 'published')
+ CourseLocator.from_string('edx:version/519665f6223ebd6980884f2b')
+ CourseLocator.from_string('version/519665f6223ebd6980884f2b')
+ CourseLocator.from_string('edx:mit.eecs+6002x')
+ CourseLocator.from_string('mit.eecs+6002x')
+ CourseLocator.from_string('edx:mit.eecs+6002x/branch/published')
+ CourseLocator.from_string('edx:mit.eecs+6002x/branch/published/version/519665f6223ebd6980884f2b')
+ CourseLocator.from_string('mit.eecs+6002x/branch/published/version/519665f6223ebd6980884f2b')
+
+ Should have at least a specific org & offering (id for the course as if it were a project w/
versions) with optional 'branch',
or version_guid (which points to a specific version). Can contain both in which case
the persistence layer may raise exceptions if the given version != the current such version
of the course.
"""
+ CANONICAL_NAMESPACE = 'course-locator'
+ KEY_FIELDS = ('org', 'offering', 'branch', 'version_guid')
- # Default values
- version_guid = None
- package_id = None
- branch = None
+ # stubs to fake out the abstractproperty class instrospection and allow treatment as attrs in instances
+ org = None
+ offering = None
- def __init__(self, url=None, version_guid=None, package_id=None, branch=None):
+ def __init__(self, org=None, offering=None, branch=None, version_guid=None):
"""
Construct a CourseLocator
- Caller may provide url (but no other parameters).
- Caller may provide version_guid (but no other parameters).
- Caller may provide package_id (optionally provide branch).
-
- Resulting CourseLocator will have either a version_guid property
- or a package_id (with optional branch) property, or both.
-
- version_guid must be an instance of bson.objectid.ObjectId or None
- url, package_id, and branch must be strings or None
+ Args:
+ version_guid (string or ObjectId): optional unique id for the version
+ org, offering (string): the standard definition. Optional only if version_guid given
+ branch (string): the branch such as 'draft', 'published', 'staged', 'beta'
"""
- self._validate_args(url, version_guid, package_id)
- if url:
- self.init_from_url(url)
if version_guid:
- self.init_from_version_guid(version_guid)
- if package_id or branch:
- self.init_from_package_id(package_id, branch)
- if self.version_guid is None and self.package_id is None:
- raise ValueError("Either version_guid or package_id should be set: {}".format(url))
+ version_guid = self.as_object_id(version_guid)
- def __unicode__(self):
- """
- Return a string representing this location.
- """
- parts = []
- if self.package_id:
- parts.append(unicode(self.package_id))
- if self.branch:
- parts.append(u"{prefix}{branch}".format(prefix=BRANCH_PREFIX, branch=self.branch))
- if self.version_guid:
- parts.append(u"{prefix}{guid}".format(prefix=VERSION_PREFIX, guid=self.version_guid))
- return u"/".join(parts)
+ if not all(field is None or ALLOWED_ID_RE.match(field) for field in [org, offering, branch]):
+ raise InvalidKeyError(self.__class__, [org, offering, branch])
- def url(self):
- """
- Return a string containing the URL for this location.
- """
- return u'edx://' + unicode(self)
+ super(CourseLocator, self).__init__(
+ org=org,
+ offering=offering,
+ branch=branch,
+ version_guid=version_guid
+ )
+
+ if self.version_guid is None and self.org is None and self.offering is None:
+ raise InvalidKeyError(self.__class__, "Either version_guid or org and offering should be set")
- def _validate_args(self, url, version_guid, package_id):
+ @classmethod
+ def _from_string(cls, serialized):
"""
- Validate provided arguments. Internal use only which is why it checks for each
- arg and doesn't use keyword
+ Return a CourseLocator parsing the given serialized string
+ :param serialized: matches the string to a CourseLocator
"""
- if not any((url, version_guid, package_id)):
- raise InsufficientSpecificationError("Must provide one of url, version_guid, package_id")
+ kwargs = cls._parse_url(serialized)
+ try:
+ return cls(**{key: kwargs.get(key) for key in cls.KEY_FIELDS})
+ except ValueError:
+ raise InvalidKeyError(cls, "Either version_guid or org and offering should be set: {}".format(serialized))
def is_fully_specified(self):
"""
- Returns True if either version_guid is specified, or package_id+branch
+ Returns True if either version_guid is specified, or org+offering+branch
are specified.
This should always return True, since this should be validated in the constructor.
"""
- return (self.version_guid is not None or
- (self.package_id is not None and self.branch is not None))
+ return (
+ self.version_guid is not None or
+ (self.org is not None and self.offering is not None and self.branch is not None)
+ )
- def set_package_id(self, new):
- """
- Initialize package_id to new value.
- If package_id has already been initialized to a different value, raise an exception.
+ def html_id(self):
"""
- self.set_property('package_id', new)
+ Generate a discussion group id based on course
- def set_branch(self, new):
- """
- Initialize branch to new value.
- If branch has already been initialized to a different value, raise an exception.
+ To make compatible with old Location object functionality. I don't believe this behavior fits at this
+ place, but I have no way to override. We should clearly define the purpose and restrictions of this
+ (e.g., I'm assuming periods are fine).
"""
- self.set_property('branch', new)
+ return unicode(self)
- def set_version_guid(self, new):
- """
- Initialize version_guid to new value.
- If version_guid has already been initialized to a different value, raise an exception.
- """
- self.set_property('version_guid', new)
+ def make_usage_key(self, block_type, block_id):
+ return BlockUsageLocator(
+ course_key=self,
+ block_id=block_id
+ )
- def as_course_locator(self):
- """
- Returns a copy of itself (downcasting) as a CourseLocator.
- The copy has the same CourseLocator fields as the original.
- The copy does not include subclass information, such as
- a block_id (a property of BlockUsageLocator).
- """
- return CourseLocator(package_id=self.package_id,
- version_guid=self.version_guid,
- branch=self.branch)
+ def make_asset_key(self, asset_type, path):
+ raise NotImplementedError()
- def url_reverse(self, prefix, postfix=''):
- """
- Do what reverse is supposed to do but seems unable to do. Generate a url using prefix unicode(self) postfix
- :param prefix: the beginning of the url (will be forced to begin and end with / if non-empty)
- :param postfix: the part to append to the url (will be forced to begin w/ / if non-empty)
+ def version_agnostic(self):
"""
- if prefix:
- if not prefix.endswith('/'):
- prefix += '/'
- if not prefix.startswith('/'):
- prefix = '/' + prefix
- else:
- prefix = '/'
- if postfix and not postfix.startswith('/'):
- postfix = '/' + postfix
- elif postfix is None:
- postfix = ''
- return prefix + unicode(self) + postfix
+ We don't care if the locator's version is not the current head; so, avoid version conflict
+ by reducing info.
+ Returns a copy of itself without any version info.
- def init_from_url(self, url):
+ :raises: ValueError if the block locator has no org & offering
"""
- url must be a string beginning with 'edx://' and containing
- either a valid version_guid or package_id (with optional branch), or both.
- """
- if isinstance(url, Locator):
- parse = url.__dict__
- elif not isinstance(url, basestring):
- raise TypeError('%s is not an instance of basestring' % url)
- else:
- parse = parse_url(url, tag_optional=True)
- if not parse:
- raise ValueError('Could not parse "%s" as a url' % url)
- self._set_value(
- parse, 'version_guid', lambda (new_guid): self.set_version_guid(self.as_object_id(new_guid))
+ return CourseLocator(
+ org=self.org,
+ offering=self.offering,
+ branch=self.branch,
+ version_guid=None
)
- self._set_value(parse, 'package_id', self.set_package_id)
- self._set_value(parse, 'branch', self.set_branch)
-
- def init_from_version_guid(self, version_guid):
- """
- version_guid must be an instance of bson.objectid.ObjectId,
- or able to be cast as one.
- If it's a string, attempt to cast it as an ObjectId first.
- """
- version_guid = self.as_object_id(version_guid)
-
- if not isinstance(version_guid, ObjectId):
- raise TypeError('%s is not an instance of ObjectId' % version_guid)
- self.set_version_guid(version_guid)
- def init_from_package_id(self, package_id, explicit_branch=None):
+ def course_agnostic(self):
"""
- package_id is a CourseLocator or a string like 'mit.eecs.6002x' or 'mit.eecs.6002x/branch/published'.
-
- Revision (optional) is a string like 'published'.
- It may be provided explicitly (explicit_branch) or embedded into package_id.
- If branch is part of package_id (".../branch/published"), parse it out separately.
- If branch is provided both ways, that's ok as long as they are the same value.
-
- If a block ('/block/HW3') is a part of package_id, it is ignored.
+ We only care about the locator's version not its course.
+ Returns a copy of itself without any course info.
+ :raises: ValueError if the block locator has no version_guid
"""
+ return CourseLocator(
+ org=None,
+ offering=None,
+ branch=None,
+ version_guid=self.version_guid
+ )
- if package_id:
- if isinstance(package_id, CourseLocator):
- package_id = package_id.package_id
- if not package_id:
- raise ValueError("%s does not have a valid package_id" % package_id)
-
- parse = parse_package_id(package_id)
- if not parse or parse['package_id'] is None:
- raise ValueError('Could not parse "%s" as a package_id' % package_id)
- self.set_package_id(parse['package_id'])
- rev = parse['branch']
- if rev:
- self.set_branch(rev)
- if explicit_branch:
- self.set_branch(explicit_branch)
-
- def version(self):
+ def for_branch(self, branch):
"""
- Returns the ObjectId referencing this specific location.
+ Return a new CourseLocator for another branch of the same course (also version agnostic)
"""
- return self.version_guid
+ if self.org is None:
+ raise InvalidKeyError(self.__class__, "Branches must have full course ids not just versions")
+ return CourseLocator(
+ org=self.org,
+ offering=self.offering,
+ branch=branch,
+ version_guid=None
+ )
- def html_id(self):
+ def for_version(self, version_guid):
"""
- Generate a discussion group id based on course
-
- To make compatible with old Location object functionality. I don't believe this behavior fits at this
- place, but I have no way to override. We should clearly define the purpose and restrictions of this
- (e.g., I'm assuming periods are fine).
+ Return a new CourseLocator for another version of the same course and branch. Usually used
+ when the head is updated (and thus the course x branch now points to this version)
"""
- return self.package_id
+ return CourseLocator(
+ org=self.org,
+ offering=self.offering,
+ branch=self.branch,
+ version_guid=version_guid
+ )
- def _set_value(self, parse, key, setter):
+ def _to_string(self):
"""
- Helper method that gets a value out of the dict returned by parse,
- and then sets the corresponding bit of information in this locator
- (via the supplied lambda 'setter'), unless the value is None.
+ Return a string representing this location.
"""
- value = parse.get(key, None)
- if value:
- setter(value)
+ parts = []
+ if self.offering:
+ parts.append(unicode(self.package_id))
+ if self.branch:
+ parts.append(u"{prefix}{branch}".format(prefix=BRANCH_PREFIX, branch=self.branch))
+ if self.version_guid:
+ parts.append(u"{prefix}{guid}".format(prefix=VERSION_PREFIX, guid=self.version_guid))
+ return u"/".join(parts)
-class BlockUsageLocator(CourseLocator):
+class BlockUsageLocator(BlockLocatorBase, UsageKey): # TODO implement UsageKey methods
"""
Encodes a location.
@@ -385,7 +295,7 @@ class BlockUsageLocator(CourseLocator):
the defined element in the course. Courses can be a version of an offering, the
current draft head, or the current production version.
- Locators can contain both a version and a package_id w/ branch. The split mongo functions
+ Locators can contain both a version and a org + offering w/ branch. The split mongo functions
may raise errors if these conflict w/ the current db state (i.e., the course's branch !=
the version_guid)
@@ -394,46 +304,33 @@ class BlockUsageLocator(CourseLocator):
block : guid
branch : string
"""
+ CANONICAL_NAMESPACE = 'edx'
+ KEY_FIELDS = ('course_key', 'block_id')
- # Default value
- block_id = None
+ # fake out class instrospection as this is an attr in this class's instances
+ course_key = None
- def __init__(self, url=None, version_guid=None, package_id=None,
- branch=None, block_id=None):
+ def __init__(self, course_key, block_id):
"""
Construct a BlockUsageLocator
- Caller may provide url, version_guid, or package_id, and optionally provide branch.
-
- The block_id may be specified, either explictly or as part of
- the url or package_id. If omitted, the locator is created but it
- has not yet been initialized.
-
- Resulting BlockUsageLocator will have a block_id property.
- It will have either a version_guid property or a package_id (with optional branch) property, or both.
-
- version_guid must be an instance of bson.objectid.ObjectId or None
- url, package_id, branch, and block_id must be strings or None
-
- """
- self._validate_args(url, version_guid, package_id)
- if url:
- self.init_block_ref_from_str(url)
- if package_id:
- self.init_block_ref_from_package_id(package_id)
- if block_id:
- self.init_block_ref(block_id)
- super(BlockUsageLocator, self).__init__(
- url=url,
- version_guid=version_guid,
- package_id=package_id,
- branch=branch
- )
+ """
+ block_id = self._parse_block_ref(block_id)
+ if block_id is None:
+ raise InvalidKeyError(self.__class__, "Missing block id")
- def is_initialized(self):
+ super(BlockUsageLocator, self).__init__(course_key=course_key, block_id=block_id)
+
+ @classmethod
+ def _from_string(cls, serialized):
"""
- Returns True if block_id has been initialized, else returns False
+ Requests CourseLocator to deserialize its part and then adds the local deserialization of block
"""
- return self.block_id is not None
+ course_key = CourseLocator._from_string(serialized)
+ parsed_parts = parse_url(serialized)
+ block_id = parsed_parts.get('block_id')
+ if block_id is None:
+ raise InvalidKeyError(cls, serialized)
+ return cls(course_key, block_id)
def version_agnostic(self):
"""
@@ -441,11 +338,12 @@ class BlockUsageLocator(CourseLocator):
by reducing info.
Returns a copy of itself without any version info.
- :raises: ValueError if the block locator has no package_id
+ :raises: ValueError if the block locator has no org and offering
"""
- return BlockUsageLocator(package_id=self.package_id,
- branch=self.branch,
- block_id=self.block_id)
+ return BlockUsageLocator(
+ course_key=self.course_key.version_agnostic(),
+ block_id=self.block_id
+ )
def course_agnostic(self):
"""
@@ -454,47 +352,63 @@ class BlockUsageLocator(CourseLocator):
:raises: ValueError if the block locator has no version_guid
"""
- return BlockUsageLocator(version_guid=self.version_guid,
- block_id=self.block_id)
+ return BlockUsageLocator(
+ course_key=self.course_key.course_agnostic(),
+ block_id=self.block_id
+ )
- def set_block_id(self, new):
+ def for_branch(self, branch):
"""
- Initialize block_id to new value.
- If block_id has already been initialized to a different value, raise an exception.
+ Return a UsageLocator for the same block in a different branch of the course.
"""
- self.set_property('block_id', new)
+ return BlockUsageLocator(
+ self.course_key.for_branch(branch),
+ block_id=self.block_id
+ )
- def init_block_ref(self, block_ref):
+ @classmethod
+ def _parse_block_ref(cls, block_ref):
if isinstance(block_ref, LocalId):
- self.set_block_id(block_ref)
+ return block_ref
else:
parse = parse_block_ref(block_ref)
if not parse:
- raise ValueError('Could not parse "%s" as a block_ref' % block_ref)
- self.set_block_id(parse['block'])
+ raise InvalidKeyError(cls, block_ref)
+ return parse.get('block_id')
+
+ @property
+ def definition_key(self):
+ raise NotImplementedError()
+
+ @property
+ def org(self):
+ return self.course_key.org
- def init_block_ref_from_str(self, value):
+ @property
+ def offering(self):
+ return self.course_key.offering
+
+ @property
+ def package_id(self):
+ return self.course_key.package_id
+
+ @property
+ def branch(self):
+ return self.course_key.branch
+
+ @property
+ def version_guid(self):
+ return self.course_key.version_guid
+
+ @property
+ def name(self):
"""
- Create a block locator from the given string which may be a url or just the repr (no tag)
+ The ambiguously named field from Location which code expects to find
"""
- if hasattr(value, 'block_id'):
- self.init_block_ref(value.block_id)
- return
- if not isinstance(value, basestring):
- return None
- parse = parse_url(value, tag_optional=True)
- if parse is None:
- raise ValueError('Could not parse "%s" as a url' % value)
- self._set_value(parse, 'block', self.set_block_id)
-
- def init_block_ref_from_package_id(self, package_id):
- if isinstance(package_id, CourseLocator):
- package_id = package_id.package_id
- assert package_id, "%s does not have a valid package_id"
- parse = parse_package_id(package_id)
- if parse is None:
- raise ValueError('Could not parse "%s" as a package_id' % package_id)
- self._set_value(parse, 'block', self.set_block_id)
+ return self.block_id
+
+ def is_fully_specified(self):
+ return self.course_key.is_fully_specified()
@classmethod
def make_relative(cls, course_locator, block_id):
@@ -502,40 +416,82 @@ class BlockUsageLocator(CourseLocator):
Return a new instance which has the given block_id in the given course
:param course_locator: may be a BlockUsageLocator in the same snapshot
"""
+ if hasattr(course_locator, 'course_key'):
+ course_locator = course_locator.course_key
return BlockUsageLocator(
- package_id=course_locator.package_id,
- version_guid=course_locator.version_guid,
- branch=course_locator.branch,
+ course_key=course_locator,
block_id=block_id
)
- def __unicode__(self):
+ def map_into_course(self, course_key):
+ """
+ Return a new instance which has the this block_id in the given course
+ :param course_key: a CourseKey object representing the new course to map into
+ """
+ return BlockUsageLocator.make_relative(course_key, self.block_id)
+
+ def url_reverse(self, prefix, postfix=''):
+ """
+ Do what reverse is supposed to do but seems unable to do. Generate a url using prefix unicode(self) postfix
+ :param prefix: the beginning of the url (will be forced to begin and end with / if non-empty)
+ :param postfix: the part to append to the url (will be forced to begin w/ / if non-empty)
+ """
+ if prefix:
+ if not prefix.endswith('/'):
+ prefix += '/'
+ if not prefix.startswith('/'):
+ prefix = '/' + prefix
+ else:
+ prefix = '/'
+ if postfix and not postfix.startswith('/'):
+ postfix = '/' + postfix
+ elif postfix is None:
+ postfix = ''
+ return prefix + unicode(self) + postfix
+
+ def _to_string(self):
"""
Return a string representing this location.
"""
- rep = super(BlockUsageLocator, self).__unicode__()
- return rep + '/' + BLOCK_PREFIX + unicode(self.block_id)
+ return u"{course_key}/{BLOCK_PREFIX}{block_id}".format(
+ course_key=self.course_key._to_string(),
+ BLOCK_PREFIX=BLOCK_PREFIX,
+ block_id=self.block_id
+ )
+
+ def html_id(self):
+ """
+ Generate a discussion group id based on course
+
+ To make compatible with old Location object functionality. I don't believe this behavior fits at this
+ place, but I have no way to override. We should clearly define the purpose and restrictions of this
+ (e.g., I'm assuming periods are fine).
+ """
+ return re.sub('[^\w-]', '-', self._to_string())
class DefinitionLocator(Locator):
"""
Container for how to locate a description (the course-independent content).
"""
+ CANONICAL_NAMESPACE = 'defx'
+ KEY_FIELDS = ('definition_id',)
+
+ URL_RE = re.compile(r'^defx:' + VERSION_PREFIX + '([^/]+)$', re.IGNORECASE)
- URL_RE = re.compile(r'^defx://' + VERSION_PREFIX + '([^/]+)$', re.IGNORECASE)
def __init__(self, definition_id):
if isinstance(definition_id, LocalId):
- self.definition_id = definition_id
+ super(DefinitionLocator, self).__init__(definition_id)
elif isinstance(definition_id, basestring):
regex_match = self.URL_RE.match(definition_id)
if regex_match is not None:
- self.definition_id = self.as_object_id(regex_match.group(1))
+ super(DefinitionLocator, self).__init__(self.as_object_id(regex_match.group(1)))
else:
- self.definition_id = self.as_object_id(definition_id)
+ super(DefinitionLocator, self).__init__(self.as_object_id(definition_id))
else:
- self.definition_id = self.as_object_id(definition_id)
+ super(DefinitionLocator, self).__init__(self.as_object_id(definition_id))
- def __unicode__(self):
+ def _to_string(self):
'''
Return a string representing this location.
unicode(self) returns something like this: "version/519665f6223ebd6980884f2b"
@@ -545,9 +501,9 @@ class DefinitionLocator(Locator):
def url(self):
"""
Return a string containing the URL for this location.
- url(self) returns something like this: 'defx://version/519665f6223ebd6980884f2b'
+ url(self) returns something like this: 'defx:version/519665f6223ebd6980884f2b'
"""
- return u'defx://' + unicode(self)
+ return u'defx:' + self._to_string()
def version(self):
"""
diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py
index 1304a902b4b..050f5aa62c9 100644
--- a/common/lib/xmodule/xmodule/modulestore/mixed.py
+++ b/common/lib/xmodule/xmodule/modulestore/mixed.py
@@ -6,16 +6,18 @@ In this way, courses can be served up both - say - XMLModuleStore or MongoModule
"""
import logging
+from uuid import uuid4
+from opaque_keys import InvalidKeyError
from . import ModuleStoreWriteBase
from xmodule.modulestore.django import create_modulestore_instance, loc_mapper
-from xmodule.modulestore import Location, SPLIT_MONGO_MODULESTORE_TYPE, XML_MODULESTORE_TYPE
-from xmodule.modulestore.locator import CourseLocator, Locator
-from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
-from uuid import uuid4
+from xmodule.modulestore import Location, XML_MODULESTORE_TYPE
+from xmodule.modulestore.locator import CourseLocator, Locator, BlockUsageLocator
+from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.modulestore.keys import CourseKey, UsageKey
from xmodule.modulestore.mongo.base import MongoModuleStore
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
-from xmodule.exceptions import UndefinedContext
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
log = logging.getLogger(__name__)
@@ -32,7 +34,13 @@ class MixedModuleStore(ModuleStoreWriteBase):
super(MixedModuleStore, self).__init__(**kwargs)
self.modulestores = {}
- self.mappings = mappings
+ self.mappings = {}
+
+ for course_id, store_name in mappings.iteritems():
+ try:
+ self.mappings[CourseKey.from_string(course_id)] = store_name
+ except InvalidKeyError:
+ self.mappings[SlashSeparatedCourseKey.from_deprecated_string(course_id)] = store_name
if 'default' not in stores:
raise Exception('Missing a default modulestore in the MixedModuleStore __init__ method.')
@@ -43,7 +51,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
# restrict xml to only load courses in mapping
store['OPTIONS']['course_ids'] = [
course_id
- for course_id, store_key in self.mappings.iteritems()
+ for course_id, store_key in mappings.iteritems()
if store_key == key
]
self.modulestores[key] = create_modulestore_instance(
@@ -53,10 +61,6 @@ class MixedModuleStore(ModuleStoreWriteBase):
store['OPTIONS'],
i18n_service=i18n_service,
)
- # If and when locations can identify their course, we won't need
- # these loc maps. They're needed for figuring out which store owns these locations.
- if is_xml:
- self.ensure_loc_maps_exist(key)
def _get_modulestore_for_courseid(self, course_id):
"""
@@ -69,54 +73,50 @@ class MixedModuleStore(ModuleStoreWriteBase):
mapping = self.mappings.get(course_id, 'default')
return self.modulestores[mapping]
- def has_item(self, course_id, reference):
+ def has_item(self, usage_key):
"""
Does the course include the xblock who's id is reference?
-
- :param course_id: a course_id or package_id (slashed or dotted)
- :param reference: a Location or BlockUsageLocator
"""
- store = self._get_modulestore_for_courseid(course_id)
- return store.has_item(course_id, reference)
+ store = self._get_modulestore_for_courseid(usage_key.course_key)
+ return store.has_item(usage_key)
- def get_item(self, location, depth=0):
+ def get_item(self, usage_key, depth=0):
"""
This method is explicitly not implemented as we need a course_id to disambiguate
We should be able to fix this when the data-model rearchitecting is done
"""
- # Although we shouldn't have both get_item and get_instance imho
- raise NotImplementedError
+ store = self._get_modulestore_for_courseid(usage_key.course_key)
+ return store.get_item(usage_key, depth)
- def get_instance(self, course_id, location, depth=0):
- store = self._get_modulestore_for_courseid(course_id)
- return store.get_instance(course_id, location, depth)
-
- def get_items(self, location, course_id=None, depth=0, qualifiers=None):
+ def get_items(self, course_key, settings=None, content=None, **kwargs):
"""
- Returns a list of XModuleDescriptor instances for the items
- that match location. Any element of location that is None is treated
- as a wildcard that matches any value. NOTE: don't use this to look for courses
- as the course_id is required. Use get_courses.
+ Returns:
+ list of XModuleDescriptor instances for the matching items within the course with
+ the given course_key
- location: either a Location possibly w/ None as wildcards for category or name or
- a Locator with at least a package_id and branch but possibly no block_id.
+ NOTE: don't use this to look for courses
+ as the course_key is required. Use get_courses.
- depth: An argument that some module stores may use to prefetch
- descendants of the queried modules for more efficient results later
- in the request. The depth is counted in the number of calls to
- get_children() to cache. None indicates to cache all descendants
+ Args:
+ course_key (CourseKey): the course identifier
+ settings (dict): fields to look for which have settings scope. Follows same syntax
+ and rules as kwargs below
+ content (dict): fields to look for which have content scope. Follows same syntax and
+ rules as kwargs below.
+ kwargs (key=value): what to look for within the course.
+ Common qualifiers are ``category`` or any field name. if the target field is a list,
+ then it searches for the given value in the list not list equivalence.
+ Substring matching pass a regex object.
+ For some modulestores, ``name`` is another commonly provided key (Location based stores)
+ For some modulestores,
+ you can search by ``edited_by``, ``edited_on`` providing either a datetime for == (probably
+ useless) or a function accepting one arg to do inequality
"""
- if not (course_id or hasattr(location, 'package_id')):
- raise Exception("Must pass in a course_id when calling get_items()")
-
- store = self._get_modulestore_for_courseid(course_id or getattr(location, 'package_id'))
- return store.get_items(location, course_id, depth, qualifiers)
+ if not isinstance(course_key, CourseKey):
+ raise Exception("Must pass in a course_key when calling get_items()")
- def _get_course_id_from_course_location(self, course_location):
- """
- Get the proper course_id based on the type of course_location
- """
- return getattr(course_location, 'course_id', None) or getattr(course_location, 'package_id', None)
+ store = self._get_modulestore_for_courseid(course_key)
+ return store.get_items(course_key, settings, content, **kwargs)
def get_courses(self):
'''
@@ -141,7 +141,8 @@ class MixedModuleStore(ModuleStoreWriteBase):
try:
# if there's no existing mapping, then the course can't have been in split
course_locator = loc_mapper().translate_location(
- course.location.course_id, course.location, add_entry_if_missing=False
+ course.location,
+ add_entry_if_missing=False
)
if unicode(course_locator) not in courses:
courses[course_location] = course
@@ -152,27 +153,47 @@ class MixedModuleStore(ModuleStoreWriteBase):
return courses.values()
- def get_course(self, course_id):
+ def get_course(self, course_key, depth=None):
"""
returns the course module associated with the course_id. If no such course exists,
it returns None
- :param course_id: must be either a string course_id or a CourseLocator
+ :param course_key: must be a CourseKey
"""
- store = self._get_modulestore_for_courseid(
- course_id.package_id if hasattr(course_id, 'package_id') else course_id
- )
+ assert(isinstance(course_key, CourseKey))
+ store = self._get_modulestore_for_courseid(course_key)
try:
- return store.get_course(course_id)
+ return store.get_course(course_key, depth=depth)
except ItemNotFoundError:
return None
- def get_parent_locations(self, location, course_id):
+ def has_course(self, course_id, ignore_case=False):
"""
- returns the parent locations for a given location and course_id
+ returns whether the course exists
+
+ Args:
+ * course_id (CourseKey)
+ * ignore_case (bool): Tf True, do a case insensitive search. If
+ False, do a case sensitive search
"""
+ assert(isinstance(course_id, CourseKey))
store = self._get_modulestore_for_courseid(course_id)
- return store.get_parent_locations(location, course_id)
+ return store.has_course(course_id, ignore_case)
+
+ def delete_course(self, course_key, user_id=None):
+ """
+ Remove the given course from its modulestore.
+ """
+ assert(isinstance(course_key, CourseKey))
+ store = self._get_modulestore_for_courseid(course_key)
+ return store.delete_course(course_key, user_id)
+
+ def get_parent_locations(self, location):
+ """
+ returns the parent locations for a given location
+ """
+ store = self._get_modulestore_for_courseid(location.course_key)
+ return store.get_parent_locations(location)
def get_modulestore_type(self, course_id):
"""
@@ -184,15 +205,14 @@ class MixedModuleStore(ModuleStoreWriteBase):
"""
return self._get_modulestore_for_courseid(course_id).get_modulestore_type(course_id)
- def get_orphans(self, course_location, branch):
+ def get_orphans(self, course_key):
"""
Get all of the xblocks in the given course which have no parents and are not of types which are
usually orphaned. NOTE: may include xblocks which still have references via xblocks which don't
use children to point to their dependents.
"""
- course_id = self._get_course_id_from_course_location(course_location)
- store = self._get_modulestore_for_courseid(course_id)
- return store.get_orphans(course_location, branch)
+ store = self._get_modulestore_for_courseid(course_key)
+ return store.get_orphans(course_key)
def get_errored_courses(self):
"""
@@ -204,106 +224,26 @@ class MixedModuleStore(ModuleStoreWriteBase):
errs.update(store.get_errored_courses())
return errs
- def _get_course_id_from_block(self, block, store):
- """
- Get the course_id from the block or from asking its store. Expensive.
- """
- try:
- return block.course_id
- except UndefinedContext:
- pass
- try:
- course = store._get_course_for_item(block.scope_ids.usage_id)
- if course is not None:
- return course.scope_ids.usage_id.course_id
- except Exception: # sorry, that method just raises vanilla Exception if it doesn't find course
- pass
-
- def _infer_course_id_try(self, location):
- """
- Create, Update, Delete operations don't require a fully-specified course_id, but
- there's no complete & sound general way to compute the course_id except via the
- proper modulestore. This method attempts several sound but not complete methods.
- :param location: an old style Location
- """
- if isinstance(location, CourseLocator):
- return location.package_id
- elif isinstance(location, basestring):
- try:
- location = Location(location)
- except InvalidLocationError:
- # try to parse as a course_id
- try:
- Location.parse_course_id(location)
- # it's already a course_id
- return location
- except ValueError:
- # cannot interpret the location
- return None
-
- # location is a Location at this point
- if location.category == 'course': # easiest case
- return location.course_id
- # try finding in loc_mapper
- try:
- # see if the loc mapper knows the course id (requires double translation)
- locator = loc_mapper().translate_location_to_course_locator(None, location)
- location = loc_mapper().translate_locator_to_location(locator, get_course=True)
- return location.course_id
- except ItemNotFoundError:
- pass
- # expensive query against all location-based modulestores to look for location.
- for store in self.modulestores.itervalues():
- if isinstance(location, store.reference_type):
- try:
- xblock = store.get_item(location)
- course_id = self._get_course_id_from_block(xblock, store)
- if course_id is not None:
- return course_id
- except NotImplementedError:
- blocks = store.get_items(location)
- if len(blocks) == 1:
- block = blocks[0]
- try:
- return block.course_id
- except UndefinedContext:
- pass
- except ItemNotFoundError:
- pass
- # if we get here, it must be in a Locator based store, but we won't be able to find
- # it.
- return None
-
- def create_course(self, course_id, user_id=None, store_name='default', **kwargs):
+ def create_course(self, org, offering, user_id=None, fields=None, store_name='default', **kwargs):
"""
Creates and returns the course.
- :param org: the org
- :param fields: a dict of xblock field name - value pairs for the course module.
- :param metadata: the old way of setting fields by knowing which ones are scope.settings v scope.content
- :param definition_data: the complement to metadata which is also a subset of fields
- :returns: course xblock
+ Args:
+ org (str): the organization that owns the course
+ offering (str): the name of the course offering
+ user_id: id of the user creating the course
+ fields (dict): Fields to set on the course at initialization
+ store_name (str): the name of the modulestore that we will create this course within
+ kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
+
+ Returns: a CourseDescriptor
"""
store = self.modulestores[store_name]
+
if not hasattr(store, 'create_course'):
raise NotImplementedError(u"Cannot create a course on store %s" % store_name)
- if store.get_modulestore_type(course_id) == SPLIT_MONGO_MODULESTORE_TYPE:
- try:
- course_dict = Location.parse_course_id(course_id)
- org = course_dict['org']
- course_id = "{org}.{course}.{name}".format(**course_dict)
- except ValueError:
- org = None
-
- org = kwargs.pop('org', org)
- fields = kwargs.pop('fields', {})
- fields.update(kwargs.pop('metadata', {}))
- fields.update(kwargs.pop('definition_data', {}))
- course = store.create_course(course_id, org, user_id, fields=fields, **kwargs)
- else: # assume mongo
- course = store.create_course(course_id, **kwargs)
- return course
+ return store.create_course(org, offering, user_id, fields, **kwargs)
def create_item(self, course_or_parent_loc, category, user_id=None, **kwargs):
"""
@@ -311,46 +251,30 @@ class MixedModuleStore(ModuleStoreWriteBase):
it installs the new item as a child of the parent (if the parent_loc is a specific
xblock reference).
- :param course_or_parent_loc: Can be a course_id (org/course/run), CourseLocator,
- Location, or BlockUsageLocator but must be what the persistence modulestore expects
+ :param course_or_parent_loc: Can be a CourseKey or UsageKey
+ :param category (str): The block_type of the item we are creating
"""
# find the store for the course
- course_id = self._infer_course_id_try(course_or_parent_loc)
- if course_id is None:
- raise ItemNotFoundError(u"Cannot find modulestore for %s" % course_or_parent_loc)
-
+ course_id = getattr(course_or_parent_loc, 'course_key', course_or_parent_loc)
store = self._get_modulestore_for_courseid(course_id)
location = kwargs.pop('location', None)
# invoke its create_item
if isinstance(store, MongoModuleStore):
block_id = kwargs.pop('block_id', getattr(location, 'name', uuid4().hex))
- # convert parent loc if it's legit
- if isinstance(course_or_parent_loc, basestring):
- parent_loc = None
- if location is None:
- loc_dict = Location.parse_course_id(course_id)
- loc_dict['name'] = block_id
- location = Location(category=category, **loc_dict)
- else:
- parent_loc = course_or_parent_loc
- # must have a legitimate location, compute if appropriate
- if location is None:
- location = parent_loc.replace(category=category, name=block_id)
+ parent_loc = course_or_parent_loc if isinstance(course_or_parent_loc, UsageKey) else None
+ # must have a legitimate location, compute if appropriate
+ if location is None:
+ location = course_id.make_usage_key(category, block_id)
# do the actual creation
xblock = store.create_and_save_xmodule(location, **kwargs)
# don't forget to attach to parent
if parent_loc is not None and not 'detached' in xblock._class_tags:
parent = store.get_item(parent_loc)
- parent.children.append(location.url())
+ parent.children.append(location)
store.update_item(parent)
elif isinstance(store, SplitMongoModuleStore):
- if isinstance(course_or_parent_loc, basestring): # course_id
- course_or_parent_loc = loc_mapper().translate_location_to_course_locator(
- # hardcode draft version until we figure out how we're handling branches from app
- course_or_parent_loc, None, published=False
- )
- elif not isinstance(course_or_parent_loc, CourseLocator):
+ if not isinstance(course_or_parent_loc, (CourseLocator, BlockUsageLocator)):
raise ValueError(u"Cannot create a child of {} in split. Wrong repr.".format(course_or_parent_loc))
# split handles all the fields in one dict not separated by scope
@@ -370,9 +294,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
Update the xblock persisted to be the same as the given for all types of fields
(content, children, and metadata) attribute the change to the given user.
"""
- course_id = self._infer_course_id_try(xblock.scope_ids.usage_id)
- if course_id is None:
- raise ItemNotFoundError(u"Cannot find modulestore for %s" % xblock.scope_ids.usage_id)
+ course_id = xblock.scope_ids.usage_id.course_key
store = self._get_modulestore_for_courseid(course_id)
return store.update_item(xblock, user_id)
@@ -380,9 +302,7 @@ class MixedModuleStore(ModuleStoreWriteBase):
"""
Delete the given item from persistence. kwargs allow modulestore specific parameters.
"""
- course_id = self._infer_course_id_try(location)
- if course_id is None:
- raise ItemNotFoundError(u"Cannot find modulestore for %s" % location)
+ course_id = location.course_key
store = self._get_modulestore_for_courseid(course_id)
return store.delete_item(location, user_id=user_id, **kwargs)
@@ -396,21 +316,6 @@ class MixedModuleStore(ModuleStoreWriteBase):
elif hasattr(mstore, 'db'):
mstore.db.connection.close()
- def ensure_loc_maps_exist(self, store_name):
- """
- Ensure location maps exist for every course in the modulestore whose
- name is the given name (mostly used for 'xml'). It creates maps for any
- missing ones.
-
- NOTE: will only work if the given store is Location based. If it's not,
- it raises NotImplementedError
- """
- store = self.modulestores[store_name]
- if store.reference_type != Location:
- raise ValueError(u"Cannot create maps from %s" % store.reference_type)
- for course in store.get_courses():
- loc_mapper().translate_location(course.location.course_id, course.location)
-
def get_courses_for_wiki(self, wiki_slug):
"""
Return the list of courses which use this wiki_slug
diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py
index 83d855bc0c7..3aaa2b65fb6 100644
--- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py
+++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py
@@ -8,7 +8,7 @@ structure:
'_id': <location.as_dict>,
'metadata': <dict containing all Scope.settings fields>
'definition': <dict containing all Scope.content fields>
- 'definition.children': <list of all child location.url()s>
+ 'definition.children': <list of all child location.to_deprecated_string()s>
}
"""
@@ -16,26 +16,27 @@ import pymongo
import sys
import logging
import copy
+import re
from bson.son import SON
from fs.osfs import OSFS
-from itertools import repeat
from path import path
from importlib import import_module
from xmodule.errortracker import null_error_tracker, exc_info_to_str
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.error_module import ErrorDescriptor
+from xmodule.html_module import AboutDescriptor
from xblock.runtime import KvsFieldData
from xblock.exceptions import InvalidScopeError
-from xblock.fields import Scope, ScopeIds
+from xblock.fields import Scope, ScopeIds, Reference, ReferenceList, ReferenceValueDict
from xmodule.modulestore import ModuleStoreWriteBase, Location, MONGO_MODULESTORE_TYPE
-from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.inheritance import own_metadata, InheritanceMixin, inherit_metadata, InheritanceKeyValueStore
-from xmodule.modulestore.xml import LocationReader
from xmodule.tabs import StaticTab, CourseTabList
from xblock.core import XBlock
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
log = logging.getLogger(__name__)
@@ -52,6 +53,7 @@ class InvalidWriteError(Exception):
Raised to indicate that writing to a particular key
in the KeyValueStore is disabled
"""
+ pass
class MongoKeyValueStore(InheritanceKeyValueStore):
@@ -120,7 +122,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
TODO (cdodge) when the 'split module store' work has been completed we can remove all
references to metadata_inheritance_tree
"""
- def __init__(self, modulestore, module_data, default_class, cached_metadata, **kwargs):
+ def __init__(self, modulestore, course_key, module_data, default_class, cached_metadata, **kwargs):
"""
modulestore: the module store that can be used to retrieve additional modules
@@ -138,7 +140,6 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
MakoDescriptorSystem
"""
super(CachingDescriptorSystem, self).__init__(
- id_reader=LocationReader(),
field_data=None,
load_item=self.load_item,
**kwargs
@@ -149,14 +150,14 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
self.default_class = default_class
# cdodge: other Systems have a course_id attribute defined. To keep things consistent, let's
# define an attribute here as well, even though it's None
- self.course_id = None
+ self.course_id = course_key
self.cached_metadata = cached_metadata
def load_item(self, location):
"""
Return an XModule instance for the specified location
"""
- location = Location(location)
+ assert isinstance(location, Location)
json_data = self.module_data.get(location)
if json_data is None:
module = self.modulestore.get_item(location)
@@ -170,6 +171,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
category = json_data['location']['category']
class_ = self.load_block_type(category)
+
definition = json_data.get('definition', {})
metadata = json_data.get('metadata', {})
for old_name, new_name in getattr(class_, 'metadata_translations', {}).items():
@@ -177,9 +179,19 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
metadata[new_name] = metadata[old_name]
del metadata[old_name]
+ children = [
+ location.course_key.make_usage_key_from_deprecated_string(childloc)
+ for childloc in definition.get('children', [])
+ ]
+ data = definition.get('data', {})
+ if isinstance(data, basestring):
+ data = {'data': data}
+ mixed_class = self.mixologist.mix(class_)
+ data = self._convert_reference_fields_to_keys(mixed_class, location.course_key, data)
+ metadata = self._convert_reference_fields_to_keys(mixed_class, location.course_key, metadata)
kvs = MongoKeyValueStore(
- definition.get('data', {}),
- definition.get('children', []),
+ data,
+ children,
metadata,
)
@@ -193,7 +205,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
# Convert the serialized fields values in self.cached_metadata
# to python values
- metadata_to_inherit = self.cached_metadata.get(non_draft_loc.url(), {})
+ metadata_to_inherit = self.cached_metadata.get(non_draft_loc.to_deprecated_string(), {})
inherit_metadata(module, metadata_to_inherit)
# decache any computed pending field settings
module.save()
@@ -203,31 +215,58 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
return ErrorDescriptor.from_json(
json_data,
self,
- json_data['location'],
+ location,
error_msg=exc_info_to_str(sys.exc_info())
)
-
-def namedtuple_to_son(namedtuple, prefix=''):
+ def _convert_reference_fields_to_keys(self, class_, course_key, jsonfields):
+ """
+ Find all fields of type reference and convert the payload into UsageKeys
+ :param class_: the XBlock class
+ :param course_key: a CourseKey object for the given course
+ :param jsonfields: a dict of the jsonified version of the fields
+ """
+ for field_name, value in jsonfields.iteritems():
+ if value:
+ field = class_.fields.get(field_name)
+ if field is None:
+ continue
+ elif isinstance(field, Reference):
+ jsonfields[field_name] = course_key.make_usage_key_from_deprecated_string(value)
+ elif isinstance(field, ReferenceList):
+ jsonfields[field_name] = [
+ course_key.make_usage_key_from_deprecated_string(ele) for ele in value
+ ]
+ elif isinstance(field, ReferenceValueDict):
+ for key, subvalue in value.iteritems():
+ assert isinstance(subvalue, basestring)
+ value[key] = course_key.make_usage_key_from_deprecated_string(subvalue)
+ return jsonfields
+
+
+def location_to_son(location, prefix='', tag='i4x'):
"""
- Converts a namedtuple into a SON object with the same key order
+ Converts a location into a SON object with the same key order
"""
- son = SON()
- # pylint: disable=protected-access
- for idx, field_name in enumerate(namedtuple._fields):
- son[prefix + field_name] = namedtuple[idx]
+ son = SON({prefix + 'tag': tag})
+ for field_name in location.KEY_FIELDS:
+ # Filter the run, because the existing data doesn't have it stored
+ if field_name != 'run':
+ son[prefix + field_name] = getattr(location, field_name)
return son
-def location_to_query(location, wildcard=True):
+# The only thing using this w/ wildcards is contentstore.mongo for asset retrieval
+def location_to_query(location, wildcard=True, tag='i4x'):
"""
- Takes a Location and returns a SON object that will query for that location.
+ Takes a Location and returns a SON object that will query for that location by subfields
+ rather than subdoc. Note: location_to_son is faster in mongo as it does subdoc equivalence.
Fields in location that are None are ignored in the query
If `wildcard` is True, then a None in a location is treated as a wildcard
query. Otherwise, it is searched for literally
"""
- query = namedtuple_to_son(Location(location), prefix='_id.')
+ query = location_to_son(location, prefix='_id.', tag=tag)
if wildcard:
for key, value in query.items():
@@ -239,11 +278,6 @@ def location_to_query(location, wildcard=True):
return query
-def metadata_cache_key(location):
- """Turn a `Location` into a useful cache key."""
- return u"{0.org}/{0.course}".format(location)
-
-
class MongoModuleStore(ModuleStoreWriteBase):
"""
A Mongodb backed ModuleStore
@@ -275,6 +309,8 @@ class MongoModuleStore(ModuleStoreWriteBase):
host=host,
port=port,
tz_aware=tz_aware,
+ # deserialize dicts as SONs
+ document_class=SON,
**kwargs
),
db
@@ -289,14 +325,6 @@ class MongoModuleStore(ModuleStoreWriteBase):
# Force mongo to report errors, at the expense of performance
self.collection.write_concern = {'w': 1}
- # Force mongo to maintain an index over _id.* that is in the same order
- # that is used when querying by a location
- # pylint: disable=no-member, protected_access
- self.collection.ensure_index(
- zip(('_id.' + field for field in Location._fields), repeat(1)),
- )
- # pylint: enable=no-member, protected_access
-
if default_class is not None:
module_path, _, class_name = default_class.rpartition('.')
class_ = getattr(import_module(module_path), class_name)
@@ -308,20 +336,24 @@ class MongoModuleStore(ModuleStoreWriteBase):
self.render_template = render_template
self.i18n_service = i18n_service
- self.ignore_write_events_on_courses = []
+ self.ignore_write_events_on_courses = set()
- def compute_metadata_inheritance_tree(self, location):
+ def _compute_metadata_inheritance_tree(self, course_id):
'''
TODO (cdodge) This method can be deleted when the 'split module store' work has been completed
'''
# get all collections in the course, this query should not return any leaf nodes
# note this is a bit ugly as when we add new categories of containers, we have to add it here
- block_types_with_children = set(name for name, class_ in XBlock.load_classes() if getattr(class_, 'has_children', False))
- query = {'_id.org': location.org,
- '_id.course': location.course,
- '_id.category': {'$in': list(block_types_with_children)}
- }
+ block_types_with_children = set(
+ name for name, class_ in XBlock.load_classes() if getattr(class_, 'has_children', False)
+ )
+ query = SON([
+ ('_id.tag', 'i4x'),
+ ('_id.org', course_id.org),
+ ('_id.course', course_id.course),
+ ('_id.category', {'$in': list(block_types_with_children)})
+ ])
# we just want the Location, children, and inheritable metadata
record_filter = {'_id': 1, 'definition.children': 1}
@@ -333,24 +365,26 @@ class MongoModuleStore(ModuleStoreWriteBase):
# call out to the DB
resultset = self.collection.find(query, record_filter)
+ # it's ok to keep these as urls b/c the overall cache is indexed by course_key and this
+ # is a dictionary relative to that course
results_by_url = {}
root = None
# now go through the results and order them by the location url
for result in resultset:
- location = Location(result['_id'])
- # We need to collate between draft and non-draft
- # i.e. draft verticals will have draft children but will have non-draft parents currently
- location = location.replace(revision=None)
- location_url = location.url()
+ # manually pick it apart b/c the db has tag and we want revision = None regardless
+ location = Location._from_deprecated_son(result['_id'], course_id.run).replace(revision=None)
+
+ location_url = location.to_deprecated_string()
if location_url in results_by_url:
+ # found either draft or live to complement the other revision
existing_children = results_by_url[location_url].get('definition', {}).get('children', [])
additional_children = result.get('definition', {}).get('children', [])
total_children = existing_children + additional_children
results_by_url[location_url].setdefault('definition', {})['children'] = total_children
- results_by_url[location.url()] = result
+ results_by_url[location_url] = result
if location.category == 'course':
- root = location.url()
+ root = location_url
# now traverse the tree and compute down the inherited metadata
metadata_to_inherit = {}
@@ -379,31 +413,30 @@ class MongoModuleStore(ModuleStoreWriteBase):
return metadata_to_inherit
- def get_cached_metadata_inheritance_tree(self, location, force_refresh=False):
+ def _get_cached_metadata_inheritance_tree(self, course_id, force_refresh=False):
'''
TODO (cdodge) This method can be deleted when the 'split module store' work has been completed
'''
- key = metadata_cache_key(location)
tree = {}
if not force_refresh:
# see if we are first in the request cache (if present)
- if self.request_cache is not None and key in self.request_cache.data.get('metadata_inheritance', {}):
- return self.request_cache.data['metadata_inheritance'][key]
+ if self.request_cache is not None and course_id in self.request_cache.data.get('metadata_inheritance', {}):
+ return self.request_cache.data['metadata_inheritance'][course_id]
# then look in any caching subsystem (e.g. memcached)
if self.metadata_inheritance_cache_subsystem is not None:
- tree = self.metadata_inheritance_cache_subsystem.get(key, {})
+ tree = self.metadata_inheritance_cache_subsystem.get(course_id, {})
else:
logging.warning('Running MongoModuleStore without a metadata_inheritance_cache_subsystem. This is OK in localdev and testing environment. Not OK in production.')
if not tree:
# if not in subsystem, or we are on force refresh, then we have to compute
- tree = self.compute_metadata_inheritance_tree(location)
+ tree = self._compute_metadata_inheritance_tree(course_id)
# now write out computed tree to caching subsystem (e.g. memcached), if available
if self.metadata_inheritance_cache_subsystem is not None:
- self.metadata_inheritance_cache_subsystem.set(key, tree)
+ self.metadata_inheritance_cache_subsystem.set(course_id, tree)
# now populate a request_cache, if available. NOTE, we are outside of the
# scope of the above if: statement so that after a memcache hit, it'll get
@@ -413,18 +446,22 @@ class MongoModuleStore(ModuleStoreWriteBase):
# defined
if 'metadata_inheritance' not in self.request_cache.data:
self.request_cache.data['metadata_inheritance'] = {}
- self.request_cache.data['metadata_inheritance'][key] = tree
+ self.request_cache.data['metadata_inheritance'][course_id] = tree
return tree
- def refresh_cached_metadata_inheritance_tree(self, location):
+ def refresh_cached_metadata_inheritance_tree(self, course_id, runtime=None):
"""
Refresh the cached metadata inheritance tree for the org/course combination
for location
+
+ If given a runtime, it replaces the cached_metadata in that runtime. NOTE: failure to provide
+ a runtime may mean that some objects report old values for inherited data.
"""
- pseudo_course_id = '/'.join([location.org, location.course])
- if pseudo_course_id not in self.ignore_write_events_on_courses:
- self.get_cached_metadata_inheritance_tree(location, force_refresh=True)
+ if course_id not in self.ignore_write_events_on_courses:
+ cached_metadata = self._get_cached_metadata_inheritance_tree(course_id, force_refresh=True)
+ if runtime:
+ runtime.cached_metadata = cached_metadata
def _clean_item_data(self, item):
"""
@@ -433,17 +470,19 @@ class MongoModuleStore(ModuleStoreWriteBase):
item['location'] = item['_id']
del item['_id']
- def _query_children_for_cache_children(self, items):
+ def _query_children_for_cache_children(self, course_key, items):
"""
Generate a pymongo in query for finding the items and return the payloads
"""
# first get non-draft in a round-trip
query = {
- '_id': {'$in': [namedtuple_to_son(Location(item)) for item in items]}
+ '_id': {'$in': [
+ location_to_son(course_key.make_usage_key_from_deprecated_string(item)) for item in items
+ ]}
}
return list(self.collection.find(query))
- def _cache_children(self, items, depth=0):
+ def _cache_children(self, course_key, items, depth=0):
"""
Returns a dictionary mapping Location -> item data, populated with json data
for all descendents of items up to the specified depth.
@@ -459,7 +498,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
for item in to_process:
self._clean_item_data(item)
children.extend(item.get('definition', {}).get('children', []))
- data[Location(item['location'])] = item
+ data[Location._from_deprecated_son(item['location'], course_key.run)] = item
if depth == 0:
break
@@ -469,7 +508,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
# for or-query syntax
to_process = []
if children:
- to_process = self._query_children_for_cache_children(children)
+ to_process = self._query_children_for_cache_children(course_key, children)
# If depth is None, then we just recurse until we hit all the descendents
if depth is not None:
@@ -477,11 +516,11 @@ class MongoModuleStore(ModuleStoreWriteBase):
return data
- def _load_item(self, item, data_cache, apply_cached_metadata=True):
+ def _load_item(self, course_key, item, data_cache, apply_cached_metadata=True):
"""
Load an XModuleDescriptor from item, using the children stored in data_cache
"""
- location = Location(item['location'])
+ location = Location._from_deprecated_son(item['location'], course_key.run)
data_dir = getattr(item, 'data_dir', location.course)
root = self.fs_root / data_dir
@@ -491,16 +530,15 @@ class MongoModuleStore(ModuleStoreWriteBase):
cached_metadata = {}
if apply_cached_metadata:
- cached_metadata = self.get_cached_metadata_inheritance_tree(location)
+ cached_metadata = self._get_cached_metadata_inheritance_tree(course_key)
services = {}
if self.i18n_service:
services["i18n"] = self.i18n_service
- # TODO (cdodge): When the 'split module store' work has been completed, we should remove
- # the 'metadata_inheritance_tree' parameter
system = CachingDescriptorSystem(
modulestore=self,
+ course_key=course_key,
module_data=data_cache,
default_class=self.default_class,
resources_fs=resource_fs,
@@ -513,70 +551,95 @@ class MongoModuleStore(ModuleStoreWriteBase):
)
return system.load_item(location)
- def _load_items(self, items, depth=0):
+ def _load_items(self, course_key, items, depth=0):
"""
Load a list of xmodules from the data in items, with children cached up
to specified depth
"""
- data_cache = self._cache_children(items, depth)
+ data_cache = self._cache_children(course_key, items, depth)
# if we are loading a course object, if we're not prefetching children (depth != 0) then don't
# bother with the metadata inheritance
- return [self._load_item(item, data_cache,
- apply_cached_metadata=(item['location']['category'] != 'course' or depth != 0)) for item in items]
+ return [
+ self._load_item(
+ course_key, item, data_cache,
+ apply_cached_metadata=(item['location']['category'] != 'course' or depth != 0)
+ )
+ for item in items
+ ]
def get_courses(self):
'''
Returns a list of course descriptors.
'''
- course_filter = Location(category="course")
- return [
- course
- for course
- in self.get_items(course_filter)
- if not (
- course.location.org == 'edx' and
- course.location.course == 'templates'
- )
- ]
+ return sum(
+ [
+ self._load_items(
+ SlashSeparatedCourseKey(course['_id']['org'], course['_id']['course'], course['_id']['name']),
+ [course]
+ )
+ for course
+ # I tried to add '$and': [{'_id.org': {'$ne': 'edx'}}, {'_id.course': {'$ne': 'templates'}}]
+ # but it didn't do the right thing (it filtered all edx and all templates out)
+ in self.collection.find({'_id.category': 'course'})
+ if not ( # TODO kill this
+ course['_id']['org'] == 'edx' and
+ course['_id']['course'] == 'templates'
+ )
+ ],
+ []
+ )
def _find_one(self, location):
'''Look for a given location in the collection. If revision is not
specified, returns the latest. If the item is not present, raise
ItemNotFoundError.
'''
+ assert isinstance(location, Location)
item = self.collection.find_one(
- location_to_query(location, wildcard=False),
+ {'_id': location_to_son(location)},
sort=[('revision', pymongo.ASCENDING)],
)
if item is None:
raise ItemNotFoundError(location)
return item
- def get_course(self, course_id):
+ def get_course(self, course_key, depth=None):
"""
Get the course with the given courseid (org/course/run)
"""
- id_components = Location.parse_course_id(course_id)
- id_components['tag'] = 'i4x'
- id_components['category'] = 'course'
+ assert(isinstance(course_key, SlashSeparatedCourseKey))
+ location = course_key.make_usage_key('course', course_key.run)
try:
- return self.get_item(Location(id_components))
+ return self.get_item(location, depth=depth)
except ItemNotFoundError:
return None
- def has_item(self, course_id, location):
+ def has_course(self, course_key, ignore_case=False):
+ """
+ Is the given course in this modulestore
+
+ If ignore_case is True, do a case insensitive search,
+ otherwise, do a case sensitive search
+ """
+ assert(isinstance(course_key, SlashSeparatedCourseKey))
+ course_query = self._course_key_to_son(course_key)
+ if ignore_case:
+ for key in course_query.iterkeys():
+ course_query[key] = re.compile(r"(?i)^{}$".format(course_query[key]))
+ return self.collection.find_one(course_query, fields={'_id': True}) is not None
+
+ def has_item(self, usage_key):
"""
Returns True if location exists in this ModuleStore.
"""
- location = Location.ensure_fully_specified(location)
try:
- self._find_one(location)
+ self._find_one(usage_key)
return True
except ItemNotFoundError:
return False
- def get_item(self, location, depth=0):
+ def get_item(self, usage_key, depth=0):
"""
Returns an XModuleDescriptor instance for the item at location.
@@ -585,52 +648,135 @@ class MongoModuleStore(ModuleStoreWriteBase):
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
- location: a Location object
+ usage_key: a :class:`.UsageKey` instance
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
in the request. The depth is counted in the number of
calls to get_children() to cache. None indicates to cache all descendents.
"""
- location = Location.ensure_fully_specified(location)
- item = self._find_one(location)
- module = self._load_items([item], depth)[0]
+ item = self._find_one(usage_key)
+ module = self._load_items(usage_key.course_key, [item], depth)[0]
return module
- def get_instance(self, course_id, location, depth=0):
+ @staticmethod
+ def _course_key_to_son(course_id, tag='i4x'):
"""
- TODO (vshnayder): implement policy tracking in mongo.
- For now, just delegate to get_item and ignore policy.
-
- depth (int): An argument that some module stores may use to prefetch
- descendents of the queried modules for more efficient results later
- in the request. The depth is counted in the number of
- calls to get_children() to cache. None indicates to cache all descendents.
+ Generate the partial key to look up items relative to a given course
"""
- return self.get_item(location, depth=depth)
+ return SON([
+ ('_id.tag', tag),
+ ('_id.org', course_id.org),
+ ('_id.course', course_id.course),
+ ])
- def get_items(self, location, course_id=None, depth=0, qualifiers=None):
+ def get_items(self, course_id, settings=None, content=None, revision=None, **kwargs):
+ """
+ Returns:
+ list of XModuleDescriptor instances for the matching items within the course with
+ the given course_id
+
+ NOTE: don't use this to look for courses
+ as the course_id is required. Use get_courses which is a lot faster anyway.
+
+ If you don't provide a value for revision, this limits the result to only ones in the
+ published course. Call this method on draft mongo store if you want to include drafts.
+
+ Args:
+ course_id (CourseKey): the course identifier
+ settings (dict): fields to look for which have settings scope. Follows same syntax
+ and rules as kwargs below
+ content (dict): fields to look for which have content scope. Follows same syntax and
+ rules as kwargs below.
+ revision (str): the revision of the items you're looking for.
+ kwargs (key=value): what to look for within the course.
+ Common qualifiers are ``category`` or any field name. if the target field is a list,
+ then it searches for the given value in the list not list equivalence.
+ Substring matching pass a regex object.
+ For this modulestore, ``name`` is a commonly provided key (Location based stores)
+ This modulestore does not allow searching dates by comparison or edited_by, previous_version,
+ update_version info.
+ """
+ query = self._course_key_to_son(course_id)
+ query['_id.revision'] = revision
+ for field in ['category', 'name']:
+ if field in kwargs:
+ query['_id.' + field] = kwargs.pop(field)
+
+ for key, value in (settings or {}).iteritems():
+ query['metadata.' + key] = value
+ for key, value in (content or {}).iteritems():
+ query['definition.data.' + key] = value
+ if 'children' in kwargs:
+ query['definition.children'] = kwargs.pop('children')
+
+ query.update(kwargs)
items = self.collection.find(
- location_to_query(location),
- sort=[('revision', pymongo.ASCENDING)],
+ query,
+ sort=[('_id.revision', pymongo.ASCENDING)],
)
- modules = self._load_items(list(items), depth)
+ modules = self._load_items(course_id, list(items))
return modules
- def create_course(self, course_id, definition_data=None, metadata=None, runtime=None):
+ def create_course(self, org, offering, user_id=None, fields=None, **kwargs):
"""
- Create a course with the given course_id.
+ Creates and returns the course.
+
+ Args:
+ org (str): the organization that owns the course
+ offering (str): the name of the course offering
+ user_id: id of the user creating the course
+ fields (dict): Fields to set on the course at initialization
+ kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
+
+ Returns: a CourseDescriptor
+
+ Raises:
+ InvalidLocationError: If a course with the same org and offering already exists
"""
- if isinstance(course_id, Location):
- location = course_id
- if location.category != 'course':
- raise ValueError(u"Course roots must be of category 'course': {}".format(unicode(location)))
- else:
- course_dict = Location.parse_course_id(course_id)
- course_dict['category'] = 'course'
- course_dict['tag'] = 'i4x'
- location = Location(course_dict)
- return self.create_and_save_xmodule(location, definition_data, metadata, runtime)
+
+ course, _, run = offering.partition('/')
+ course_id = SlashSeparatedCourseKey(org, course, run)
+
+ # Check if a course with this org/course has been defined before (case-insensitive)
+ course_search_location = SON([
+ ('_id.tag', 'i4x'),
+ ('_id.org', re.compile(u'^{}$'.format(course_id.org), re.IGNORECASE)),
+ ('_id.course', re.compile(u'^{}$'.format(course_id.course), re.IGNORECASE)),
+ ('_id.category', 'course'),
+ ])
+ courses = self.collection.find(course_search_location, fields=('_id'))
+ if courses.count() > 0:
+ raise InvalidLocationError(
+ "There are already courses with the given org and course id: {}".format([
+ course['_id'] for course in courses
+ ]))
+
+ location = course_id.make_usage_key('course', course_id.run)
+ course = self.create_and_save_xmodule(location, fields=fields, **kwargs)
+
+ # clone a default 'about' overview module as well
+ about_location = location.replace(
+ category='about',
+ name='overview'
+ )
+ overview_template = AboutDescriptor.get_template('overview.yaml')
+ self.create_and_save_xmodule(
+ about_location,
+ system=course.system,
+ definition_data=overview_template.get('data')
+ )
+
+ return course
+
+ def delete_course(self, course_key, user_id=None):
+ """
+ The impl removes all of the db records for the course.
+ :param course_key:
+ :param user_id:
+ """
+ course_query = self._course_key_to_son(course_key)
+ self.collection.remove(course_query, multi=True)
def create_xmodule(self, location, definition_data=None, metadata=None, system=None, fields={}):
"""
@@ -641,8 +787,6 @@ class MongoModuleStore(ModuleStoreWriteBase):
:param metadata: can be empty, the initial metadata for the kvs
:param system: if you already have an xblock from the course, the xblock.runtime value
"""
- if not isinstance(location, Location):
- location = Location(location)
# differs from split mongo in that I believe most of this logic should be above the persistence
# layer but added it here to enable quick conversion. I'll need to reconcile these.
if metadata is None:
@@ -659,6 +803,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
system = CachingDescriptorSystem(
modulestore=self,
module_data={},
+ course_key=location.course_key,
default_class=self.default_class,
resources_fs=None,
error_tracker=self.error_tracker,
@@ -678,8 +823,9 @@ class MongoModuleStore(ModuleStoreWriteBase):
ScopeIds(None, location.category, location, location),
dbmodel,
)
- for key, value in fields.iteritems():
- setattr(xmodule, key, value)
+ if fields is not None:
+ for key, value in fields.iteritems():
+ setattr(xmodule, key, value)
# decache any pending field settings from init
xmodule.save()
return xmodule
@@ -700,7 +846,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
# differs from split mongo in that I believe most of this logic should be above the persistence
# layer but added it here to enable quick conversion. I'll need to reconcile these.
new_object = self.create_xmodule(location, definition_data, metadata, system, fields)
- location = new_object.location
+ location = new_object.scope_ids.usage_id
self.update_item(new_object, allow_not_found=True)
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
@@ -712,47 +858,20 @@ class MongoModuleStore(ModuleStoreWriteBase):
course.tabs.append(
StaticTab(
name=new_object.display_name,
- url_slug=new_object.location.name,
+ url_slug=new_object.scope_ids.usage_id.name,
)
)
self.update_item(course)
return new_object
- def fire_updated_modulestore_signal(self, course_id, location):
- """
- Send a signal using `self.modulestore_update_signal`, if that has been set
- """
- if self.modulestore_update_signal is not None:
- self.modulestore_update_signal.send(self, modulestore=self, course_id=course_id,
- location=location)
-
def _get_course_for_item(self, location, depth=0):
'''
- VS[compat]
- cdodge: for a given Xmodule, return the course that it belongs to
- NOTE: This makes a lot of assumptions about the format of the course location
+ for a given Xmodule, return the course that it belongs to
Also we have to assert that this module maps to only one course item - it'll throw an
assert if not
- This is only used to support static_tabs as we need to be course module aware
'''
-
- # @hack! We need to find the course location however, we don't
- # know the 'name' parameter in this context, so we have
- # to assume there's only one item in this query even though we are not specifying a name
- course_search_location = Location('i4x', location.org, location.course, 'course', None)
- courses = self.get_items(course_search_location, depth=depth)
-
- # make sure we found exactly one match on this above course search
- found_cnt = len(courses)
- if found_cnt == 0:
- raise Exception('Could not find course at {0}'.format(course_search_location))
-
- if found_cnt > 1:
- raise Exception('Found more than one course at {0}. There should only be one!!! '
- 'Dump = {1}'.format(course_search_location, courses))
-
- return courses[0]
+ return self.get_course(location.course_key, depth)
def _update_single_item(self, location, update):
"""
@@ -763,7 +882,7 @@ class MongoModuleStore(ModuleStoreWriteBase):
# See http://www.mongodb.org/display/DOCS/Updating for
# atomic update syntax
result = self.collection.update(
- {'_id': namedtuple_to_son(Location(location))},
+ {'_id': location_to_son(location)},
{'$set': update},
multi=False,
upsert=True,
@@ -774,50 +893,70 @@ class MongoModuleStore(ModuleStoreWriteBase):
if result['n'] == 0:
raise ItemNotFoundError(location)
- def update_item(self, xblock, user=None, allow_not_found=False):
+ def update_item(self, xblock, user_id=None, allow_not_found=False, force=False):
"""
Update the persisted version of xblock to reflect its current values.
- location: Something that can be passed to Location
- data: A nested dictionary of problem data
+ xblock: which xblock to persist
+ user_id: who made the change (ignored for now by this modulestore)
+ allow_not_found: whether to create a new object if one didn't already exist or give an error
+ force: force is meaningless for this modulestore
"""
try:
- definition_data = xblock.get_explicitly_set_fields_by_scope()
+ definition_data = self._convert_reference_fields_to_strings(xblock, xblock.get_explicitly_set_fields_by_scope())
payload = {
'definition.data': definition_data,
- 'metadata': own_metadata(xblock),
+ 'metadata': self._convert_reference_fields_to_strings(xblock, own_metadata(xblock)),
}
if xblock.has_children:
- # convert all to urls
- xblock.children = [child.url() if isinstance(child, Location) else child
- for child in xblock.children]
- payload.update({'definition.children': xblock.children})
- self._update_single_item(xblock.location, payload)
+ children = self._convert_reference_fields_to_strings(xblock, {'children': xblock.children})
+ payload.update({'definition.children': children['children']})
+ self._update_single_item(xblock.scope_ids.usage_id, payload)
# for static tabs, their containing course also records their display name
- if xblock.category == 'static_tab':
- course = self._get_course_for_item(xblock.location)
+ if xblock.scope_ids.block_type == 'static_tab':
+ course = self._get_course_for_item(xblock.scope_ids.usage_id)
# find the course's reference to this tab and update the name.
- static_tab = CourseTabList.get_tab_by_slug(course.tabs, xblock.location.name)
+ static_tab = CourseTabList.get_tab_by_slug(course.tabs, xblock.scope_ids.usage_id.name)
# only update if changed
if static_tab and static_tab['name'] != xblock.display_name:
static_tab['name'] = xblock.display_name
- self.update_item(course, user)
+ self.update_item(course, user_id)
# recompute (and update) the metadata inheritance tree which is cached
- # was conditional on children or metadata having changed before dhm made one update to rule them all
- self.refresh_cached_metadata_inheritance_tree(xblock.location)
+ self.refresh_cached_metadata_inheritance_tree(xblock.scope_ids.usage_id.course_key, xblock.runtime)
# fire signal that we've written to DB
- self.fire_updated_modulestore_signal(get_course_id_no_run(xblock.location), xblock.location)
except ItemNotFoundError:
if not allow_not_found:
raise
+ def _convert_reference_fields_to_strings(self, xblock, jsonfields):
+ """
+ Find all fields of type reference and convert the payload from UsageKeys to deprecated strings
+ :param xblock: the XBlock class
+ :param jsonfields: a dict of the jsonified version of the fields
+ """
+ assert isinstance(jsonfields, dict)
+ for field_name, value in jsonfields.iteritems():
+ if value:
+ if isinstance(xblock.fields[field_name], Reference):
+ jsonfields[field_name] = value.to_deprecated_string()
+ elif isinstance(xblock.fields[field_name], ReferenceList):
+ jsonfields[field_name] = [
+ ele.to_deprecated_string() for ele in value
+ ]
+ elif isinstance(xblock.fields[field_name], ReferenceValueDict):
+ for key, subvalue in value.iteritems():
+ assert isinstance(subvalue, Location)
+ value[key] = subvalue.to_deprecated_string()
+ return jsonfields
+
# pylint: disable=unused-argument
def delete_item(self, location, **kwargs):
"""
- Delete an item from this modulestore
+ Delete an item from this modulestore.
- location: Something that can be passed to Location
+ Args:
+ location (UsageKey)
"""
# pylint: enable=unused-argument
# VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
@@ -825,26 +964,28 @@ class MongoModuleStore(ModuleStoreWriteBase):
# we should remove this once we can break this reference from the course to static tabs
if location.category == 'static_tab':
item = self.get_item(location)
- course = self._get_course_for_item(item.location)
+ course = self._get_course_for_item(item.scope_ids.usage_id)
existing_tabs = course.tabs or []
course.tabs = [tab for tab in existing_tabs if tab.get('url_slug') != location.name]
self.update_item(course, '**replace_user**')
# Must include this to avoid the django debug toolbar (which defines the deprecated "safe=False")
# from overriding our default value set in the init method.
- self.collection.remove({'_id': Location(location).dict()}, safe=self.collection.safe)
+ self.collection.remove({'_id': location_to_son(location)}, safe=self.collection.safe)
# recompute (and update) the metadata inheritance tree which is cached
- self.refresh_cached_metadata_inheritance_tree(Location(location))
- self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))
+ self.refresh_cached_metadata_inheritance_tree(location.course_key)
- def get_parent_locations(self, location, course_id):
+ def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
course. Needed for path_to_location().
'''
- location = Location.ensure_fully_specified(location)
- items = self.collection.find({'definition.children': location.url()},
- {'_id': True})
- return [Location(i['_id']) for i in items]
+ query = self._course_key_to_son(location.course_key)
+ query['definition.children'] = location.to_deprecated_string()
+ items = self.collection.find(query, {'_id': True})
+ return [
+ location.course_key.make_usage_key(i['_id']['category'], i['_id']['name'])
+ for i in items
+ ]
def get_modulestore_type(self, course_id):
"""
@@ -856,21 +997,22 @@ class MongoModuleStore(ModuleStoreWriteBase):
"""
return MONGO_MODULESTORE_TYPE
- def get_orphans(self, course_location, _branch):
+ def get_orphans(self, course_key):
"""
- Return an array all of the locations for orphans in the course.
+ Return an array all of the locations (deprecated string format) for orphans in the course.
"""
detached_categories = [name for name, __ in XBlock.load_tagged_classes("detached")]
- all_items = self.collection.find({
- '_id.org': course_location.org,
- '_id.course': course_location.course,
- '_id.category': {'$nin': detached_categories}
- })
+ query = self._course_key_to_son(course_key)
+ query['_id.category'] = {'$nin': detached_categories}
+ all_items = self.collection.find(query)
all_reachable = set()
item_locs = set()
for item in all_items:
if item['_id']['category'] != 'course':
- item_locs.add(Location(item['_id']).replace(revision=None).url())
+ # It would be nice to change this method to return UsageKeys instead of the deprecated string.
+ item_locs.add(
+ Location._from_deprecated_son(item['_id'], course_key.run).replace(revision=None).to_deprecated_string()
+ )
all_reachable = all_reachable.union(item.get('definition', {}).get('children', []))
item_locs -= all_reachable
return list(item_locs)
@@ -882,7 +1024,8 @@ class MongoModuleStore(ModuleStoreWriteBase):
:return: list of course locations
"""
courses = self.collection.find({'definition.data.wiki_slug': wiki_slug})
- return [Location(course['_id']) for course in courses]
+ # the course's run == its name. It's the only xblock for which that's necessarily true.
+ return [Location._from_deprecated_son(course['_id'], course['_id']['name']) for course in courses]
def _create_new_field_data(self, _category, _location, definition_data, metadata):
"""
diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py
index c8ce76e1ec3..f468e30e81d 100644
--- a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py
+++ b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py
@@ -7,13 +7,13 @@ and otherwise returns i4x://org/course/cat/name).
"""
from datetime import datetime
+import pymongo
+from pytz import UTC
from xmodule.exceptions import InvalidVersionError
-from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError
-from xmodule.modulestore.mongo.base import location_to_query, namedtuple_to_son, get_course_id_no_run, MongoModuleStore
-import pymongo
-from pytz import UTC
+from xmodule.modulestore.mongo.base import MongoModuleStore
+from xmodule.modulestore.locations import Location
DRAFT = 'draft'
# Things w/ these categories should never be marked as version='draft'
@@ -24,14 +24,14 @@ def as_draft(location):
"""
Returns the Location that is the draft for `location`
"""
- return Location(location).replace(revision=DRAFT)
+ return location.replace(revision=DRAFT)
def as_published(location):
"""
Returns the Location that is the published version for `location`
"""
- return Location(location).replace(revision=None)
+ return location.replace(revision=None)
def wrap_draft(item):
@@ -56,19 +56,19 @@ class DraftModuleStore(MongoModuleStore):
their children) to published modules.
"""
- def get_item(self, location, depth=0):
+ def get_item(self, usage_key, depth=0):
"""
- Returns an XModuleDescriptor instance for the item at location.
- If location.revision is None, returns the item with the most
+ Returns an XModuleDescriptor instance for the item at usage_key.
+ If usage_key.revision is None, returns the item with the most
recent revision
- If any segment of the location is None except revision, raises
+ If any segment of the usage_key is None except revision, raises
xmodule.modulestore.exceptions.InsufficientSpecificationError
- If no object is found at that location, raises
+ If no object is found at that usage_key, raises
xmodule.modulestore.exceptions.ItemNotFoundError
- location: Something that can be passed to Location
+ usage_key: A :class:`.UsageKey` instance
depth (int): An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
@@ -77,20 +77,9 @@ class DraftModuleStore(MongoModuleStore):
"""
try:
- return wrap_draft(super(DraftModuleStore, self).get_item(as_draft(location), depth=depth))
+ return wrap_draft(super(DraftModuleStore, self).get_item(as_draft(usage_key), depth=depth))
except ItemNotFoundError:
- return wrap_draft(super(DraftModuleStore, self).get_item(location, depth=depth))
-
- def get_instance(self, course_id, location, depth=0):
- """
- Get an instance of this location, with policy for course_id applied.
- TODO (vshnayder): this may want to live outside the modulestore eventually
- """
-
- try:
- return wrap_draft(super(DraftModuleStore, self).get_instance(course_id, as_draft(location), depth=depth))
- except ItemNotFoundError:
- return wrap_draft(super(DraftModuleStore, self).get_instance(course_id, location, depth=depth))
+ return wrap_draft(super(DraftModuleStore, self).get_item(usage_key, depth=depth))
def create_xmodule(self, location, definition_data=None, metadata=None, system=None, fields={}):
"""
@@ -101,37 +90,42 @@ class DraftModuleStore(MongoModuleStore):
:param metadata: can be empty, the initial metadata for the kvs
:param system: if you already have an xmodule from the course, the xmodule.system value
"""
- draft_loc = as_draft(location)
- if draft_loc.category in DIRECT_ONLY_CATEGORIES:
+ if location.category in DIRECT_ONLY_CATEGORIES:
raise InvalidVersionError(location)
+ draft_loc = as_draft(location)
return super(DraftModuleStore, self).create_xmodule(draft_loc, definition_data, metadata, system, fields)
- def get_items(self, location, course_id=None, depth=0, qualifiers=None):
+ def get_items(self, course_key, settings=None, content=None, **kwargs):
"""
- Returns a list of XModuleDescriptor instances for the items
- that match location. Any element of location that is None is treated
- as a wildcard that matches any value
+ Returns:
+ list of XModuleDescriptor instances for the matching items within the course with
+ the given course_key
- location: Something that can be passed to Location
+ NOTE: don't use this to look for courses
+ as the course_key is required. Use get_courses.
- depth: An argument that some module stores may use to prefetch
- descendents of the queried modules for more efficient results later
- in the request. The depth is counted in the number of calls to
- get_children() to cache. None indicates to cache all descendents
+ Args:
+ course_key (CourseKey): the course identifier
+ settings: not used
+ content: not used
+ kwargs (key=value): what to look for within the course.
+ Common qualifiers are ``category`` or any field name. if the target field is a list,
+ then it searches for the given value in the list not list equivalence.
+ Substring matching pass a regex object.
+ ``name`` is another commonly provided key (Location based stores)
"""
- draft_loc = as_draft(location)
-
- draft_items = super(DraftModuleStore, self).get_items(draft_loc, course_id=course_id, depth=depth)
- items = super(DraftModuleStore, self).get_items(location, course_id=course_id, depth=depth)
-
- draft_locs_found = set(item.location.replace(revision=None) for item in draft_items)
+ draft_items = [
+ wrap_draft(item) for item in
+ super(DraftModuleStore, self).get_items(course_key, revision='draft', **kwargs)
+ ]
+ draft_items_locations = {item.location for item in draft_items}
non_draft_items = [
- item
- for item in items
- if (item.location.revision != DRAFT
- and item.location.replace(revision=None) not in draft_locs_found)
+ item for item in
+ super(DraftModuleStore, self).get_items(course_key, revision=None, **kwargs)
+ # filter out items that are not already in draft
+ if item.location not in draft_items_locations
]
- return [wrap_draft(item) for item in draft_items + non_draft_items]
+ return draft_items + non_draft_items
def convert_to_draft(self, source_location):
"""
@@ -139,40 +133,38 @@ class DraftModuleStore(MongoModuleStore):
:param source: the location of the source (its revision must be None)
"""
- original = self.collection.find_one(location_to_query(source_location))
+ original = self.collection.find_one({'_id': source_location.to_deprecated_son()})
draft_location = as_draft(source_location)
if draft_location.category in DIRECT_ONLY_CATEGORIES:
raise InvalidVersionError(source_location)
if not original:
raise ItemNotFoundError(source_location)
- original['_id'] = namedtuple_to_son(draft_location)
+ original['_id'] = draft_location.to_deprecated_son()
try:
self.collection.insert(original)
except pymongo.errors.DuplicateKeyError:
raise DuplicateItemError(original['_id'])
- self.refresh_cached_metadata_inheritance_tree(draft_location)
- self.fire_updated_modulestore_signal(get_course_id_no_run(draft_location), draft_location)
+ self.refresh_cached_metadata_inheritance_tree(draft_location.course_key)
- return self._load_items([original])[0]
+ return self._load_items(source_location.course_key, [original])[0]
- def update_item(self, xblock, user=None, allow_not_found=False):
+ def update_item(self, xblock, user_id=None, allow_not_found=False, force=False):
"""
- Save the current values to persisted version of the xblock
-
- location: Something that can be passed to Location
- data: A nested dictionary of problem data
+ See superclass doc.
+ In addition to the superclass's behavior, this method converts the unit to draft if it's not
+ already draft.
"""
draft_loc = as_draft(xblock.location)
try:
- if not self.has_item(None, draft_loc):
+ if not self.has_item(draft_loc):
self.convert_to_draft(xblock.location)
except ItemNotFoundError:
if not allow_not_found:
raise
xblock.location = draft_loc
- super(DraftModuleStore, self).update_item(xblock, user, allow_not_found)
+ super(DraftModuleStore, self).update_item(xblock, user_id, allow_not_found)
# don't allow locations to truly represent themselves as draft outside of this file
xblock.location = as_published(xblock.location)
@@ -188,14 +180,6 @@ class DraftModuleStore(MongoModuleStore):
return
- def get_parent_locations(self, location, course_id):
- '''Find all locations that are the parents of this location. Needed
- for path_to_location().
-
- returns an iterable of things that can be passed to Location.
- '''
- return super(DraftModuleStore, self).get_parent_locations(location, course_id)
-
def publish(self, location, published_by_id):
"""
Save a current draft to the underlying modulestore
@@ -216,8 +200,8 @@ class DraftModuleStore(MongoModuleStore):
# 2) child moved
for child in original_published.children:
if child not in draft.children:
- rents = [Location(mom) for mom in self.get_parent_locations(child, None)]
- if (len(rents) == 1 and rents[0] == Location(location)): # the 1 is this original_published
+ rents = self.get_parent_locations(child)
+ if (len(rents) == 1 and rents[0] == location): # the 1 is this original_published
self.delete_item(child, True)
super(DraftModuleStore, self).update_item(draft, '**replace_user**')
self.delete_item(location)
@@ -229,17 +213,19 @@ class DraftModuleStore(MongoModuleStore):
self.convert_to_draft(location)
super(DraftModuleStore, self).delete_item(location)
- def _query_children_for_cache_children(self, items):
+ def _query_children_for_cache_children(self, course_key, items):
# first get non-draft in a round-trip
- to_process_non_drafts = super(DraftModuleStore, self)._query_children_for_cache_children(items)
+ to_process_non_drafts = super(DraftModuleStore, self)._query_children_for_cache_children(course_key, items)
to_process_dict = {}
for non_draft in to_process_non_drafts:
- to_process_dict[Location(non_draft["_id"])] = non_draft
+ to_process_dict[Location._from_deprecated_son(non_draft["_id"], course_key.run)] = non_draft
# now query all draft content in another round-trip
query = {
- '_id': {'$in': [namedtuple_to_son(as_draft(Location(item))) for item in items]}
+ '_id': {'$in': [
+ as_draft(course_key.make_usage_key_from_deprecated_string(item)).to_deprecated_son() for item in items
+ ]}
}
to_process_drafts = list(self.collection.find(query))
@@ -247,7 +233,7 @@ class DraftModuleStore(MongoModuleStore):
# with the draft. This is because the semantics of the DraftStore is to
# always return the draft - if available
for draft in to_process_drafts:
- draft_loc = Location(draft["_id"])
+ draft_loc = Location._from_deprecated_son(draft["_id"], course_key.run)
draft_as_non_draft_loc = draft_loc.replace(revision=None)
# does non-draft exist in the collection
diff --git a/common/lib/xmodule/xmodule/modulestore/parsers.py b/common/lib/xmodule/xmodule/modulestore/parsers.py
index 9d046effdb6..1bb2d123177 100644
--- a/common/lib/xmodule/xmodule/modulestore/parsers.py
+++ b/common/lib/xmodule/xmodule/modulestore/parsers.py
@@ -7,15 +7,15 @@ BLOCK_PREFIX = r"block/"
# Prefix for the version portion of a locator URL, when it is preceded by a course ID
VERSION_PREFIX = r"version/"
-ALLOWED_ID_CHARS = r'[\w\-~.:]'
-
+ALLOWED_ID_CHARS = r'[\w\-~.:+]'
+ALLOWED_ID_RE = re.compile(r'^{}+$'.format(ALLOWED_ID_CHARS), re.UNICODE)
+# NOTE: if we need to support period in place of +, make it aggressive (take the first period in the string)
URL_RE_SOURCE = r"""
- (?P<tag>edx://)?
- ((?P<package_id>{ALLOWED_ID_CHARS}+)/?)?
+ ((?P<org>{ALLOWED_ID_CHARS}+)\+(?P<offering>{ALLOWED_ID_CHARS}+)/?)?
({BRANCH_PREFIX}(?P<branch>{ALLOWED_ID_CHARS}+)/?)?
({VERSION_PREFIX}(?P<version_guid>[A-F0-9]+)/?)?
- ({BLOCK_PREFIX}(?P<block>{ALLOWED_ID_CHARS}+))?
+ ({BLOCK_PREFIX}(?P<block_id>{ALLOWED_ID_CHARS}+))?
""".format(
ALLOWED_ID_CHARS=ALLOWED_ID_CHARS, BRANCH_PREFIX=BRANCH_PREFIX,
VERSION_PREFIX=VERSION_PREFIX, BLOCK_PREFIX=BLOCK_PREFIX
@@ -24,40 +24,33 @@ URL_RE_SOURCE = r"""
URL_RE = re.compile('^' + URL_RE_SOURCE + '$', re.IGNORECASE | re.VERBOSE | re.UNICODE)
-def parse_url(string, tag_optional=False):
+def parse_url(string):
"""
- A url usually begins with 'edx://' (case-insensitive match),
- followed by either a version_guid or a package_id. If tag_optional, then
+ followed by either a version_guid or a org + offering pair. If tag_optional, then
the url does not have to start with the tag and edx will be assumed.
Examples:
- 'edx://version/0123FFFF'
- 'edx://mit.eecs.6002x'
- 'edx://mit.eecs.6002x/branch/published'
- 'edx://mit.eecs.6002x/branch/published/block/HW3'
- 'edx://mit.eecs.6002x/branch/published/version/000eee12345/block/HW3'
+ 'edx:version/0123FFFF'
+ 'edx:mit.eecs.6002x'
+ 'edx:mit.eecs.6002x/branch/published'
+ 'edx:mit.eecs.6002x/branch/published/block/HW3'
+ 'edx:mit.eecs.6002x/branch/published/version/000eee12345/block/HW3'
This returns None if string cannot be parsed.
- If it can be parsed as a version_guid with no preceding package_id, returns a dict
+ If it can be parsed as a version_guid with no preceding org + offering, returns a dict
with key 'version_guid' and the value,
- If it can be parsed as a package_id, returns a dict
+ If it can be parsed as a org + offering, returns a dict
with key 'id' and optional keys 'branch' and 'version_guid'.
-
"""
match = URL_RE.match(string)
if not match:
return None
matched_dict = match.groupdict()
- if matched_dict['tag'] is None and not tag_optional:
- return None
return matched_dict
-BLOCK_RE = re.compile(r'^' + ALLOWED_ID_CHARS + r'+$', re.IGNORECASE | re.UNICODE)
-
-
def parse_block_ref(string):
r"""
A block_ref is a string of url safe characters (see ALLOWED_ID_CHARS)
@@ -65,46 +58,6 @@ def parse_block_ref(string):
If string is a block_ref, returns a dict with key 'block_ref' and the value,
otherwise returns None.
"""
- if len(string) > 0 and BLOCK_RE.match(string):
- return {'block': string}
+ if ALLOWED_ID_RE.match(string):
+ return {'block_id': string}
return None
-
-
-def parse_package_id(string):
- r"""
-
- A package_id has a main id component.
- There may also be an optional branch (/branch/published or /branch/draft).
- There may also be an optional version (/version/519665f6223ebd6980884f2b).
- There may also be an optional block (/block/HW3 or /block/Quiz2).
-
- Examples of valid package_ids:
-
- 'mit.eecs.6002x'
- 'mit.eecs.6002x/branch/published'
- 'mit.eecs.6002x/block/HW3'
- 'mit.eecs.6002x/branch/published/block/HW3'
- 'mit.eecs.6002x/branch/published/version/519665f6223ebd6980884f2b/block/HW3'
-
-
- Syntax:
-
- package_id = main_id [/branch/ branch] [/version/ version ] [/block/ block]
-
- main_id = name [. name]*
-
- branch = name
-
- block = name
-
- name = ALLOWED_ID_CHARS
-
- If string is a package_id, returns a dict with keys 'id', 'branch', and 'block'.
- Revision is optional: if missing returned_dict['branch'] is None.
- Block is optional: if missing returned_dict['block'] is None.
- Else returns None.
- """
- match = URL_RE.match(string)
- if not match:
- return None
- return match.groupdict()
diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py
index 804cdb01941..613b50e417a 100644
--- a/common/lib/xmodule/xmodule/modulestore/search.py
+++ b/common/lib/xmodule/xmodule/modulestore/search.py
@@ -1,34 +1,28 @@
from itertools import repeat
-
-from xmodule.course_module import CourseDescriptor
-
from .exceptions import (ItemNotFoundError, NoPathToItem)
-from . import Location
-def path_to_location(modulestore, course_id, location):
+def path_to_location(modulestore, usage_key):
'''
Try to find a course_id/chapter/section[/position] path to location in
modulestore. The courseware insists that the first level in the course is
chapter, but any kind of module can be a "section".
- location: something that can be passed to Location
- course_id: Search for paths in this course.
-
- raise ItemNotFoundError if the location doesn't exist.
-
- raise NoPathToItem if the location exists, but isn't accessible via
- a chapter/section path in the course(s) being searched.
+ Args:
+ modulestore: which store holds the relevant objects
+ usage_key: :class:`UsageKey` the id of the location to which to generate the path
- Return a tuple (course_id, chapter, section, position) suitable for the
- courseware index view.
+ Raises
+ ItemNotFoundError if the location doesn't exist.
+ NoPathToItem if the location exists, but isn't accessible via
+ a chapter/section path in the course(s) being searched.
- A location may be accessible via many paths. This method may
- return any valid path.
+ Returns:
+ a tuple (course_id, chapter, section, position) suitable for the
+ courseware index view.
- If the section is a sequential or vertical, position will be the position
- of this location in that sequence. Otherwise, position will
- be None. TODO (vshnayder): Not true yet.
+ If the section is a sequential or vertical, position will be the children index
+ of this location under that sequence.
'''
def flatten(xs):
@@ -55,41 +49,38 @@ def path_to_location(modulestore, course_id, location):
# tuples (location, path-so-far). To avoid lots of
# copying, the path-so-far is stored as a lisp-style
# list--nested hd::tl tuples, and flattened at the end.
- queue = [(location, ())]
+ queue = [(usage_key, ())]
while len(queue) > 0:
- (loc, path) = queue.pop() # Takes from the end
- loc = Location(loc)
+ (next_usage, path) = queue.pop() # Takes from the end
# get_parent_locations should raise ItemNotFoundError if location
# isn't found so we don't have to do it explicitly. Call this
# first to make sure the location is there (even if it's a course, and
# we would otherwise immediately exit).
- parents = modulestore.get_parent_locations(loc, course_id)
+ parents = modulestore.get_parent_locations(next_usage)
- # print 'Processing loc={0}, path={1}'.format(loc, path)
- if loc.category == "course":
- # confirm that this is the right course
- if course_id == CourseDescriptor.location_to_id(loc):
- # Found it!
- path = (loc, path)
- return flatten(path)
+ # print 'Processing loc={0}, path={1}'.format(next_usage, path)
+ if next_usage.definition_key.block_type == "course":
+ # Found it!
+ path = (next_usage, path)
+ return flatten(path)
# otherwise, add parent locations at the end
- newpath = (loc, path)
+ newpath = (next_usage, path)
queue.extend(zip(parents, repeat(newpath)))
# If we're here, there is no path
return None
- if not modulestore.has_item(course_id, location):
- raise ItemNotFoundError
+ if not modulestore.has_item(usage_key):
+ raise ItemNotFoundError(usage_key)
path = find_path_to_course()
if path is None:
- raise NoPathToItem(location)
+ raise NoPathToItem(usage_key)
n = len(path)
- course_id = CourseDescriptor.location_to_id(path[0])
+ course_id = path[0].course_key
# pull out the location names
chapter = path[1].name if n > 1 else None
section = path[2].name if n > 2 else None
@@ -105,9 +96,9 @@ def path_to_location(modulestore, course_id, location):
if n > 3:
position_list = []
for path_index in range(2, n - 1):
- category = path[path_index].category
+ category = path[path_index].definition_key.block_type
if category == 'sequential' or category == 'videosequence':
- section_desc = modulestore.get_instance(course_id, path[path_index])
+ section_desc = modulestore.get_item(path[path_index])
child_locs = [c.location for c in section_desc.get_children()]
# positions are 1-indexed, and should be strings to be consistent with
# url parsing.
diff --git a/common/lib/xmodule/xmodule/modulestore/split_migrator.py b/common/lib/xmodule/xmodule/modulestore/split_migrator.py
index 80eba980292..fcbeb0f9d69 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_migrator.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_migrator.py
@@ -6,9 +6,7 @@ Exists at the top level of modulestore b/c it needs to know about and access eac
In general, it's strategy is to treat the other modulestores as read-only and to never directly
manipulate storage but use existing api's.
'''
-from xmodule.modulestore import Location
-from xmodule.modulestore.locator import CourseLocator
-from xmodule.modulestore.mongo import draft
+from xblock.fields import Reference, ReferenceList, ReferenceValueDict
class SplitMigrator(object):
@@ -23,65 +21,64 @@ class SplitMigrator(object):
self.draft_modulestore = draft_modulestore
self.loc_mapper = loc_mapper
- def migrate_mongo_course(self, course_location, user, new_package_id=None):
+ def migrate_mongo_course(self, course_key, user, new_org=None, new_offering=None):
"""
Create a new course in split_mongo representing the published and draft versions of the course from the
- original mongo store. And return the new_package_id (which the caller can also get by calling
- self.loc_mapper.translate_location(old_course_location)
+ original mongo store. And return the new CourseLocator
If the new course already exists, this raises DuplicateItemError
:param course_location: a Location whose category is 'course' and points to the course
:param user: the user whose action is causing this migration
- :param new_package_id: (optional) the Locator.package_id for the new course. Defaults to
- whatever translate_location_to_locator returns
+ :param new_org: (optional) the Locator.org for the new course. Defaults to
+ whatever translate_location_to_locator returns
+ :param new_offering: (optional) the Locator.offering for the new course. Defaults to
+ whatever translate_location_to_locator returns
"""
- new_package_id = self.loc_mapper.create_map_entry(course_location, package_id=new_package_id)
- old_course_id = course_location.course_id
+ new_course_locator = self.loc_mapper.create_map_entry(course_key, new_org, new_offering)
# the only difference in data between the old and split_mongo xblocks are the locations;
# so, any field which holds a location must change to a Locator; otherwise, the persistence
# layer and kvs's know how to store it.
# locations are in location, children, conditionals, course.tab
- # create the course: set fields to explicitly_set for each scope, id_root = new_package_id, master_branch = 'production'
- original_course = self.direct_modulestore.get_item(course_location)
- new_course_root_locator = self.loc_mapper.translate_location(old_course_id, course_location)
+ # create the course: set fields to explicitly_set for each scope, id_root = new_course_locator, master_branch = 'production'
+ original_course = self.direct_modulestore.get_course(course_key)
+ new_course_root_locator = self.loc_mapper.translate_location(original_course.location)
new_course = self.split_modulestore.create_course(
- new_package_id, course_location.org,
- user.id,
- fields=self._get_json_fields_translate_children(original_course, old_course_id, True),
+ new_course_root_locator.org, new_course_root_locator.offering, user.id,
+ fields=self._get_json_fields_translate_references(original_course, course_key, True),
root_block_id=new_course_root_locator.block_id,
master_branch=new_course_root_locator.branch
)
- self._copy_published_modules_to_course(new_course, course_location, old_course_id, user)
- self._add_draft_modules_to_course(new_package_id, old_course_id, course_location, user)
+ self._copy_published_modules_to_course(new_course, original_course.location, course_key, user)
+ self._add_draft_modules_to_course(new_course.id, course_key, user)
- return new_package_id
+ return new_course_locator
- def _copy_published_modules_to_course(self, new_course, old_course_loc, old_course_id, user):
+ def _copy_published_modules_to_course(self, new_course, old_course_loc, course_key, user):
"""
Copy all of the modules from the 'direct' version of the course to the new split course.
"""
- course_version_locator = new_course.location.as_course_locator()
+ course_version_locator = new_course.id
# iterate over published course elements. Wildcarding rather than descending b/c some elements are orphaned (e.g.,
# course about pages, conditionals)
- for module in self.direct_modulestore.get_items(
- old_course_loc.replace(category=None, name=None, revision=None),
- old_course_id
- ):
+ for module in self.direct_modulestore.get_items(course_key):
# don't copy the course again. No drafts should get here but check
if module.location != old_course_loc and not getattr(module, 'is_draft', False):
# create split_xblock using split.create_item
# where block_id is computed by translate_location_to_locator
new_locator = self.loc_mapper.translate_location(
- old_course_id, module.location, True, add_entry_if_missing=True
+ module.location, True, add_entry_if_missing=True
)
+ # NOTE: the below auto populates the children when it migrates the parent; so,
+ # it doesn't need the parent as the first arg. That is, it translates and populates
+ # the 'children' field as it goes.
_new_module = self.split_modulestore.create_item(
course_version_locator, module.category, user.id,
block_id=new_locator.block_id,
- fields=self._get_json_fields_translate_children(module, old_course_id, True),
+ fields=self._get_json_fields_translate_references(module, course_key, True),
continue_version=True
)
# after done w/ published items, add version for 'draft' pointing to the published structure
@@ -94,25 +91,22 @@ class SplitMigrator(object):
# children which meant some pointers were to non-existent locations in 'direct'
self.split_modulestore.internal_clean_children(course_version_locator)
- def _add_draft_modules_to_course(self, new_package_id, old_course_id, old_course_loc, user):
+ def _add_draft_modules_to_course(self, published_course_key, course_key, user):
"""
update each draft. Create any which don't exist in published and attach to their parents.
"""
# each true update below will trigger a new version of the structure. We may want to just have one new version
# but that's for a later date.
- new_draft_course_loc = CourseLocator(package_id=new_package_id, branch='draft')
+ new_draft_course_loc = published_course_key.for_branch('draft')
# to prevent race conditions of grandchilden being added before their parents and thus having no parent to
# add to
awaiting_adoption = {}
- for module in self.draft_modulestore.get_items(
- old_course_loc.replace(category=None, name=None, revision=draft.DRAFT),
- old_course_id
- ):
+ for module in self.draft_modulestore.get_items(course_key):
if getattr(module, 'is_draft', False):
new_locator = self.loc_mapper.translate_location(
- old_course_id, module.location, False, add_entry_if_missing=True
+ module.location, False, add_entry_if_missing=True
)
- if self.split_modulestore.has_item(new_package_id, new_locator):
+ if self.split_modulestore.has_item(new_locator):
# was in 'direct' so draft is a new version
split_module = self.split_modulestore.get_item(new_locator)
# need to remove any no-longer-explicitly-set values and add/update any now set values.
@@ -131,25 +125,24 @@ class SplitMigrator(object):
_new_module = self.split_modulestore.create_item(
new_draft_course_loc, module.category, user.id,
block_id=new_locator.block_id,
- fields=self._get_json_fields_translate_children(module, old_course_id, True)
+ fields=self._get_json_fields_translate_references(module, course_key, True)
)
awaiting_adoption[module.location] = new_locator.block_id
for draft_location, new_block_id in awaiting_adoption.iteritems():
- for parent_loc in self.draft_modulestore.get_parent_locations(draft_location, old_course_id):
+ for parent_loc in self.draft_modulestore.get_parent_locations(draft_location):
old_parent = self.draft_modulestore.get_item(parent_loc)
new_parent = self.split_modulestore.get_item(
- self.loc_mapper.translate_location(old_course_id, old_parent.location, False)
+ self.loc_mapper.translate_location(old_parent.location, False)
)
# this only occurs if the parent was also awaiting adoption
if new_block_id in new_parent.children:
break
# find index for module: new_parent may be missing quite a few of old_parent's children
new_parent_cursor = 0
- draft_location = draft_location.url() # need as string
for old_child_loc in old_parent.children:
if old_child_loc == draft_location:
break
- sibling_loc = self.loc_mapper.translate_location(old_course_id, Location(old_child_loc), False)
+ sibling_loc = self.loc_mapper.translate_location(old_child_loc, False)
# sibling may move cursor
for idx in range(new_parent_cursor, len(new_parent.children)):
if new_parent.children[idx] == sibling_loc.block_id:
@@ -158,24 +151,32 @@ class SplitMigrator(object):
new_parent.children.insert(new_parent_cursor, new_block_id)
new_parent = self.split_modulestore.update_item(new_parent, user.id)
- def _get_json_fields_translate_children(self, xblock, old_course_id, published):
+ def _get_json_fields_translate_references(self, xblock, old_course_id, published):
"""
- Return the json repr for explicitly set fields but convert all children to their block_id's
+ Return the json repr for explicitly set fields but convert all references to their block_id's
"""
- fields = self.get_json_fields_explicitly_set(xblock)
- # this will too generously copy the children even for ones that don't exist in the published b/c the old mongo
- # had no way of not having parents point to draft only children :-(
- if 'children' in fields:
- fields['children'] = [
- self.loc_mapper.translate_location(
- old_course_id, Location(child), published, add_entry_if_missing=True
- ).block_id
- for child in fields['children']]
- return fields
-
- def get_json_fields_explicitly_set(self, xblock):
- """
- Get the json repr for fields set on this specific xblock
- :param xblock:
- """
- return {field.name: field.read_json(xblock) for field in xblock.fields.itervalues() if field.is_set_on(xblock)}
+ # FIXME change split to take field values as pythonic values not json values
+ result = {}
+ for field_name, field in xblock.fields.iteritems():
+ if field.is_set_on(xblock):
+ if isinstance(field, Reference):
+ result[field_name] = unicode(self.loc_mapper.translate_location(
+ getattr(xblock, field_name), published, add_entry_if_missing=True
+ ))
+ elif isinstance(field, ReferenceList):
+ result[field_name] = [
+ unicode(self.loc_mapper.translate_location(
+ ele, published, add_entry_if_missing=True
+ )) for ele in getattr(xblock, field_name)
+ ]
+ elif isinstance(field, ReferenceValueDict):
+ result[field_name] = {
+ key: unicode(self.loc_mapper.translate_location(
+ subvalue, published, add_entry_if_missing=True
+ ))
+ for key, subvalue in getattr(xblock, field_name).iteritems()
+ }
+ else:
+ result[field_name] = field.read_json(xblock)
+
+ return result
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
index 83d585b9641..bb50171d38b 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
@@ -1,10 +1,10 @@
import sys
import logging
from xmodule.mako_module import MakoDescriptorSystem
-from xmodule.modulestore.locator import BlockUsageLocator, LocalId
+from xmodule.modulestore.locator import BlockUsageLocator, LocalId, CourseLocator
from xmodule.error_module import ErrorDescriptor
from xmodule.errortracker import exc_info_to_str
-from xblock.runtime import KvsFieldData, IdReader
+from xblock.runtime import KvsFieldData
from ..exceptions import ItemNotFoundError
from .split_mongo_kvs import SplitMongoKVS
from xblock.fields import ScopeIds
@@ -13,23 +13,6 @@ from xmodule.modulestore.loc_mapper_store import LocMapperStore
log = logging.getLogger(__name__)
-class SplitMongoIdReader(IdReader):
- """
- An :class:`~xblock.runtime.IdReader` associated with a particular
- :class:`.CachingDescriptorSystem`.
- """
- def __init__(self, system):
- self.system = system
-
- def get_definition_id(self, usage_id):
- usage = self.system.load_item(usage_id)
- return usage.definition_locator
-
- def get_block_type(self, def_id):
- definition = self.system.modulestore.db_connection.get_definition(def_id)
- return definition['category']
-
-
class CachingDescriptorSystem(MakoDescriptorSystem):
"""
A system that has a cache of a course version's json that it will use to load modules
@@ -44,15 +27,14 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
modulestore: the module store that can be used to retrieve additional
modules
- course_entry: the originally fetched enveloped course_structure w/ branch and package_id info.
+ course_entry: the originally fetched enveloped course_structure w/ branch and course id info.
Callers to _load_item provide an override but that function ignores the provided structure and
- only looks at the branch and package_id
+ only looks at the branch and course id
module_data: a dict mapping Location -> json that was cached from the
underlying modulestore
"""
super(CachingDescriptorSystem, self).__init__(
- id_reader=SplitMongoIdReader(self),
field_data=None,
load_item=self._load_item,
**kwargs
@@ -72,11 +54,14 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
self.local_modules = {}
def _load_item(self, block_id, course_entry_override=None):
- if isinstance(block_id, BlockUsageLocator) and isinstance(block_id.block_id, LocalId):
- try:
- return self.local_modules[block_id]
- except KeyError:
- raise ItemNotFoundError
+ if isinstance(block_id, BlockUsageLocator):
+ if isinstance(block_id.block_id, LocalId):
+ try:
+ return self.local_modules[block_id]
+ except KeyError:
+ raise ItemNotFoundError
+ else:
+ block_id = block_id.block_id
json_data = self.module_data.get(block_id)
if json_data is None:
@@ -99,14 +84,15 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
# the thread is working with more than one named container pointing to the same specific structure is
# low; thus, the course_entry is most likely correct. If the thread is looking at > 1 named container
# pointing to the same structure, the access is likely to be chunky enough that the last known container
- # is the intended one when not given a course_entry_override; thus, the caching of the last branch/package_id.
+ # is the intended one when not given a course_entry_override; thus, the caching of the last branch/course id.
def xblock_from_json(self, class_, block_id, json_data, course_entry_override=None):
if course_entry_override is None:
course_entry_override = self.course_entry
else:
# most recent retrieval is most likely the right one for next caller (see comment above fn)
self.course_entry['branch'] = course_entry_override['branch']
- self.course_entry['package_id'] = course_entry_override['package_id']
+ self.course_entry['org'] = course_entry_override['org']
+ self.course_entry['offering'] = course_entry_override['offering']
# most likely a lazy loader or the id directly
definition = json_data.get('definition', {})
definition_id = self.modulestore.definition_locator(definition)
@@ -116,10 +102,13 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
block_id = LocalId()
block_locator = BlockUsageLocator(
- version_guid=course_entry_override['structure']['_id'],
+ CourseLocator(
+ version_guid=course_entry_override['structure']['_id'],
+ org=course_entry_override.get('org'),
+ offering=course_entry_override.get('offering'),
+ branch=course_entry_override.get('branch'),
+ ),
block_id=block_id,
- package_id=course_entry_override.get('package_id'),
- branch=course_entry_override.get('branch')
)
kvs = SplitMongoKVS(
@@ -141,7 +130,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
json_data,
self,
BlockUsageLocator(
- version_guid=course_entry_override['structure']['_id'],
+ CourseLocator(version_guid=course_entry_override['structure']['_id']),
block_id=block_id
),
error_msg=exc_info_to_str(sys.exc_info())
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
index 510c100048f..a3aad8c3d3c 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
@@ -1,7 +1,9 @@
"""
Segregation of pymongo functions from the data modeling mechanisms for split modulestore.
"""
+import re
import pymongo
+from bson import son
class MongoConnection(object):
"""
@@ -18,6 +20,7 @@ class MongoConnection(object):
host=host,
port=port,
tz_aware=tz_aware,
+ document_class=son.SON,
**kwargs
),
db
@@ -63,11 +66,17 @@ class MongoConnection(object):
"""
self.structures.update({'_id': structure['_id']}, structure)
- def get_course_index(self, key):
+ def get_course_index(self, key, ignore_case=False):
"""
Get the course_index from the persistence mechanism whose id is the given key
"""
- return self.course_index.find_one({'_id': key})
+ case_regex = r"(?i)^{}$" if ignore_case else r"{}"
+ return self.course_index.find_one(
+ son.SON([
+ (key_attr, re.compile(case_regex.format(getattr(key, key_attr))))
+ for key_attr in ('org', 'offering')
+ ])
+ )
def find_matching_course_indexes(self, query):
"""
@@ -86,13 +95,16 @@ class MongoConnection(object):
"""
Update the db record for course_index
"""
- self.course_index.update({'_id': course_index['_id']}, course_index)
+ self.course_index.update(
+ son.SON([('org', course_index['org']), ('offering', course_index['offering'])]),
+ course_index
+ )
- def delete_course_index(self, key):
+ def delete_course_index(self, course_index):
"""
- Delete the course_index from the persistence mechanism whose id is the given key
+ Delete the course_index from the persistence mechanism whose id is the given course_index
"""
- return self.course_index.remove({'_id': key})
+ return self.course_index.remove(son.SON([('org', course_index['org']), ('offering', course_index['offering'])]))
def get_definition(self, key):
"""
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
index 10e9185deeb..91f94a8bcc6 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
@@ -3,8 +3,9 @@ Provides full versioning CRUD and representation for collections of xblocks (e.g
Representation:
* course_index: a dictionary:
- ** '_id': package_id (e.g., myu.mydept.mycourse.myrun),
+ ** '_id': a unique id which cannot change,
** 'org': the org's id. Only used for searching not identity,
+ ** 'offering': the course's catalog number and run id or whatever user decides,
** 'edited_by': user_id of user who created the original entry,
** 'edited_on': the datetime of the original creation,
** 'versions': versions_dict: {branch_id: structure_id, ...}
@@ -47,7 +48,6 @@ Representation:
import threading
import datetime
import logging
-import re
from importlib import import_module
from path import path
import copy
@@ -122,7 +122,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
self.thread_cache = threading.local()
if default_class is not None:
- module_path, _, class_name = default_class.rpartition('.')
+ module_path, __, class_name = default_class.rpartition('.')
class_ = getattr(import_module(module_path), class_name)
self.default_class = class_
else:
@@ -235,7 +235,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
return the CourseDescriptor! It returns the actual db json from
structures.
- Semantics: if package_id and branch given, then it will get that branch. If
+ Semantics: if course id and branch given, then it will get that branch. If
also give a version_guid, it will see if the current head of that branch == that guid. If not
it raises VersionConflictError (the version now differs from what it was when you got your
reference)
@@ -247,9 +247,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if not course_locator.is_fully_specified():
raise InsufficientSpecificationError('Not fully specified: %s' % course_locator)
- if course_locator.package_id is not None and course_locator.branch is not None:
- # use the package_id
- index = self.db_connection.get_course_index(course_locator.package_id)
+ if course_locator.org and course_locator.offering and course_locator.branch:
+ # use the course id
+ index = self.db_connection.get_course_index(course_locator)
if index is None:
raise ItemNotFoundError(course_locator)
if course_locator.branch not in index['versions']:
@@ -266,11 +266,12 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
version_guid = course_locator.as_object_id(version_guid)
entry = self.db_connection.get_structure(version_guid)
- # b/c more than one course can use same structure, the 'package_id' and 'branch' are not intrinsic to structure
+ # b/c more than one course can use same structure, the 'org', 'offering', and 'branch' are not intrinsic to structure
# and the one assoc'd w/ it by another fetch may not be the one relevant to this fetch; so,
# add it in the envelope for the structure.
envelope = {
- 'package_id': course_locator.package_id,
+ 'org': course_locator.org,
+ 'offering': course_locator.offering,
'branch': course_locator.branch,
'structure': entry,
}
@@ -300,15 +301,17 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
for structure in matching:
version_guid = structure['versions'][branch]
version_guids.append(version_guid)
- id_version_map[version_guid] = structure['_id']
+ id_version_map[version_guid] = structure
course_entries = self.db_connection.find_matching_structures({'_id': {'$in': version_guids}})
# get the block for the course element (s/b the root)
result = []
for entry in course_entries:
+ course_info = id_version_map[entry['_id']]
envelope = {
- 'package_id': id_version_map[entry['_id']],
+ 'org': course_info['org'],
+ 'offering': course_info['offering'],
'branch': branch,
'structure': entry,
}
@@ -316,42 +319,44 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
result.extend(self._load_items(envelope, [root], 0, lazy=True))
return result
- def get_course(self, course_locator):
+ def get_course(self, course_id, depth=None):
'''
Gets the course descriptor for the course identified by the locator
which may or may not be a blockLocator.
raises InsufficientSpecificationError
'''
- course_entry = self._lookup_course(course_locator)
+ assert(isinstance(course_id, CourseLocator))
+ course_entry = self._lookup_course(course_id)
root = course_entry['structure']['root']
result = self._load_items(course_entry, [root], 0, lazy=True)
return result[0]
- def get_course_for_item(self, location):
+ def has_course(self, course_id, ignore_case=False):
'''
- Provided for backward compatibility. Is equivalent to calling get_course
- :param location:
+ Does this course exist in this modulestore.
'''
- return self.get_course(location)
+ assert(isinstance(course_id, CourseLocator))
+ course_entry = self.db_connection.get_course_index(course_id, ignore_case)
+ return course_entry is not None
- def has_item(self, package_id, block_location):
+ def has_item(self, usage_key):
"""
Returns True if location exists in its course. Returns false if
the course or the block w/in the course do not exist for the given version.
raises InsufficientSpecificationError if the locator does not id a block
"""
- if block_location.block_id is None:
- raise InsufficientSpecificationError(block_location)
+ if usage_key.block_id is None:
+ raise InsufficientSpecificationError(usage_key)
try:
- course_structure = self._lookup_course(block_location)['structure']
+ course_structure = self._lookup_course(usage_key)['structure']
except ItemNotFoundError:
# this error only occurs if the course does not exist
return False
- return self._get_block_from_structure(course_structure, block_location.block_id) is not None
+ return self._get_block_from_structure(course_structure, usage_key.block_id) is not None
- def get_item(self, location, depth=0):
+ def get_item(self, usage_key, depth=0):
"""
depth (int): An argument that some module stores may use to prefetch
descendants of the queried modules for more efficient results later
@@ -361,52 +366,77 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
raises InsufficientSpecificationError or ItemNotFoundError
"""
# intended for temporary support of some pointers being old-style
- if isinstance(location, Location):
+ if isinstance(usage_key, Location):
if self.loc_mapper is None:
raise InsufficientSpecificationError('No location mapper configured')
else:
- location = self.loc_mapper.translate_location(
- None, location, location.revision is None,
+ usage_key = self.loc_mapper.translate_location(
+ usage_key,
+ usage_key.revision is None,
add_entry_if_missing=False
)
- assert isinstance(location, BlockUsageLocator)
- if not location.is_initialized():
- raise InsufficientSpecificationError("Not yet initialized: %s" % location)
- course = self._lookup_course(location)
- items = self._load_items(course, [location.block_id], depth, lazy=True)
+ assert isinstance(usage_key, BlockUsageLocator)
+ course = self._lookup_course(usage_key)
+ items = self._load_items(course, [usage_key.block_id], depth, lazy=True)
if len(items) == 0:
- raise ItemNotFoundError(location)
+ raise ItemNotFoundError(usage_key)
return items[0]
- def get_items(self, locator, course_id=None, depth=0, qualifiers=None):
- """
- Get all of the modules in the given course matching the qualifiers. The
- qualifiers should only be fields in the structures collection (sorry).
- There will be a separate search method for searching through
- definitions.
-
- Common qualifiers are category, definition (provide definition id),
- display_name, anyfieldname, children (return
- block if its children includes the one given value). If you want
- substring matching use {$regex: /acme.*corp/i} type syntax.
-
- Although these
- look like mongo queries, it is all done in memory; so, you cannot
- try arbitrary queries.
+ def get_items(self, course_locator, settings=None, content=None, **kwargs):
+ """
+ Returns:
+ list of XModuleDescriptor instances for the matching items within the course with
+ the given course_id
+
+ NOTE: don't use this to look for courses
+ as the course_id is required. Use get_courses.
+
+ Args:
+ course_locator (CourseLocator): the course identifier
+ settings (dict): fields to look for which have settings scope. Follows same syntax
+ and rules as kwargs below
+ content (dict): fields to look for which have content scope. Follows same syntax and
+ rules as kwargs below.
+ kwargs (key=value): what to look for within the course.
+ Common qualifiers are ``category`` or any field name. if the target field is a list,
+ then it searches for the given value in the list not list equivalence.
+ Substring matching pass a regex object.
+ For split,
+ you can search by ``edited_by``, ``edited_on`` providing a function testing limits.
+ """
+ course = self._lookup_course(course_locator)
+ items = []
- :param locator: CourseLocator or BlockUsageLocator restricting search scope
- :param course_id: ignored. Only included for API compatibility.
- :param depth: ignored. Only included for API compatibility.
- :param qualifiers: a dict restricting which elements should match
+ def _block_matches_all(block_json):
+ """
+ Check that the block matches all the criteria
+ """
+ # do the checks which don't require loading any additional data
+ if (
+ self._block_matches(block_json, kwargs) and
+ self._block_matches(block_json.get('fields', {}), settings)
+ ):
+ if content:
+ definition_block = self.db_connection.get_definition(block_json['definition'])
+ return self._block_matches(definition_block.get('fields', {}), content)
+ else:
+ return True
- """
- # TODO extend to only search a subdag of the course?
- if qualifiers is None:
- qualifiers = {}
- course = self._lookup_course(locator)
- items = []
+ if settings is None:
+ settings = {}
+ if 'name' in kwargs:
+ # odd case where we don't search just confirm
+ block_id = kwargs.pop('name')
+ block = course['structure']['blocks'].get(block_id)
+ if _block_matches_all(block):
+ return self._load_items(course, [block_id], lazy=True)
+ else:
+ return []
+ # don't expect caller to know that children are in fields
+ if 'children' in kwargs:
+ settings['children'] = kwargs.pop('children')
for block_id, value in course['structure']['blocks'].iteritems():
- if self._block_matches(value, qualifiers):
+ if _block_matches_all(value):
items.append(block_id)
if len(items) > 0:
@@ -414,20 +444,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
else:
return []
- def get_instance(self, course_id, location, depth=0):
- """
- Get an instance of this location.
-
- For now, just delegate to get_item and ignore course policy.
-
- depth (int): An argument that some module stores may use to prefetch
- descendants of the queried modules for more efficient results later
- in the request. The depth is counted in the number of
- calls to get_children() to cache. None indicates to cache all descendants.
- """
- return self.get_item(location, depth=depth)
-
- def get_parent_locations(self, locator, course_id=None):
+ def get_parent_locations(self, locator):
'''
Return the locations (Locators w/ block_ids) for the parents of this location in this
course. Could use get_items(location, {'children': block_id}) but this is slightly faster.
@@ -438,20 +455,20 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
'''
course = self._lookup_course(locator)
items = self._get_parents_from_structure(locator.block_id, course['structure'])
- return [BlockUsageLocator(
- url=locator.as_course_locator(),
- block_id=LocMapperStore.decode_key_from_mongo(parent_id),
- )
- for parent_id in items]
+ return [
+ BlockUsageLocator.make_relative(
+ locator,
+ block_id=LocMapperStore.decode_key_from_mongo(parent_id),
+ )
+ for parent_id in items
+ ]
- def get_orphans(self, package_id, branch):
+ def get_orphans(self, course_key):
"""
Return a dict of all of the orphans in the course.
-
- :param package_id:
"""
detached_categories = [name for name, __ in XBlock.load_tagged_classes("detached")]
- course = self._lookup_course(CourseLocator(package_id=package_id, branch=branch))
+ course = self._lookup_course(course_key)
items = {LocMapperStore.decode_key_from_mongo(block_id) for block_id in course['structure']['blocks'].keys()}
items.remove(course['structure']['root'])
for block_id, block_data in course['structure']['blocks'].iteritems():
@@ -459,7 +476,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if block_data['category'] in detached_categories:
items.discard(LocMapperStore.decode_key_from_mongo(block_id))
return [
- BlockUsageLocator(package_id=package_id, branch=branch, block_id=block_id)
+ BlockUsageLocator(course_key=course_key, block_id=block_id)
for block_id in items
]
@@ -468,7 +485,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
The index records the initial creation of the indexed course and tracks the current version
heads. This function is primarily for test verification but may serve some
more general purpose.
- :param course_locator: must have a package_id set
+ :param course_locator: must have a org and offering set
:return {'org': string,
versions: {'draft': the head draft version id,
'published': the head published version id if any,
@@ -477,9 +494,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
'edited_on': when the course was originally created
}
"""
- if course_locator.package_id is None:
+ if not (course_locator.offering and course_locator.org):
return None
- index = self.db_connection.get_course_index(course_locator.package_id)
+ index = self.db_connection.get_course_index(course_locator)
return index
# TODO figure out a way to make this info accessible from the course descriptor
@@ -529,6 +546,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if course_locator.version_guid is None:
course = self._lookup_course(course_locator)
version_guid = course['structure']['_id']
+ course_locator = course_locator.for_version(version_guid)
else:
version_guid = course_locator.version_guid
@@ -547,7 +565,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
for course_structure in next_versions:
result.setdefault(course_structure['previous_version'], []).append(
CourseLocator(version_guid=struct['_id']))
- return VersionTree(CourseLocator(course_locator, version_guid=version_guid), result)
+ return VersionTree(course_locator, result)
def get_block_generations(self, block_locator):
@@ -562,8 +580,12 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
course_struct = self._lookup_course(block_locator.course_agnostic())['structure']
block_id = block_locator.block_id
update_version_field = 'blocks.{}.edit_info.update_version'.format(block_id)
- all_versions_with_block = self.db_connection.find_matching_structures({'original_version': course_struct['original_version'],
- update_version_field: {'$exists': True}})
+ all_versions_with_block = self.db_connection.find_matching_structures(
+ {
+ 'original_version': course_struct['original_version'],
+ update_version_field: {'$exists': True}
+ }
+ )
# find (all) root versions and build map {previous: {successors}..}
possible_roots = []
result = {}
@@ -590,9 +612,14 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
return None
# convert the results value sets to locators
for k, versions in result.iteritems():
- result[k] = [BlockUsageLocator(version_guid=version, block_id=block_id)
- for version in versions]
- return VersionTree(BlockUsageLocator(version_guid=possible_roots[0], block_id=block_id), result)
+ result[k] = [
+ BlockUsageLocator(CourseLocator(version_guid=version), block_id=block_id)
+ for version in versions
+ ]
+ return VersionTree(
+ BlockUsageLocator(CourseLocator(version_guid=possible_roots[0]), block_id=block_id),
+ result
+ )
def get_definition_successors(self, definition_locator, version_history_depth=1):
'''
@@ -646,7 +673,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# actual change b/c the descriptor and cache probably point to the same objects
old_definition = self.db_connection.get_definition(definition_locator.definition_id)
if old_definition is None:
- raise ItemNotFoundError(definition_locator.url())
+ raise ItemNotFoundError(definition_locator.to_deprecated_string())
if needs_saved():
# new id to create new version
@@ -693,11 +720,12 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
:param course_or_parent_locator: If BlockUsageLocator, then it's assumed to be the parent.
If it's a CourseLocator, then it's
- merely the containing course.
+ merely the containing course. If it has a version_guid and a course org + offering + branch, this
+ method ensures that the version is the head of the given course branch before making the change.
raises InsufficientSpecificationError if there is no course locator.
- raises VersionConflictError if package_id and version_guid given and the current version head != version_guid
- and force is not True.
+ raises VersionConflictError if the version_guid of the course_or_parent_locator is not the head
+ of the its course unless force is true.
:param force: fork the structure and don't update the course draftVersion if the above
:param continue_revision: for multistep transactions, continue revising the given version rather than creating
a new version. Setting force to True conflicts with setting this to True and will cause a VersionConflictError
@@ -722,11 +750,11 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
Rules for course locator:
- * If the course locator specifies a package_id and either it doesn't
+ * If the course locator specifies a org and offering and either it doesn't
specify version_guid or the one it specifies == the current head of the branch,
it progresses the course to point
to the new head and sets the active version to point to the new head
- * If the locator has a package_id but its version_guid != current head, it raises VersionConflictError.
+ * If the locator has a org and offering but its version_guid != current head, it raises VersionConflictError.
NOTE: using a version_guid will end up creating a new version of the course. Your new item won't be in
the course id'd by version_guid but instead in one w/ a new version_guid. Ensure in this case that you get
@@ -800,29 +828,36 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if not continue_version:
self._update_head(index_entry, course_or_parent_locator.branch, new_id)
item_loc = BlockUsageLocator(
- package_id=course_or_parent_locator.package_id,
- branch=course_or_parent_locator.branch,
+ course_or_parent_locator.version_agnostic(),
block_id=new_block_id,
)
else:
item_loc = BlockUsageLocator(
+ CourseLocator(version_guid=new_id),
block_id=new_block_id,
- version_guid=new_id,
)
# reconstruct the new_item from the cache
return self.get_item(item_loc)
def create_course(
- self, course_id, org, user_id, fields=None,
+ self, org, offering, user_id, fields=None,
master_branch='draft', versions_dict=None, root_category='course',
- root_block_id='course'
+ root_block_id='course', **kwargs
):
"""
Create a new entry in the active courses index which points to an existing or new structure. Returns
the course root of the resulting entry (the location has the course id)
- course_id: If it's already taken, this method will raise DuplicateCourseError
+ Arguments:
+
+ org (str): the organization that owns the course
+ offering (str): the name of the course offering
+ user_id: id of the user creating the course
+ fields (dict): Fields to set on the course at initialization
+ kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
+
+ offering: If it's already taken, this method will raise DuplicateCourseError
fields: if scope.settings fields provided, will set the fields of the root course object in the
new course. If both
@@ -848,10 +883,11 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
provide any fields overrides, see above). if not provided, will create a mostly empty course
structure with just a category course root xblock.
"""
- # check course_id's uniqueness
- index = self.db_connection.get_course_index(course_id)
+ # check offering's uniqueness
+ locator = CourseLocator(org=org, offering=offering, branch=master_branch)
+ index = self.db_connection.get_course_index(locator)
if index is not None:
- raise DuplicateCourseError(course_id, index)
+ raise DuplicateCourseError(locator, index)
partitioned_fields = self.partition_fields_by_scope(root_category, fields)
block_fields = partitioned_fields.setdefault(Scope.settings, {})
@@ -920,15 +956,16 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
versions_dict[master_branch] = new_id
index_entry = {
- '_id': course_id,
+ '_id': ObjectId(),
'org': org,
+ 'offering': offering,
'edited_by': user_id,
'edited_on': datetime.datetime.now(UTC),
'versions': versions_dict,
'schema_version': self.SCHEMA_VERSION,
}
self.db_connection.insert_course_index(index_entry)
- return self.get_course(CourseLocator(package_id=course_id, branch=master_branch))
+ return self.get_course(locator)
def update_item(self, descriptor, user_id, allow_not_found=False, force=False):
"""
@@ -937,7 +974,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
raises ItemNotFoundError if the location does not exist.
- Creates a new course version. If the descriptor's location has a package_id, it moves the course head
+ Creates a new course version. If the descriptor's location has a org and offering, it moves the course head
pointer. If the version_guid of the descriptor points to a non-head version and there's been an intervening
change to this item, it raises a VersionConflictError unless force is True. In the force case, it forks
the course but leaves the head pointer where it is (this change will not be in the course head).
@@ -983,10 +1020,16 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# update the index entry if appropriate
if index_entry is not None:
self._update_head(index_entry, descriptor.location.branch, new_id)
+ course_key = CourseLocator(
+ org=index_entry['org'], offering=index_entry['offering'],
+ branch=descriptor.location.branch,
+ version_guid=new_id
+ )
+ else:
+ course_key = CourseLocator(version_guid=new_id)
# fetch and return the new item--fetching is unnecessary but a good qc step
- new_locator = BlockUsageLocator(descriptor.location)
- new_locator.version_guid = new_id
+ new_locator = BlockUsageLocator(course_key, descriptor.location.block_id)
return self.get_item(new_locator)
else:
# nothing changed, just return the one sent in
@@ -1060,10 +1103,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# fetch and return the new item--fetching is unnecessary but a good qc step
return self.get_item(
BlockUsageLocator(
- package_id=xblock.location.package_id,
+ xblock.location.course_key.for_version(new_id),
block_id=xblock.location.block_id,
- branch=xblock.location.branch,
- version_guid=new_id
)
)
else:
@@ -1088,7 +1129,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
if block_id is None:
block_id = self._generate_block_id(structure_blocks, xblock.category)
encoded_block_id = LocMapperStore.encode_key_for_mongo(block_id)
- xblock.scope_ids.usage_id.block_id = block_id
+ new_usage_id = xblock.scope_ids.usage_id.replace(block_id=block_id)
+ xblock.scope_ids = xblock.scope_ids._replace(usage_id=new_usage_id) # pylint: disable=protected-access
else:
is_new = False
encoded_block_id = LocMapperStore.encode_key_for_mongo(xblock.location.block_id)
@@ -1179,7 +1221,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
"""
# get the destination's index, and source and destination structures.
source_structure = self._lookup_course(source_course)['structure']
- index_entry = self.db_connection.get_course_index(destination_course.package_id)
+ index_entry = self.db_connection.get_course_index(destination_course)
if index_entry is None:
# brand new course
raise ItemNotFoundError(destination_course)
@@ -1244,13 +1286,13 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
raises ItemNotFoundError if the location does not exist.
raises ValueError if usage_locator points to the structure root
- Creates a new course version. If the descriptor's location has a package_id, it moves the course head
+ Creates a new course version. If the descriptor's location has a org and offering, it moves the course head
pointer. If the version_guid of the descriptor points to a non-head version and there's been an intervening
change to this item, it raises a VersionConflictError unless force is True. In the force case, it forks
the course but leaves the head pointer where it is (this change will not be in the course head).
"""
- assert isinstance(usage_locator, BlockUsageLocator) and usage_locator.is_initialized()
- original_structure = self._lookup_course(usage_locator)['structure']
+ assert isinstance(usage_locator, BlockUsageLocator)
+ original_structure = self._lookup_course(usage_locator.course_key)['structure']
if original_structure['root'] == usage_locator.block_id:
raise ValueError("Cannot delete the root of a course")
index_entry = self._get_index_if_valid(usage_locator, force)
@@ -1283,32 +1325,29 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# update index if appropriate and structures
self.db_connection.insert_structure(new_structure)
- result = CourseLocator(version_guid=new_id)
-
- # update the index entry if appropriate
if index_entry is not None:
+ # update the index entry if appropriate
self._update_head(index_entry, usage_locator.branch, new_id)
- result.package_id = usage_locator.package_id
- result.branch = usage_locator.branch
+ result = usage_locator.course_key.for_version(new_id)
+ else:
+ result = CourseLocator(version_guid=new_id)
return result
- def delete_course(self, package_id):
+ def delete_course(self, course_key, user_id=None):
"""
Remove the given course from the course index.
Only removes the course from the index. The data remains. You can use create_course
with a versions hash to restore the course; however, the edited_on and
edited_by won't reflect the originals, of course.
-
- :param package_id: uses package_id rather than locator to emphasize its global effect
"""
- index = self.db_connection.get_course_index(package_id)
+ index = self.db_connection.get_course_index(course_key)
if index is None:
- raise ItemNotFoundError(package_id)
+ raise ItemNotFoundError(course_key)
# this is the only real delete in the system. should it do something else?
- log.info(u"deleting course from split-mongo: %s", package_id)
- self.db_connection.delete_course_index(index['_id'])
+ log.info(u"deleting course from split-mongo: %s", course_key)
+ self.db_connection.delete_course_index(index)
def get_errored_courses(self):
"""
@@ -1413,38 +1452,6 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
# clear cache again b/c inheritance may be wrong over orphans
self._clear_cache(original_structure['_id'])
- def _block_matches(self, value, qualifiers):
- '''
- Return True or False depending on whether the value (block contents)
- matches the qualifiers as per get_items
- :param value:
- :param qualifiers:
- '''
- for key, criteria in qualifiers.iteritems():
- if key in value:
- target = value[key]
- if not self._value_matches(target, criteria):
- return False
- elif criteria is not None:
- return False
- return True
-
- def _value_matches(self, target, criteria):
- ''' helper for _block_matches '''
- if isinstance(target, list):
- return any(self._value_matches(ele, criteria)
- for ele in target)
- elif isinstance(criteria, dict):
- if '$regex' in criteria:
- return re.search(criteria['$regex'], target) is not None
- elif not isinstance(target, dict):
- return False
- else:
- return (isinstance(target, dict) and
- self._block_matches(target, criteria))
- else:
- return criteria == target
-
def _get_index_if_valid(self, locator, force=False, continue_version=False):
"""
If the locator identifies a course and points to its draft (or plausibly its draft),
@@ -1458,7 +1465,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
:param continue_version: if True, assumes this operation requires a head version and will not create a new
version but instead continue an existing transaction on this version. This flag cannot be True if force is True.
"""
- if locator.package_id is None or locator.branch is None:
+ if locator.org is None or locator.offering is None or locator.branch is None:
if continue_version:
raise InsufficientSpecificationError(
"To continue a version, the locator must point to one ({}).".format(locator)
@@ -1466,7 +1473,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
else:
return None
else:
- index_entry = self.db_connection.get_course_index(locator.package_id)
+ index_entry = self.db_connection.get_course_index(locator)
is_head = (
locator.version_guid is None or
index_entry['versions'][locator.branch] == locator.version_guid
diff --git a/common/lib/xmodule/xmodule/modulestore/store_utilities.py b/common/lib/xmodule/xmodule/modulestore/store_utilities.py
index 4ba433b0e79..925441abfb8 100644
--- a/common/lib/xmodule/xmodule/modulestore/store_utilities.py
+++ b/common/lib/xmodule/xmodule/modulestore/store_utilities.py
@@ -2,7 +2,6 @@ import re
import logging
from xmodule.contentstore.content import StaticContent
-from xmodule.modulestore import Location
def _prefix_only_url_replace_regex(prefix):
@@ -46,10 +45,6 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
"""
- course_id_dict = Location.parse_course_id(source_course_id)
- course_id_dict['tag'] = 'i4x'
- course_id_dict['category'] = 'course'
-
def portable_asset_link_subtitution(match):
quote = match.group('quote')
rest = match.group('rest')
@@ -60,27 +55,21 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
rest = match.group('rest')
return quote + '/jump_to_id/' + rest + quote
- def generic_courseware_link_substitution(match):
- parts = Location.parse_course_id(dest_course_id)
- parts['quote'] = match.group('quote')
- parts['rest'] = match.group('rest')
- return u'{quote}/courses/{org}/{course}/{name}/{rest}{quote}'.format(**parts)
-
- course_location = Location(course_id_dict)
-
# NOTE: ultimately link updating is not a hard requirement, so if something blows up with
- # the regex subsitution, log the error and continue
+ # the regex substitution, log the error and continue
+ c4x_link_base = StaticContent.get_base_url_path_for_course_assets(source_course_id)
try:
- c4x_link_base = u'{0}/'.format(StaticContent.get_base_url_path_for_course_assets(course_location))
text = re.sub(_prefix_only_url_replace_regex(c4x_link_base), portable_asset_link_subtitution, text)
- except Exception, e:
- logging.warning("Error going regex subtituion %r on text = %r.\n\nError msg = %s", c4x_link_base, text, str(e))
+ except Exception as exc: # pylint: disable=broad-except
+ logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", c4x_link_base, text, str(exc))
+ jump_to_link_base = u'/courses/{course_key_string}/jump_to/i4x://{course_key.org}/{course_key.course}/'.format(
+ course_key_string=source_course_id.to_deprecated_string(), course_key=source_course_id
+ )
try:
- jump_to_link_base = u'/courses/{org}/{course}/{name}/jump_to/i4x://{org}/{course}/'.format(**course_id_dict)
text = re.sub(_prefix_and_category_url_replace_regex(jump_to_link_base), portable_jump_to_link_substitution, text)
- except Exception, e:
- logging.warning("Error going regex subtituion %r on text = %r.\n\nError msg = %s", jump_to_link_base, text, str(e))
+ except Exception as exc: # pylint: disable=broad-except
+ logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", jump_to_link_base, text, str(exc))
# Also, there commonly is a set of link URL's used in the format:
# /courses/<org>/<course>/<name> which will be broken if migrated to a different course_id
@@ -90,65 +79,46 @@ def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
#
if source_course_id != dest_course_id:
try:
- generic_courseware_link_base = u'/courses/{org}/{course}/{name}/'.format(**course_id_dict)
+ generic_courseware_link_base = u'/courses/{}/'.format(source_course_id.to_deprecated_string())
text = re.sub(_prefix_only_url_replace_regex(generic_courseware_link_base), portable_asset_link_subtitution, text)
- except Exception, e:
- logging.warning("Error going regex subtituion %r on text = %r.\n\nError msg = %s", generic_courseware_link_base, text, str(e))
+ except Exception as exc: # pylint: disable=broad-except
+ logging.warning("Error producing regex substitution %r for text = %r.\n\nError msg = %s", source_course_id, text, str(exc))
return text
-def _clone_modules(modulestore, modules, source_location, dest_location):
+def _clone_modules(modulestore, modules, source_course_id, dest_course_id):
for module in modules:
- original_loc = Location(module.location)
-
- if original_loc.category != 'course':
- module.location = module.location._replace(
- tag=dest_location.tag,
- org=dest_location.org,
- course=dest_location.course
- )
- else:
- # on the course module we also have to update the module name
- module.location = module.location._replace(
- tag=dest_location.tag,
- org=dest_location.org,
- course=dest_location.course,
- name=dest_location.name
- )
+ original_loc = module.location
+ module.location = module.location.map_into_course(dest_course_id)
print "Cloning module {0} to {1}....".format(original_loc, module.location)
if 'data' in module.fields and module.fields['data'].is_set_on(module) and isinstance(module.data, basestring):
module.data = rewrite_nonportable_content_links(
- source_location.course_id, dest_location.course_id, module.data
+ source_course_id, dest_course_id, module.data
)
# repoint children
if module.has_children:
new_children = []
- for child_loc_url in module.children:
- child_loc = Location(child_loc_url)
- child_loc = child_loc._replace(
- tag=dest_location.tag,
- org=dest_location.org,
- course=dest_location.course
- )
- new_children.append(child_loc.url())
+ for child_loc in module.children:
+ child_loc = child_loc.map_into_course(dest_course_id)
+ new_children.append(child_loc)
module.children = new_children
modulestore.update_item(module, '**replace_user**')
-def clone_course(modulestore, contentstore, source_location, dest_location, delete_original=False):
+def clone_course(modulestore, contentstore, source_course_id, dest_course_id):
# check to see if the dest_location exists as an empty course
# we need an empty course because the app layers manage the permissions and users
- if not modulestore.has_item(dest_location.course_id, dest_location):
- raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location))
+ if not modulestore.has_course(dest_course_id):
+ raise Exception(u"An empty course at {0} must have already been created. Aborting...".format(dest_course_id))
# verify that the dest_location really is an empty course, which means only one with an optional 'overview'
- dest_modules = modulestore.get_items([dest_location.tag, dest_location.org, dest_location.course, None, None, None])
+ dest_modules = modulestore.get_items(dest_course_id)
basically_empty = True
for module in dest_modules:
@@ -163,107 +133,63 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
# check to see if the source course is actually there
- if not modulestore.has_item(source_location.course_id, source_location):
- raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
+ if not modulestore.has_course(source_course_id):
+ raise Exception("Cannot find a course at {0}. Aborting".format(source_course_id))
# Get all modules under this namespace which is (tag, org, course) tuple
- modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
- _clone_modules(modulestore, modules, source_location, dest_location)
+ modules = modulestore.get_items(source_course_id, revision=None)
+ _clone_modules(modulestore, modules, source_course_id, dest_course_id)
- modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, 'draft'])
- _clone_modules(modulestore, modules, source_location, dest_location)
+ modules = modulestore.get_items(source_course_id, revision='draft')
+ _clone_modules(modulestore, modules, source_course_id, dest_course_id)
# now iterate through all of the assets and clone them
# first the thumbnails
- thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
- for thumb in thumbs:
- thumb_loc = Location(thumb["_id"])
- content = contentstore.find(thumb_loc)
- content.location = content.location._replace(org=dest_location.org,
- course=dest_location.course)
+ thumb_keys = contentstore.get_all_content_thumbnails_for_course(source_course_id)
+ for thumb_key in thumb_keys:
+ content = contentstore.find(thumb_key)
+ content.location = content.location.map_into_course(dest_course_id)
- print "Cloning thumbnail {0} to {1}".format(thumb_loc, content.location)
+ print "Cloning thumbnail {0} to {1}".format(thumb_key, content.location)
contentstore.save(content)
# now iterate through all of the assets, also updating the thumbnail pointer
- assets, __ = contentstore.get_all_content_for_course(source_location)
- for asset in assets:
- asset_loc = Location(asset["_id"])
- content = contentstore.find(asset_loc)
- content.location = content.location._replace(org=dest_location.org,
- course=dest_location.course)
+ asset_keys, __ = contentstore.get_all_content_for_course(source_course_id)
+ for asset_key in asset_keys:
+ content = contentstore.find(asset_key)
+ content.location = content.location.map_into_course(dest_course_id)
# be sure to update the pointer to the thumbnail
if content.thumbnail_location is not None:
- content.thumbnail_location = content.thumbnail_location._replace(org=dest_location.org,
- course=dest_location.course)
+ content.thumbnail_location = content.thumbnail_location.map_into_course(dest_course_id)
- print "Cloning asset {0} to {1}".format(asset_loc, content.location)
+ print "Cloning asset {0} to {1}".format(asset_key, content.location)
contentstore.save(content)
return True
-def _delete_modules_except_course(modulestore, modules, source_location, commit):
- """
- This helper method will just enumerate through a list of modules and delete them, except for the
- top-level course module
- """
- for module in modules:
- if module.category != 'course':
- logging.warning("Deleting {0}...".format(module.location))
- if commit:
- # sanity check. Make sure we're not deleting a module in the incorrect course
- if module.location.org != source_location.org or module.location.course != source_location.course:
- raise Exception('Module {0} is not in same namespace as {1}. This should not happen! Aborting...'.format(module.location, source_location))
- modulestore.delete_item(module.location)
-
-
-def _delete_assets(contentstore, assets, commit):
- """
- This helper method will enumerate through a list of assets and delete them
- """
- for asset in assets:
- asset_loc = Location(asset["_id"])
- id = StaticContent.get_id_from_location(asset_loc)
- logging.warning("Deleting {0}...".format(id))
- if commit:
- contentstore.delete(id)
-
-
-def delete_course(modulestore, contentstore, source_location, commit=False):
+def delete_course(modulestore, contentstore, course_key, commit=False):
"""
This method will actually do the work to delete all content in a course in a MongoDB backed
courseware store. BE VERY CAREFUL, this is not reversable.
"""
# check to see if the source course is actually there
- if not modulestore.has_item(source_location.course_id, source_location):
- raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
+ if not modulestore.has_course(course_key):
+ raise Exception("Cannot find a course at {0}. Aborting".format(course_key))
- # first delete all of the thumbnails
- thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
- _delete_assets(contentstore, thumbs, commit)
-
- # then delete all of the assets
- assets, __ = contentstore.get_all_content_for_course(source_location)
- _delete_assets(contentstore, assets, commit)
-
- # then delete all course modules
- modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
- _delete_modules_except_course(modulestore, modules, source_location, commit)
-
- # then delete all draft course modules
- modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, 'draft'])
- _delete_modules_except_course(modulestore, modules, source_location, commit)
+ if commit:
+ print "Deleting assets and thumbnails {}".format(course_key)
+ contentstore.delete_all_course_assets(course_key)
- # finally delete the top-level course module itself
- print "Deleting {0}...".format(source_location)
+ # finally delete the course
+ print "Deleting {0}...".format(course_key)
if commit:
- modulestore.delete_item(source_location)
+ modulestore.delete_course(course_key, '**replace-user**')
return True
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
index 07c23f8f42d..3c1d9a0c6a9 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
@@ -205,7 +205,7 @@ class ModuleStoreTestCase(TestCase):
"""
store = editable_modulestore()
store.update_item(course, '**replace_user**')
- updated_course = store.get_instance(course.id, course.location)
+ updated_course = store.get_course(course.id)
return updated_course
@staticmethod
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/factories.py b/common/lib/xmodule/xmodule/modulestore/tests/factories.py
index 8f4cf1ffa8b..3f1c1b78ba2 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/factories.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/factories.py
@@ -2,7 +2,8 @@ from factory import Factory, lazy_attribute_sequence, lazy_attribute
from factory.containers import CyclicDefinitionError
from uuid import uuid4
-from xmodule.modulestore import Location, prefer_xmodules
+from xmodule.modulestore import prefer_xmodules
+from xmodule.modulestore.locations import Location
from xblock.core import XBlock
@@ -36,6 +37,7 @@ class CourseFactory(XModuleFactory):
number = '999'
display_name = 'Robot Super Course'
+ # pylint: disable=unused-argument
@classmethod
def _create(cls, target_class, **kwargs):
@@ -46,8 +48,10 @@ class CourseFactory(XModuleFactory):
# because the factory provides a default 'number' arg, prefer the non-defaulted 'course' arg if any
number = kwargs.pop('course', kwargs.pop('number', None))
store = kwargs.pop('modulestore')
+ name = kwargs.get('name', kwargs.get('run', Location.clean(kwargs.get('display_name'))))
+ run = kwargs.get('run', name)
- location = Location('i4x', org, number, 'course', Location.clean(kwargs.get('display_name')))
+ location = Location(org, number, run, 'course', name)
# Write the data to the mongo datastore
new_course = store.create_xmodule(location, metadata=kwargs.get('metadata', None))
@@ -82,11 +86,15 @@ class ItemFactory(XModuleFactory):
else:
dest_name = self.display_name.replace(" ", "_")
- return self.parent_location.replace(category=self.category, name=dest_name)
+ new_location = self.parent_location.course_key.make_usage_key(
+ self.category,
+ dest_name
+ )
+ return new_location
@lazy_attribute
def parent_location(self):
- default_location = Location('i4x://MITx/999/course/Robot_Super_Course')
+ default_location = Location('MITx', '999', 'Robot_Super_Course', 'course', 'Robot_Super_Course', None)
try:
parent = self.parent
# This error is raised if the caller hasn't provided either parent or parent_location
@@ -127,12 +135,14 @@ class ItemFactory(XModuleFactory):
# catch any old style users before they get into trouble
assert 'template' not in kwargs
- parent_location = Location(kwargs.pop('parent_location', None))
+ parent_location = kwargs.pop('parent_location', None)
data = kwargs.pop('data', None)
category = kwargs.pop('category', None)
display_name = kwargs.pop('display_name', None)
metadata = kwargs.pop('metadata', {})
location = kwargs.pop('location')
+
+ assert isinstance(location, Location)
assert location != parent_location
store = kwargs.pop('modulestore')
@@ -164,7 +174,7 @@ class ItemFactory(XModuleFactory):
store.update_item(module)
if 'detached' not in module._class_tags:
- parent.children.append(location.url())
+ parent.children.append(location)
store.update_item(parent, '**replace_user**')
return store.get_item(location)
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_location.py b/common/lib/xmodule/xmodule/modulestore/tests/test_location.py
index c2ab1fb4b79..0a9ffd53c99 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_location.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_location.py
@@ -1,8 +1,11 @@
+"""
+Thorough tests of the Location class
+"""
import ddt
from unittest import TestCase
-from xmodule.modulestore import Location
-from xmodule.modulestore.exceptions import InvalidLocationError
+from opaque_keys import InvalidKeyError
+from xmodule.modulestore.locations import Location, AssetLocation, SlashSeparatedCourseKey
# Pairs for testing the clean* functions.
# The first item in the tuple is the input string.
@@ -23,117 +26,110 @@ class TestLocations(TestCase):
Tests of :class:`.Location`
"""
@ddt.data(
- "tag://org/course/category/name",
- "tag://org/course/category/name@revision"
+ "org+course+run+category+name",
+ "org+course+run+category+name@revision"
)
def test_string_roundtrip(self, url):
- self.assertEquals(url, Location(url).url())
- self.assertEquals(url, str(Location(url)))
+ self.assertEquals(url, Location._from_string(url)._to_string()) # pylint: disable=protected-access
@ddt.data(
- {
- 'tag': 'tag',
+ "i4x://org/course/category/name",
+ "i4x://org/course/category/name@revision"
+ )
+ def test_deprecated_roundtrip(self, url):
+ course_id = SlashSeparatedCourseKey('org', 'course', 'run')
+ self.assertEquals(
+ url,
+ course_id.make_usage_key_from_deprecated_string(url).to_deprecated_string()
+ )
+
+ def test_invalid_chars_ssck(self):
+ """
+ Test that the ssck constructor fails if given invalid chars
+ """
+ valid_base = SlashSeparatedCourseKey(u'org.dept-1%2', u'course.sub-2%3', u'run.faster-4%5')
+ for key in SlashSeparatedCourseKey.KEY_FIELDS:
+ with self.assertRaises(InvalidKeyError):
+ # this ends up calling the constructor where the legality check should occur
+ valid_base.replace(**{key: u'funny thing'})
+
+ def test_invalid_chars_location(self):
+ """
+ Test that the location constructor fails if given invalid chars
+ """
+ course_key = SlashSeparatedCourseKey(u'org.dept-1%2', u'course.sub-2%3', u'run.faster-4%5')
+ valid_base = course_key.make_usage_key('tomato-again%9', 'block-head:sub-4%9')
+ for key in SlashSeparatedCourseKey.KEY_FIELDS:
+ with self.assertRaises(InvalidKeyError):
+ # this ends up calling the constructor where the legality check should occur
+ valid_base.replace(**{key: u'funny thing'})
+
+ @ddt.data(
+ ((), {
+ 'org': 'org',
'course': 'course',
+ 'run': 'run',
'category': 'category',
'name': 'name',
- 'org': 'org'
- },
- {
- 'tag': 'tag',
+ }, 'org', 'course', 'run', 'category', 'name', None),
+ ((), {
+ 'org': 'org',
'course': 'course',
+ 'run': 'run',
'category': 'category',
'name': 'name:more_name',
- 'org': 'org'
- },
- ['tag', 'org', 'course', 'category', 'name'],
- "tag://org/course/category/name",
- "tag://org/course/category/name@revision",
- u"tag://org/course/category/name",
- u"tag://org/course/category/name@revision",
+ }, 'org', 'course', 'run', 'category', 'name:more_name', None),
+ (['org', 'course', 'run', 'category', 'name'], {}, 'org', 'course', 'run', 'category', 'name', None),
)
- def test_is_valid(self, loc):
- self.assertTrue(Location.is_valid(loc))
+ @ddt.unpack
+ def test_valid_locations(self, args, kwargs, org, course, run, category, name, revision):
+ location = Location(*args, **kwargs)
+ self.assertEquals(org, location.org)
+ self.assertEquals(course, location.course)
+ self.assertEquals(run, location.run)
+ self.assertEquals(category, location.category)
+ self.assertEquals(name, location.name)
+ self.assertEquals(revision, location.revision)
@ddt.data(
- {
+ (("foo",), {}),
+ (["foo", "bar"], {}),
+ (["foo", "bar", "baz", "blat/blat", "foo"], {}),
+ (["foo", "bar", "baz", "blat", "foo/bar"], {}),
+ (["foo", "bar", "baz", "blat:blat", "foo:bar"], {}), # ':' ok in name, not in category
+ (('org', 'course', 'run', 'category', 'name with spaces', 'revision'), {}),
+ (('org', 'course', 'run', 'category', 'name/with/slashes', 'revision'), {}),
+ (('org', 'course', 'run', 'category', 'name', u'\xae'), {}),
+ (('org', 'course', 'run', 'category', u'\xae', 'revision'), {}),
+ ((), {
'tag': 'tag',
'course': 'course',
'category': 'category',
'name': 'name@more_name',
'org': 'org'
- },
- {
+ }),
+ ((), {
'tag': 'tag',
'course': 'course',
'category': 'category',
'name': 'name ', # extra space
'org': 'org'
- },
- "foo",
- ["foo"],
- ["foo", "bar"],
- ["foo", "bar", "baz", "blat:blat", "foo:bar"], # ':' ok in name, not in category
- "tag://org/course/category/name with spaces@revision",
- "tag://org/course/category/name/with/slashes@revision",
- u"tag://org/course/category/name\xae", # No non-ascii characters for now
- u"tag://org/course/category\xae/name", # No non-ascii characters for now
+ }),
)
- def test_is_invalid(self, loc):
- self.assertFalse(Location.is_valid(loc))
-
- def test_dict(self):
- input_dict = {
- 'tag': 'tag',
- 'course': 'course',
- 'category': 'category',
- 'name': 'name',
- 'org': 'org'
- }
-
- self.assertEquals("tag://org/course/category/name", Location(input_dict).url())
- self.assertEquals(dict(revision=None, **input_dict), Location(input_dict).dict())
-
- input_dict['revision'] = 'revision'
- self.assertEquals("tag://org/course/category/name@revision", Location(input_dict).url())
- self.assertEquals(input_dict, Location(input_dict).dict())
-
- def test_list(self):
- input_list = ['tag', 'org', 'course', 'category', 'name']
- self.assertEquals("tag://org/course/category/name", Location(input_list).url())
- self.assertEquals(input_list + [None], Location(input_list).list())
-
- input_list.append('revision')
- self.assertEquals("tag://org/course/category/name@revision", Location(input_list).url())
- self.assertEquals(input_list, Location(input_list).list())
-
- def test_location(self):
- input_list = ['tag', 'org', 'course', 'category', 'name']
- self.assertEquals("tag://org/course/category/name", Location(Location(input_list)).url())
-
- def test_none(self):
- self.assertEquals([None] * 6, Location(None).list())
-
- @ddt.data(
- "foo",
- ["foo", "bar"],
- ["foo", "bar", "baz", "blat/blat", "foo"],
- ["foo", "bar", "baz", "blat", "foo/bar"],
- "tag://org/course/category/name with spaces@revision",
- "tag://org/course/category/name/revision",
- )
- def test_invalid_locations(self, loc):
- with self.assertRaises(InvalidLocationError):
- Location(loc)
+ @ddt.unpack
+ def test_invalid_locations(self, *args, **kwargs):
+ with self.assertRaises(TypeError):
+ Location(*args, **kwargs)
def test_equality(self):
self.assertEquals(
- Location('tag', 'org', 'course', 'category', 'name'),
- Location('tag', 'org', 'course', 'category', 'name')
+ Location('tag', 'org', 'course', 'run', 'category', 'name'),
+ Location('tag', 'org', 'course', 'run', 'category', 'name')
)
self.assertNotEquals(
- Location('tag', 'org', 'course', 'category', 'name1'),
- Location('tag', 'org', 'course', 'category', 'name')
+ Location('tag', 'org', 'course', 'run', 'category', 'name1'),
+ Location('tag', 'org', 'course', 'run', 'category', 'name')
)
@ddt.data(
@@ -164,42 +160,38 @@ class TestLocations(TestCase):
self.assertEquals(Location.clean_for_html(pair[0]), pair[1])
def test_html_id(self):
- loc = Location("tag://org/course/cat/name:more_name@rev")
- self.assertEquals(loc.html_id(), "tag-org-course-cat-name_more_name-rev")
-
- def test_course_id(self):
- loc = Location('i4x', 'mitX', '103', 'course', 'test2')
- self.assertEquals('mitX/103/test2', loc.course_id)
-
- loc = Location('i4x', 'mitX', '103', '_not_a_course', 'test2')
- with self.assertRaises(InvalidLocationError):
- loc.course_id # pylint: disable=pointless-statement
+ loc = Location('org', 'course', 'run', 'cat', 'name:more_name', 'rev')
+ self.assertEquals(loc.html_id(), "i4x-org-course-cat-name_more_name-rev")
def test_replacement(self):
# pylint: disable=protected-access
self.assertEquals(
- Location('t://o/c/c/n@r')._replace(name='new_name'),
- Location('t://o/c/c/new_name@r'),
+ Location('o', 'c', 'r', 'c', 'n', 'r').replace(name='new_name'),
+ Location('o', 'c', 'r', 'c', 'new_name', 'r'),
)
- with self.assertRaises(InvalidLocationError):
- Location('t://o/c/c/n@r')._replace(name=u'name\xae')
+ with self.assertRaises(InvalidKeyError):
+ Location('o', 'c', 'r', 'c', 'n', 'r').replace(name=u'name\xae')
@ddt.data('org', 'course', 'category', 'name', 'revision')
def test_immutable(self, attr):
- loc = Location('t://o/c/c/n@r')
+ loc = Location('o', 'c', 'r', 'c', 'n', 'r')
with self.assertRaises(AttributeError):
setattr(loc, attr, attr)
- def test_parse_course_id(self):
- """
- Test the parse_course_id class method
- """
- source_string = "myorg/mycourse/myrun"
- parsed = Location.parse_course_id(source_string)
- self.assertEqual(parsed['org'], 'myorg')
- self.assertEqual(parsed['course'], 'mycourse')
- self.assertEqual(parsed['name'], 'myrun')
- with self.assertRaises(ValueError):
- Location.parse_course_id('notlegit.id/foo')
+ def test_map_into_course_location(self):
+ loc = Location('org', 'course', 'run', 'cat', 'name:more_name', 'rev')
+ course_key = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")
+ self.assertEquals(
+ Location("edX", "toy", "2012_Fall", 'cat', 'name:more_name', 'rev'),
+ loc.map_into_course(course_key)
+ )
+
+ def test_map_into_course_asset_location(self):
+ loc = AssetLocation('org', 'course', 'run', 'asset', 'foo.bar')
+ course_key = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")
+ self.assertEquals(
+ AssetLocation("edX", "toy", "2012_Fall", 'asset', 'foo.bar'),
+ loc.map_into_course(course_key)
+ )
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py b/common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
index 395271f91b3..4dba9b11f7e 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_location_mapper.py
@@ -1,15 +1,15 @@
-'''
-Created on Aug 5, 2013
-
-@author: dmitchell
-'''
+"""
+Test the loc mapper store
+"""
import unittest
import uuid
from xmodule.modulestore import Location
-from xmodule.modulestore.locator import BlockUsageLocator
+from xmodule.modulestore.locator import BlockUsageLocator, CourseLocator
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.loc_mapper_store import LocMapperStore
from mock import Mock
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
+import bson.son
class LocMapperSetupSansDjango(unittest.TestCase):
@@ -41,41 +41,47 @@ class TestLocationMapper(LocMapperSetupSansDjango):
Test the location to locator mapper
"""
def test_create_map(self):
+ def _construct_course_son(org, course, run):
+ """
+ Make a lookup son
+ """
+ return bson.son.SON([
+ ('org', org),
+ ('course', course),
+ ('name', run)
+ ])
+
org = 'foo_org'
- course = 'bar_course'
- loc_mapper().create_map_entry(Location('i4x', org, course, 'course', 'baz_run'))
+ course1 = 'bar_course'
+ run = 'baz_run'
+ loc_mapper().create_map_entry(SlashSeparatedCourseKey(org, course1, run))
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
- '_id': loc_mapper()._construct_location_son(org, course, 'baz_run')
+ '_id': _construct_course_son(org, course1, run)
})
self.assertIsNotNone(entry, "Didn't find entry")
- self.assertEqual(entry['course_id'], '{}.{}.baz_run'.format(org, course))
+ self.assertEqual(entry['org'], org)
+ self.assertEqual(entry['offering'], '{}.{}'.format(course1, run))
self.assertEqual(entry['draft_branch'], 'draft')
self.assertEqual(entry['prod_branch'], 'published')
self.assertEqual(entry['block_map'], {})
- # ensure create_entry does the right thing when not given a course (creates org/course
- # rather than org/course/run course_id)
- loc_mapper().create_map_entry(Location('i4x', org, course, 'vertical', 'baz_vert'))
- # find the one which has no name
- entry = loc_mapper().location_map.find_one({
- '_id' : loc_mapper()._construct_location_son(org, course, None)
- })
- self.assertIsNotNone(entry, "Didn't find entry")
- self.assertEqual(entry['course_id'], '{}.{}'.format(org, course))
-
- course = 'quux_course'
+ course2 = 'quux_course'
# oldname: {category: newname}
block_map = {'abc123': {'problem': 'problem2'}}
loc_mapper().create_map_entry(
- Location('i4x', org, course, 'problem', 'abc123', 'draft'),
- 'foo_org.geek_dept.quux_course.baz_run',
+ SlashSeparatedCourseKey(org, course2, run),
+ 'foo_org.geek_dept',
+ 'quux_course.baz_run',
'wip',
'live',
block_map)
- entry = loc_mapper().location_map.find_one({'_id.org': org, '_id.course': course})
+ entry = loc_mapper().location_map.find_one({
+ '_id': _construct_course_son(org, course2, run)
+ })
self.assertIsNotNone(entry, "Didn't find entry")
- self.assertEqual(entry['course_id'], 'foo_org.geek_dept.quux_course.baz_run')
+ self.assertEqual(entry['org'], 'foo_org.geek_dept')
+ self.assertEqual(entry['offering'], '{}.{}'.format(course2, run))
self.assertEqual(entry['draft_branch'], 'wip')
self.assertEqual(entry['prod_branch'], 'live')
self.assertEqual(entry['block_map'], block_map)
@@ -87,51 +93,50 @@ class TestLocationMapper(LocMapperSetupSansDjango):
org = u'foo_org'
course = u'bar_course'
run = u'baz_run'
- course_location = Location('i4x', org, course, 'course', run)
- course_locator = loc_mapper().translate_location(course_location.course_id, course_location)
+ course_location = SlashSeparatedCourseKey(org, course, run)
loc_mapper().create_map_entry(course_location)
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
- '_id': loc_mapper()._construct_location_son(org, course, run)
+ '_id': loc_mapper()._construct_course_son(course_location)
})
self.assertIsNotNone(entry, 'Entry not found in loc_mapper')
- self.assertEqual(entry['course_id'], u'{0}.{1}.{2}'.format(org, course, run))
+ self.assertEqual(entry['offering'], u'{1}.{2}'.format(org, course, run))
# now delete course location from loc_mapper and cache and test that course location no longer
# exists in loca_mapper and cache
loc_mapper().delete_course_mapping(course_location)
# pylint: disable=protected-access
entry = loc_mapper().location_map.find_one({
- '_id': loc_mapper()._construct_location_son(org, course, run)
+ '_id': loc_mapper()._construct_course_son(course_location)
})
self.assertIsNone(entry, 'Entry found in loc_mapper')
# pylint: disable=protected-access
- cached_value = loc_mapper()._get_location_from_cache(course_locator)
+ cached_value = loc_mapper()._get_location_from_cache(course_location.make_usage_key('course', run))
self.assertIsNone(cached_value, 'course_locator found in cache')
# pylint: disable=protected-access
- cached_value = loc_mapper()._get_course_location_from_cache(course_locator.package_id)
+ cached_value = loc_mapper()._get_course_location_from_cache(course_location)
self.assertIsNone(cached_value, 'Entry found in cache')
- def translate_n_check(self, location, old_style_course_id, new_style_package_id, block_id, branch, add_entry=False):
+ def translate_n_check(self, location, org, offering, block_id, branch, add_entry=False):
"""
- Request translation, check package_id, block_id, and branch
+ Request translation, check org, offering, block_id, and branch
"""
prob_locator = loc_mapper().translate_location(
- old_style_course_id,
location,
- published= (branch=='published'),
+ published=(branch == 'published'),
add_entry_if_missing=add_entry
)
- self.assertEqual(prob_locator.package_id, new_style_package_id)
+ self.assertEqual(prob_locator.org, org)
+ self.assertEqual(prob_locator.offering, offering)
self.assertEqual(prob_locator.block_id, block_id)
self.assertEqual(prob_locator.branch, branch)
course_locator = loc_mapper().translate_location_to_course_locator(
- old_style_course_id,
- location,
- published=(branch == 'published'),
+ location.course_key,
+ published=(branch == 'published'),
)
- self.assertEqual(course_locator.package_id, new_style_package_id)
+ self.assertEqual(course_locator.org, org)
+ self.assertEqual(course_locator.offering, offering)
self.assertEqual(course_locator.branch, branch)
def test_translate_location_read_only(self):
@@ -141,46 +146,45 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# lookup before there are any maps
org = 'foo_org'
course = 'bar_course'
- old_style_course_id = '{}/{}/{}'.format(org, course, 'baz_run')
+ run = 'baz_run'
+ slash_course_key = SlashSeparatedCourseKey(org, course, run)
with self.assertRaises(ItemNotFoundError):
_ = loc_mapper().translate_location(
- old_style_course_id,
- Location('i4x', org, course, 'problem', 'abc123'),
+ Location(org, course, run, 'problem', 'abc123'),
add_entry_if_missing=False
)
- new_style_package_id = '{}.geek_dept.{}.baz_run'.format(org, course)
+ new_style_org = '{}.geek_dept'.format(org)
+ new_style_offering = '.{}.{}'.format(course, run)
block_map = {
'abc123': {'problem': 'problem2', 'vertical': 'vertical2'},
'def456': {'problem': 'problem4'},
'ghi789': {'problem': 'problem7'},
}
loc_mapper().create_map_entry(
- Location('i4x', org, course, 'course', 'baz_run'),
- new_style_package_id,
+ slash_course_key,
+ new_style_org, new_style_offering,
block_map=block_map
)
- test_problem_locn = Location('i4x', org, course, 'problem', 'abc123')
- # only one course matches
+ test_problem_locn = Location(org, course, run, 'problem', 'abc123')
- # look for w/ only the Location (works b/c there's only one possible course match). Will force
- # cache as default translation for this problemid
- self.translate_n_check(test_problem_locn, None, new_style_package_id, 'problem2', 'published')
+ self.translate_n_check(test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published')
# look for non-existent problem
with self.assertRaises(ItemNotFoundError):
loc_mapper().translate_location(
- None,
- Location('i4x', org, course, 'problem', '1def23'),
+ Location(org, course, run, 'problem', '1def23'),
add_entry_if_missing=False
)
test_no_cat_locn = test_problem_locn.replace(category=None)
with self.assertRaises(InvalidLocationError):
loc_mapper().translate_location(
- old_style_course_id, test_no_cat_locn, False, False
+ slash_course_key.make_usage_key(None, 'abc123'), test_no_cat_locn, False, False
)
test_no_cat_locn = test_no_cat_locn.replace(name='def456')
- # only one course matches
- self.translate_n_check(test_no_cat_locn, old_style_course_id, new_style_package_id, 'problem4', 'published')
+
+ self.translate_n_check(
+ test_no_cat_locn, new_style_org, new_style_offering, 'problem4', 'published'
+ )
# add a distractor course (note that abc123 has a different translation in this one)
distractor_block_map = {
@@ -188,37 +192,23 @@ class TestLocationMapper(LocMapperSetupSansDjango):
'def456': {'problem': 'problem4'},
'ghi789': {'problem': 'problem7'},
}
- test_delta_new_id = '{}.geek_dept.{}.{}'.format(org, course, 'delta_run')
- test_delta_old_id = '{}/{}/{}'.format(org, course, 'delta_run')
+ run = 'delta_run'
+ test_delta_new_org = '{}.geek_dept'.format(org)
+ test_delta_new_offering = '{}.{}'.format(course, run)
loc_mapper().create_map_entry(
- Location('i4x', org, course, 'course', 'delta_run'),
- test_delta_new_id,
+ SlashSeparatedCourseKey(org, course, run),
+ test_delta_new_org, test_delta_new_offering,
block_map=distractor_block_map
)
# test that old translation still works
- self.translate_n_check(test_problem_locn, old_style_course_id, new_style_package_id, 'problem2', 'published')
- # and new returns new id
- self.translate_n_check(test_problem_locn, test_delta_old_id, test_delta_new_id, 'problem3', 'published')
- # look for default translation of uncached Location (not unique; so, just verify it returns something)
- prob_locator = loc_mapper().translate_location(
- None,
- Location('i4x', org, course, 'problem', 'def456'),
- add_entry_if_missing=False
+ self.translate_n_check(
+ test_problem_locn, new_style_org, new_style_offering, 'problem2', 'published'
)
- self.assertIsNotNone(prob_locator, "couldn't find ambiguous location")
-
- # make delta_run default course: anything not cached using None as old_course_id will use this
- loc_mapper().create_map_entry(
- Location('i4x', org, course, 'problem', '789abc123efg456'),
- test_delta_new_id,
- block_map=block_map
+ # and new returns new id
+ self.translate_n_check(
+ test_problem_locn.replace(run=run), test_delta_new_org, test_delta_new_offering,
+ 'problem3', 'published'
)
- # now an uncached ambiguous query should return delta
- test_unused_locn = Location('i4x', org, course, 'problem', 'ghi789')
- self.translate_n_check(test_unused_locn, None, test_delta_new_id, 'problem7', 'published')
-
- # get the draft one (I'm sorry this is getting long)
- self.translate_n_check(test_unused_locn, None, test_delta_new_id, 'problem7', 'draft')
def test_translate_location_dwim(self):
"""
@@ -227,27 +217,27 @@ class TestLocationMapper(LocMapperSetupSansDjango):
"""
org = 'foo_org'
course = 'bar_course'
- old_style_course_id = '{}/{}/{}'.format(org, course, 'baz_run')
+ run = 'baz_run'
problem_name = 'abc123abc123abc123abc123abc123f9'
- location = Location('i4x', org, course, 'problem', problem_name)
- new_style_package_id = '{}.{}.{}'.format(org, course, 'baz_run')
- self.translate_n_check(location, old_style_course_id, new_style_package_id, 'problemabc', 'published', True)
- # look for w/ only the Location (works b/c there's only one possible course match): causes cache
- self.translate_n_check(location, None, new_style_package_id, 'problemabc', 'published', True)
+ location = Location(org, course, run, 'problem', problem_name)
+ new_offering = '{}.{}'.format(course, run)
+ self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
# create an entry w/o a guid name
- other_location = Location('i4x', org, course, 'chapter', 'intro')
- self.translate_n_check(other_location, old_style_course_id, new_style_package_id, 'intro', 'published', True)
+ other_location = Location(org, course, run, 'chapter', 'intro')
+ self.translate_n_check(other_location, org, new_offering, 'intro', 'published', True)
# add a distractor course
- delta_new_package_id = '{}.geek_dept.{}.{}'.format(org, course, 'delta_run')
- delta_course_locn = Location('i4x', org, course, 'course', 'delta_run')
+ delta_new_org = '{}.geek_dept'.format(org)
+ run = 'delta_run'
+ delta_new_offering = '{}.{}'.format(course, run)
+ delta_course_locn = SlashSeparatedCourseKey(org, course, run)
loc_mapper().create_map_entry(
delta_course_locn,
- delta_new_package_id,
+ delta_new_org, delta_new_offering,
block_map={problem_name: {'problem': 'problem3'}}
)
- self.translate_n_check(location, old_style_course_id, new_style_package_id, 'problemabc', 'published', True)
+ self.translate_n_check(location, org, new_offering, 'problemabc', 'published', True)
# add a new one to both courses (ensure name doesn't have same beginning)
new_prob_name = uuid.uuid4().hex
@@ -255,35 +245,11 @@ class TestLocationMapper(LocMapperSetupSansDjango):
new_prob_name = uuid.uuid4().hex
new_prob_locn = location.replace(name=new_prob_name)
new_usage_id = 'problem{}'.format(new_prob_name[:3])
- self.translate_n_check(new_prob_locn, old_style_course_id, new_style_package_id, new_usage_id, 'published', True)
+ self.translate_n_check(new_prob_locn, org, new_offering, new_usage_id, 'published', True)
+ new_prob_locn = new_prob_locn.replace(run=run)
self.translate_n_check(
- new_prob_locn, delta_course_locn.course_id, delta_new_package_id, new_usage_id, 'published', True
- )
- # look for w/ only the Location: causes caching and not unique; so, can't check which course
- prob_locator = loc_mapper().translate_location(
- None,
- new_prob_locn,
- add_entry_if_missing=True
+ new_prob_locn, delta_new_org, delta_new_offering, new_usage_id, 'published', True
)
- self.assertIsNotNone(prob_locator, "couldn't find ambiguous location")
-
- # add a default course pointing to the delta_run
- loc_mapper().create_map_entry(
- Location('i4x', org, course, 'problem', '789abc123efg456'),
- delta_new_package_id,
- block_map={problem_name: {'problem': 'problem3'}}
- )
- # now the ambiguous query should return delta
- again_prob_name = uuid.uuid4().hex
- while again_prob_name.startswith('abc') or again_prob_name.startswith(new_prob_name[:3]):
- again_prob_name = uuid.uuid4().hex
- again_prob_locn = location.replace(name=again_prob_name)
- again_usage_id = 'problem{}'.format(again_prob_name[:3])
- self.translate_n_check(again_prob_locn, old_style_course_id, new_style_package_id, again_usage_id, 'published', True)
- self.translate_n_check(
- again_prob_locn, delta_course_locn.course_id, delta_new_package_id, again_usage_id, 'published', True
- )
- self.translate_n_check(again_prob_locn, None, delta_new_package_id, again_usage_id, 'published', True)
def test_translate_locator(self):
"""
@@ -292,18 +258,23 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# lookup for non-existent course
org = 'foo_org'
course = 'bar_course'
- new_style_package_id = '{}.geek_dept.{}.baz_run'.format(org, course)
+ run = 'baz_run'
+ new_style_org = '{}.geek_dept'.format(org)
+ new_style_offering = '{}.{}'.format(course, run)
+ prob_course_key = CourseLocator(
+ org=new_style_org, offering=new_style_offering,
+ branch='published',
+ )
prob_locator = BlockUsageLocator(
- package_id=new_style_package_id,
+ prob_course_key,
block_id='problem2',
- branch='published'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
self.assertIsNone(prob_location, 'found entry in empty map table')
loc_mapper().create_map_entry(
- Location('i4x', org, course, 'course', 'baz_run'),
- new_style_package_id,
+ SlashSeparatedCourseKey(org, course, run),
+ new_style_org, new_style_offering,
block_map={
'abc123': {'problem': 'problem2'},
'48f23a10395384929234': {'chapter': 'chapter48f'},
@@ -313,74 +284,56 @@ class TestLocationMapper(LocMapperSetupSansDjango):
# only one course matches
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# default branch
- self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
+ self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
# test get_course keyword
prob_location = loc_mapper().translate_locator_to_location(prob_locator, get_course=True)
- self.assertEqual(prob_location, Location('i4x', org, course, 'course', 'baz_run', None))
+ self.assertEqual(prob_location, SlashSeparatedCourseKey(org, course, run))
# explicit branch
prob_locator = BlockUsageLocator(
- package_id=prob_locator.package_id, branch='draft', block_id=prob_locator.block_id
+ prob_course_key.for_branch('draft'), block_id=prob_locator.block_id
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
# Even though the problem was set as draft, we always return revision=None to work
# with old mongo/draft modulestores.
- self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
- prob_locator = BlockUsageLocator(
- package_id=new_style_package_id, block_id='problem2', branch='production'
- )
+ self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
+ prob_locator = BlockUsageLocator(prob_course_key.for_branch('production'), block_id='problem2')
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
- self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
+ self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
# same for chapter except chapter cannot be draft in old system
chap_locator = BlockUsageLocator(
- package_id=new_style_package_id,
+ prob_course_key.for_branch('production'),
block_id='chapter48f',
- branch='production'
)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
- self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
+ self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
# explicit branch
- chap_locator.branch = 'draft'
+ chap_locator = chap_locator.for_branch('draft')
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
- self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
+ self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
chap_locator = BlockUsageLocator(
- package_id=new_style_package_id, block_id='chapter48f', branch='production'
+ prob_course_key.for_branch('production'), block_id='chapter48f'
)
chap_location = loc_mapper().translate_locator_to_location(chap_locator)
- self.assertEqual(chap_location, Location('i4x', org, course, 'chapter', '48f23a10395384929234'))
+ self.assertEqual(chap_location, Location(org, course, run, 'chapter', '48f23a10395384929234'))
# look for non-existent problem
prob_locator2 = BlockUsageLocator(
- package_id=new_style_package_id,
- branch='draft',
+ prob_course_key.for_branch('draft'),
block_id='problem3'
)
prob_location = loc_mapper().translate_locator_to_location(prob_locator2)
self.assertIsNone(prob_location, 'Found non-existent problem')
# add a distractor course
- new_style_package_id = '{}.geek_dept.{}.{}'.format(org, course, 'delta_run')
- loc_mapper().create_map_entry(
- Location('i4x', org, course, 'course', 'delta_run'),
- new_style_package_id,
- block_map={'abc123': {'problem': 'problem3'}}
- )
- prob_location = loc_mapper().translate_locator_to_location(prob_locator)
- self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123', None))
-
- # add a default course pointing to the delta_run
+ delta_run = 'delta_run'
+ new_style_offering = '{}.{}'.format(course, delta_run)
loc_mapper().create_map_entry(
- Location('i4x', org, course, 'problem', '789abc123efg456'),
- new_style_package_id,
+ SlashSeparatedCourseKey(org, course, delta_run),
+ new_style_org, new_style_offering,
block_map={'abc123': {'problem': 'problem3'}}
)
- # now query delta (2 entries point to it)
- prob_locator = BlockUsageLocator(
- package_id=new_style_package_id,
- branch='production',
- block_id='problem3'
- )
prob_location = loc_mapper().translate_locator_to_location(prob_locator)
- self.assertEqual(prob_location, Location('i4x', org, course, 'problem', 'abc123'))
+ self.assertEqual(prob_location, Location(org, course, run, 'problem', 'abc123', None))
def test_special_chars(self):
"""
@@ -390,10 +343,8 @@ class TestLocationMapper(LocMapperSetupSansDjango):
org = 'foo.org.edu'
course = 'bar.course-4'
name = 'baz.run_4-3'
- old_style_course_id = '{}/{}/{}'.format(org, course, name)
- location = Location('i4x', org, course, 'course', name)
+ location = Location(org, course, name, 'course', name)
prob_locator = loc_mapper().translate_location(
- old_style_course_id,
location,
add_entry_if_missing=True
)
@@ -407,17 +358,17 @@ class TestLocationMapper(LocMapperSetupSansDjango):
org = "myorg"
course = "another_course"
name = "running_again"
- course_location = Location('i4x', org, course, 'course', name)
- course_xlate = loc_mapper().translate_location(None, course_location, add_entry_if_missing=True)
+ course_location = Location(org, course, name, 'course', name)
+ course_xlate = loc_mapper().translate_location(course_location, add_entry_if_missing=True)
self.assertEqual(course_location, loc_mapper().translate_locator_to_location(course_xlate))
eponymous_block = course_location.replace(category='chapter')
- chapter_xlate = loc_mapper().translate_location(None, eponymous_block, add_entry_if_missing=True)
+ chapter_xlate = loc_mapper().translate_location(eponymous_block, add_entry_if_missing=True)
self.assertEqual(course_location, loc_mapper().translate_locator_to_location(course_xlate))
self.assertEqual(eponymous_block, loc_mapper().translate_locator_to_location(chapter_xlate))
# and a non-existent one w/o add
eponymous_block = course_location.replace(category='problem')
with self.assertRaises(ItemNotFoundError):
- chapter_xlate = loc_mapper().translate_location(None, eponymous_block, add_entry_if_missing=False)
+ chapter_xlate = loc_mapper().translate_location(eponymous_block, add_entry_if_missing=False)
#==================================
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_locators.py b/common/lib/xmodule/xmodule/modulestore/tests/test_locators.py
index cc275f91a24..acd9c384962 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_locators.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_locators.py
@@ -3,12 +3,12 @@ Tests for xmodule.modulestore.locator.
"""
from unittest import TestCase
+import random
from bson.objectid import ObjectId
+from opaque_keys import InvalidKeyError
from xmodule.modulestore.locator import Locator, CourseLocator, BlockUsageLocator, DefinitionLocator
from xmodule.modulestore.parsers import BRANCH_PREFIX, BLOCK_PREFIX, VERSION_PREFIX
-from xmodule.modulestore.exceptions import InsufficientSpecificationError, OverSpecificationError
-from xmodule.modulestore import Location
-import random
+from ddt import ddt
class LocatorTest(TestCase):
@@ -19,232 +19,190 @@ class LocatorTest(TestCase):
def test_cant_instantiate_abstract_class(self):
self.assertRaises(TypeError, Locator)
- def test_course_constructor_overspecified(self):
- self.assertRaises(
- OverSpecificationError,
- CourseLocator,
- url='edx://mit.eecs.6002x',
- package_id='harvard.history',
- branch='published',
- version_guid=ObjectId())
- self.assertRaises(
- OverSpecificationError,
- CourseLocator,
- url='edx://mit.eecs.6002x',
- package_id='harvard.history',
- version_guid=ObjectId())
- self.assertRaises(
- OverSpecificationError,
- CourseLocator,
- url='edx://mit.eecs.6002x/' + BRANCH_PREFIX + 'published',
- branch='draft')
- self.assertRaises(
- OverSpecificationError,
- CourseLocator,
- package_id='mit.eecs.6002x/' + BRANCH_PREFIX + 'published',
- branch='draft')
-
def test_course_constructor_underspecified(self):
- self.assertRaises(InsufficientSpecificationError, CourseLocator)
- self.assertRaises(InsufficientSpecificationError, CourseLocator, branch='published')
+ with self.assertRaises(InvalidKeyError):
+ CourseLocator()
+ with self.assertRaises(InvalidKeyError):
+ CourseLocator(branch='published')
def test_course_constructor_bad_version_guid(self):
- self.assertRaises(ValueError, CourseLocator, version_guid="012345")
- self.assertRaises(InsufficientSpecificationError, CourseLocator, version_guid=None)
+ with self.assertRaises(ValueError):
+ CourseLocator(version_guid="012345")
+
+ with self.assertRaises(InvalidKeyError):
+ CourseLocator(version_guid=None)
def test_course_constructor_version_guid(self):
# generate a random location
test_id_1 = ObjectId()
test_id_1_loc = str(test_id_1)
testobj_1 = CourseLocator(version_guid=test_id_1)
- self.check_course_locn_fields(testobj_1, 'version_guid', version_guid=test_id_1)
+ self.check_course_locn_fields(testobj_1, version_guid=test_id_1)
self.assertEqual(str(testobj_1.version_guid), test_id_1_loc)
- self.assertEqual(str(testobj_1), VERSION_PREFIX + test_id_1_loc)
- self.assertEqual(testobj_1.url(), 'edx://' + VERSION_PREFIX + test_id_1_loc)
+ self.assertEqual(testobj_1._to_string(), VERSION_PREFIX + test_id_1_loc)
# Test using a given string
test_id_2_loc = '519665f6223ebd6980884f2b'
test_id_2 = ObjectId(test_id_2_loc)
testobj_2 = CourseLocator(version_guid=test_id_2)
- self.check_course_locn_fields(testobj_2, 'version_guid', version_guid=test_id_2)
+ self.check_course_locn_fields(testobj_2, version_guid=test_id_2)
self.assertEqual(str(testobj_2.version_guid), test_id_2_loc)
- self.assertEqual(str(testobj_2), VERSION_PREFIX + test_id_2_loc)
- self.assertEqual(testobj_2.url(), 'edx://' + VERSION_PREFIX + test_id_2_loc)
-
- def test_course_constructor_bad_package_id(self):
+ self.assertEqual(testobj_2._to_string(), VERSION_PREFIX + test_id_2_loc)
+
+ @ddt.data(
+ ' mit.eecs',
+ 'mit.eecs ',
+ VERSION_PREFIX + 'mit.eecs',
+ BLOCK_PREFIX + 'black/mit.eecs',
+ 'mit.ee cs',
+ 'mit.ee,cs',
+ 'mit.ee/cs',
+ 'mit.ee&cs',
+ 'mit.ee()cs',
+ BRANCH_PREFIX + 'this',
+ 'mit.eecs/' + BRANCH_PREFIX,
+ 'mit.eecs/' + BRANCH_PREFIX + 'this/' + BRANCH_PREFIX + 'that',
+ 'mit.eecs/' + BRANCH_PREFIX + 'this/' + BRANCH_PREFIX,
+ 'mit.eecs/' + BRANCH_PREFIX + 'this ',
+ 'mit.eecs/' + BRANCH_PREFIX + 'th%is ',
+ )
+ def test_course_constructor_bad_package_id(self, bad_id):
"""
Test all sorts of badly-formed package_ids (and urls with those package_ids)
"""
- for bad_id in (' mit.eecs',
- 'mit.eecs ',
- VERSION_PREFIX + 'mit.eecs',
- BLOCK_PREFIX + 'black/mit.eecs',
- 'mit.ee cs',
- 'mit.ee,cs',
- 'mit.ee/cs',
- 'mit.ee&cs',
- 'mit.ee()cs',
- BRANCH_PREFIX + 'this',
- 'mit.eecs/' + BRANCH_PREFIX,
- 'mit.eecs/' + BRANCH_PREFIX + 'this/' + BRANCH_PREFIX + 'that',
- 'mit.eecs/' + BRANCH_PREFIX + 'this/' + BRANCH_PREFIX,
- 'mit.eecs/' + BRANCH_PREFIX + 'this ',
- 'mit.eecs/' + BRANCH_PREFIX + 'th%is ',
- ):
- self.assertRaises(ValueError, CourseLocator, package_id=bad_id)
- self.assertRaises(ValueError, CourseLocator, url='edx://' + bad_id)
+ with self.assertRaises(InvalidKeyError):
+ CourseLocator(org=bad_id, offering='test')
- def test_course_constructor_bad_url(self):
- for bad_url in ('edx://',
- 'edx:/mit.eecs',
- 'http://mit.eecs',
- 'edx//mit.eecs'):
- self.assertRaises(ValueError, CourseLocator, url=bad_url)
+ with self.assertRaises(InvalidKeyError):
+ CourseLocator(org='test', offering=bad_id)
- def test_course_constructor_redundant_001(self):
- testurn = 'mit.eecs.6002x'
- testobj = CourseLocator(package_id=testurn, url='edx://' + testurn)
- self.check_course_locn_fields(testobj, 'package_id', package_id=testurn)
+ with self.assertRaises(InvalidKeyError):
+ CourseLocator.from_string('course-locator:' + bad_id)
- def test_course_constructor_redundant_002(self):
- testurn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published'
- expected_urn = 'mit.eecs.6002x'
- expected_rev = 'published'
- testobj = CourseLocator(package_id=testurn, url='edx://' + testurn)
- self.check_course_locn_fields(testobj, 'package_id',
- package_id=expected_urn,
- branch=expected_rev)
+ @ddt.data('course-locator:', 'course-locator:/mit.eecs', 'http:mit.eecs', 'course-locator//mit.eecs')
+ def test_course_constructor_bad_url(self, bad_url):
+ with self.assertRaises(InvalidKeyError):
+ CourseLocator.from_string(bad_url)
def test_course_constructor_url(self):
# Test parsing a url when it starts with a version ID and there is also a block ID.
# This hits the parsers parse_guid method.
test_id_loc = '519665f6223ebd6980884f2b'
- testobj = CourseLocator(url="edx://{}{}/{}hw3".format(VERSION_PREFIX, test_id_loc, BLOCK_PREFIX))
+ testobj = CourseLocator.from_string("course-locator:{}{}/{}hw3".format(VERSION_PREFIX, test_id_loc, BLOCK_PREFIX))
self.check_course_locn_fields(
testobj,
- 'test_block constructor',
version_guid=ObjectId(test_id_loc)
)
def test_course_constructor_url_package_id_and_version_guid(self):
test_id_loc = '519665f6223ebd6980884f2b'
- testobj = CourseLocator(url='edx://mit.eecs-honors.6002x/' + VERSION_PREFIX + test_id_loc)
- self.check_course_locn_fields(testobj, 'error parsing url with both course ID and version GUID',
- package_id='mit.eecs-honors.6002x',
- version_guid=ObjectId(test_id_loc))
+ testobj = CourseLocator.from_string('course-locator:mit.eecs+honors.6002x/' + VERSION_PREFIX + test_id_loc)
+ self.check_course_locn_fields(
+ testobj,
+ org='mit.eecs',
+ offering='honors.6002x',
+ version_guid=ObjectId(test_id_loc)
+ )
def test_course_constructor_url_package_id_branch_and_version_guid(self):
test_id_loc = '519665f6223ebd6980884f2b'
- testobj = CourseLocator(url='edx://mit.eecs.~6002x/' + BRANCH_PREFIX + 'draft-1/' + VERSION_PREFIX + test_id_loc)
- self.check_course_locn_fields(testobj, 'error parsing url with both course ID branch, and version GUID',
- package_id='mit.eecs.~6002x',
- branch='draft-1',
- version_guid=ObjectId(test_id_loc))
+ org = 'mit.eecs'
+ offering = '~6002x'
+ testobj = CourseLocator.from_string('course-locator:{}+{}/{}draft-1/{}{}'.format(
+ org, offering, BRANCH_PREFIX, VERSION_PREFIX, test_id_loc
+ ))
+ self.check_course_locn_fields(
+ testobj,
+ org=org,
+ offering=offering,
+ branch='draft-1',
+ version_guid=ObjectId(test_id_loc)
+ )
def test_course_constructor_package_id_no_branch(self):
- testurn = 'mit.eecs.6002x'
- testobj = CourseLocator(package_id=testurn)
- self.check_course_locn_fields(testobj, 'package_id', package_id=testurn)
+ org = 'mit.eecs'
+ offering = '6002x'
+ testurn = '{}+{}'.format(org, offering)
+ testobj = CourseLocator(org=org, offering=offering)
+ self.check_course_locn_fields(testobj, org=org, offering=offering)
self.assertEqual(testobj.package_id, testurn)
- self.assertEqual(str(testobj), testurn)
- self.assertEqual(testobj.url(), 'edx://' + testurn)
-
- def test_course_constructor_package_id_with_branch(self):
- testurn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published'
- expected_id = 'mit.eecs.6002x'
- expected_branch = 'published'
- testobj = CourseLocator(package_id=testurn)
- self.check_course_locn_fields(testobj, 'package_id with branch',
- package_id=expected_id,
- branch=expected_branch,
- )
- self.assertEqual(testobj.package_id, expected_id)
- self.assertEqual(testobj.branch, expected_branch)
- self.assertEqual(str(testobj), testurn)
- self.assertEqual(testobj.url(), 'edx://' + testurn)
+ self.assertEqual(testobj._to_string(), testurn)
def test_course_constructor_package_id_separate_branch(self):
- test_id = 'mit.eecs.6002x'
+ org = 'mit.eecs'
+ offering = '6002x'
+ testurn = '{}+{}'.format(org, offering)
test_branch = 'published'
- expected_urn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published'
- testobj = CourseLocator(package_id=test_id, branch=test_branch)
- self.check_course_locn_fields(testobj, 'package_id with separate branch',
- package_id=test_id,
- branch=test_branch,
- )
- self.assertEqual(testobj.package_id, test_id)
- self.assertEqual(testobj.branch, test_branch)
- self.assertEqual(str(testobj), expected_urn)
- self.assertEqual(testobj.url(), 'edx://' + expected_urn)
-
- def test_course_constructor_package_id_repeated_branch(self):
- """
- The same branch appears in the package_id and the branch field.
- """
- test_id = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published'
- test_branch = 'published'
- expected_id = 'mit.eecs.6002x'
- expected_urn = test_id
- testobj = CourseLocator(package_id=test_id, branch=test_branch)
- self.check_course_locn_fields(testobj, 'package_id with repeated branch',
- package_id=expected_id,
- branch=test_branch,
- )
- self.assertEqual(testobj.package_id, expected_id)
+ expected_urn = '{}+{}/{}{}'.format(org, offering, BRANCH_PREFIX, test_branch)
+ testobj = CourseLocator(org=org, offering=offering, branch=test_branch)
+ self.check_course_locn_fields(
+ testobj,
+ org=org,
+ offering=offering,
+ branch=test_branch,
+ )
+ self.assertEqual(testobj.package_id, testurn)
self.assertEqual(testobj.branch, test_branch)
- self.assertEqual(str(testobj), expected_urn)
- self.assertEqual(testobj.url(), 'edx://' + expected_urn)
+ self.assertEqual(testobj._to_string(), expected_urn)
def test_block_constructor(self):
- testurn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published/' + BLOCK_PREFIX + 'HW3'
- expected_id = 'mit.eecs.6002x'
+ expected_org = 'mit.eecs'
+ expected_offering = '6002x'
expected_branch = 'published'
expected_block_ref = 'HW3'
- testobj = BlockUsageLocator(url=testurn)
- self.check_block_locn_fields(testobj, 'test_block constructor',
- package_id=expected_id,
+ testurn = 'edx:{}+{}/{}{}/{}{}'.format(
+ expected_org, expected_offering, BRANCH_PREFIX, expected_branch, BLOCK_PREFIX, 'HW3'
+ )
+ testobj = BlockUsageLocator.from_string(testurn)
+ self.check_block_locn_fields(testobj,
+ org=expected_org,
+ offering=expected_offering,
branch=expected_branch,
block=expected_block_ref)
- self.assertEqual(str(testobj), testurn)
- self.assertEqual(testobj.url(), 'edx://' + testurn)
- testobj = BlockUsageLocator(url=testurn, version_guid=ObjectId())
+ self.assertEqual(unicode(testobj), testurn)
+ testobj = BlockUsageLocator(testobj.course_key.for_version(ObjectId()), testobj.block_id)
agnostic = testobj.version_agnostic()
self.assertIsNone(agnostic.version_guid)
- self.check_block_locn_fields(agnostic, 'test_block constructor',
- package_id=expected_id,
+ self.check_block_locn_fields(agnostic,
+ org=expected_org,
+ offering=expected_offering,
branch=expected_branch,
block=expected_block_ref)
def test_block_constructor_url_version_prefix(self):
test_id_loc = '519665f6223ebd6980884f2b'
- testobj = BlockUsageLocator(
- url='edx://mit.eecs.6002x/{}{}/{}lab2'.format(VERSION_PREFIX, test_id_loc, BLOCK_PREFIX)
+ testobj = BlockUsageLocator.from_string(
+ 'edx:mit.eecs+6002x/{}{}/{}lab2'.format(VERSION_PREFIX, test_id_loc, BLOCK_PREFIX)
)
self.check_block_locn_fields(
- testobj, 'error parsing URL with version and block',
- package_id='mit.eecs.6002x',
+ testobj,
+ org='mit.eecs',
+ offering='6002x',
block='lab2',
version_guid=ObjectId(test_id_loc)
)
agnostic = testobj.course_agnostic()
self.check_block_locn_fields(
- agnostic, 'error parsing URL with version and block',
+ agnostic,
block='lab2',
- package_id=None,
+ org=None,
+ offering=None,
version_guid=ObjectId(test_id_loc)
)
- self.assertIsNone(agnostic.package_id)
+ self.assertIsNone(agnostic.offering)
+ self.assertIsNone(agnostic.org)
def test_block_constructor_url_kitchen_sink(self):
test_id_loc = '519665f6223ebd6980884f2b'
- testobj = BlockUsageLocator(
- url='edx://mit.eecs.6002x/{}draft/{}{}/{}lab2'.format(
+ testobj = BlockUsageLocator.from_string(
+ 'edx:mit.eecs+6002x/{}draft/{}{}/{}lab2'.format(
BRANCH_PREFIX, VERSION_PREFIX, test_id_loc, BLOCK_PREFIX
)
)
self.check_block_locn_fields(
- testobj, 'error parsing URL with branch, version, and block',
- package_id='mit.eecs.6002x',
+ testobj,
+ org='mit.eecs',
+ offering='6002x',
branch='draft',
block='lab2',
version_guid=ObjectId(test_id_loc)
@@ -254,71 +212,50 @@ class LocatorTest(TestCase):
"""
It seems we used to use colons in names; so, ensure they're acceptable.
"""
- package_id = 'mit.eecs-1'
+ org = 'mit.eecs'
+ offering = '1'
branch = 'foo'
block_id = 'problem:with-colon~2'
- testobj = BlockUsageLocator(package_id=package_id, branch=branch, block_id=block_id)
- self.check_block_locn_fields(testobj, 'Cannot handle colon', package_id=package_id, branch=branch, block=block_id)
+ testobj = BlockUsageLocator(
+ CourseLocator(org=org, offering=offering, branch=branch),
+ block_id=block_id
+ )
+ self.check_block_locn_fields(
+ testobj, org=org, offering=offering, branch=branch, block=block_id
+ )
def test_relative(self):
"""
Test making a relative usage locator.
"""
- package_id = 'mit.eecs-1'
+ org = 'mit.eecs'
+ offering = '1'
branch = 'foo'
- baseobj = CourseLocator(package_id=package_id, branch=branch)
+ baseobj = CourseLocator(org=org, offering=offering, branch=branch)
block_id = 'problem:with-colon~2'
testobj = BlockUsageLocator.make_relative(baseobj, block_id)
self.check_block_locn_fields(
- testobj, 'Cannot make relative to course', package_id=package_id, branch=branch, block=block_id
+ testobj, org=org, offering=offering, branch=branch, block=block_id
)
block_id = 'completely_different'
testobj = BlockUsageLocator.make_relative(testobj, block_id)
self.check_block_locn_fields(
- testobj, 'Cannot make relative to block usage', package_id=package_id, branch=branch, block=block_id
+ testobj, org=org, offering=offering, branch=branch, block=block_id
)
def test_repr(self):
- testurn = 'mit.eecs.6002x/' + BRANCH_PREFIX + 'published/' + BLOCK_PREFIX + 'HW3'
- testobj = BlockUsageLocator(package_id=testurn)
- self.assertEqual('BlockUsageLocator("mit.eecs.6002x/branch/published/block/HW3")', repr(testobj))
-
- def test_old_location_helpers(self):
- """
- Test the functions intended to help with the conversion from old locations to locators
- """
- location_tuple = ('i4x', 'mit', 'eecs.6002x', 'course', 't3_2013')
- location = Location(location_tuple)
- self.assertEqual(location, Locator.to_locator_or_location(location))
- self.assertEqual(location, Locator.to_locator_or_location(location_tuple))
- self.assertEqual(location, Locator.to_locator_or_location(list(location_tuple)))
- self.assertEqual(location, Locator.to_locator_or_location(location.dict()))
-
- locator = BlockUsageLocator(package_id='foo.bar', branch='alpha', block_id='deep')
- self.assertEqual(locator, Locator.to_locator_or_location(locator))
- self.assertEqual(locator.as_course_locator(), Locator.to_locator_or_location(locator.as_course_locator()))
- self.assertEqual(location, Locator.to_locator_or_location(location.url()))
- self.assertEqual(locator, Locator.to_locator_or_location(locator.url()))
- self.assertEqual(locator, Locator.to_locator_or_location(locator.__dict__))
-
- asset_location = Location(['c4x', 'mit', 'eecs.6002x', 'asset', 'selfie.jpeg'])
- self.assertEqual(asset_location, Locator.to_locator_or_location(asset_location))
- self.assertEqual(asset_location, Locator.to_locator_or_location(asset_location.url()))
-
- def_location_url = "defx://version/" + '{:024x}'.format(random.randrange(16 ** 24))
- self.assertEqual(DefinitionLocator(def_location_url), Locator.to_locator_or_location(def_location_url))
-
- with self.assertRaises(ValueError):
- Locator.to_locator_or_location(22)
- with self.assertRaises(ValueError):
- Locator.to_locator_or_location("hello.world.not.a.url")
- self.assertIsNone(Locator.parse_url("unknown://foo.bar/baz"))
+ testurn = 'edx:mit.eecs+6002x/' + BRANCH_PREFIX + 'published/' + BLOCK_PREFIX + 'HW3'
+ testobj = BlockUsageLocator.from_string(testurn)
+ self.assertEqual("BlockUsageLocator(CourseLocator(u'mit.eecs', u'6002x', u'published', None), u'HW3')", repr(testobj))
def test_url_reverse(self):
"""
Test the url_reverse method
"""
- locator = CourseLocator(package_id="a.fancy_course-id", branch="branch_1.2-3")
+ locator = BlockUsageLocator(
+ CourseLocator(org="a", offering="fancy_course-id", branch="branch_1.2-3"),
+ block_id='element'
+ )
self.assertEqual(
'/expression/{}/format'.format(unicode(locator)),
locator.url_reverse('expression', 'format')
@@ -339,8 +276,7 @@ class LocatorTest(TestCase):
def test_description_locator_url(self):
object_id = '{:024x}'.format(random.randrange(16 ** 24))
definition_locator = DefinitionLocator(object_id)
- self.assertEqual('defx://' + VERSION_PREFIX + object_id, definition_locator.url())
- self.assertEqual(definition_locator, DefinitionLocator(definition_locator.url()))
+ self.assertEqual('defx:' + VERSION_PREFIX + object_id, unicode(definition_locator))
def test_description_locator_version(self):
object_id = '{:024x}'.format(random.randrange(16 ** 24))
@@ -350,20 +286,21 @@ class LocatorTest(TestCase):
# ------------------------------------------------------------------
# Utilities
- def check_course_locn_fields(self, testobj, msg, version_guid=None,
- package_id=None, branch=None):
+ def check_course_locn_fields(self, testobj, version_guid=None,
+ org=None, offering=None, branch=None):
"""
- Checks the version, package_id, and branch in testobj
+ Checks the version, org, offering, and branch in testobj
"""
- self.assertEqual(testobj.version_guid, version_guid, msg)
- self.assertEqual(testobj.package_id, package_id, msg)
- self.assertEqual(testobj.branch, branch, msg)
+ self.assertEqual(testobj.version_guid, version_guid)
+ self.assertEqual(testobj.org, org)
+ self.assertEqual(testobj.offering, offering)
+ self.assertEqual(testobj.branch, branch)
- def check_block_locn_fields(self, testobj, msg, version_guid=None,
- package_id=None, branch=None, block=None):
+ def check_block_locn_fields(self, testobj, version_guid=None,
+ org=None, offering=None, branch=None, block=None):
"""
Does adds a block id check over and above the check_course_locn_fields tests
"""
- self.check_course_locn_fields(testobj, msg, version_guid, package_id,
+ self.check_course_locn_fields(testobj, version_guid, org, offering,
branch)
self.assertEqual(testobj.block_id, block)
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
index 1bc868f9823..ca4daecc18c 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
@@ -14,6 +14,8 @@ from xmodule.modulestore.tests.test_location_mapper import LocMapperSetupSansDja
# Mixed modulestore depends on django, so we'll manually configure some django settings
# before importing the module
from django.conf import settings
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
+import bson.son
if not settings.configured:
settings.configure()
from xmodule.modulestore.mixed import MixedModuleStore
@@ -83,7 +85,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
AssertEqual replacement for CourseLocator
"""
- if not (loc1.package_id == loc2.package_id and loc1.branch == loc2.branch and loc1.block_id == loc2.block_id):
+ if loc1.version_agnostic() != loc2.version_agnostic():
self.fail(self._formatMessage(msg, u"{} != {}".format(unicode(loc1), unicode(loc2))))
def setUp(self):
@@ -95,6 +97,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
host=self.HOST,
port=self.PORT,
tz_aware=True,
+ document_class=bson.son.SON,
)
self.connection.drop_database(self.DB)
self.addCleanup(self.connection.drop_database, self.DB)
@@ -109,29 +112,40 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
patcher.start()
self.addCleanup(patcher.stop)
self.addTypeEqualityFunc(BlockUsageLocator, '_compareIgnoreVersion')
+ self.addTypeEqualityFunc(CourseLocator, '_compareIgnoreVersion')
# define attrs which get set in initdb to quell pylint
self.import_chapter_location = self.store = self.fake_location = self.xml_chapter_location = None
self.course_locations = []
# pylint: disable=invalid-name
- def _create_course(self, default, course_id):
+ def _create_course(self, default, course_key):
"""
Create a course w/ one item in the persistence store using the given course & item location.
"""
- course = self.store.create_course(course_id, store_name=default)
+ if default == 'split':
+ offering = course_key.offering.replace('/', '.')
+ else:
+ offering = course_key.offering
+ course = self.store.create_course(course_key.org, offering, store_name=default)
category = self.import_chapter_location.category
block_id = self.import_chapter_location.name
chapter = self.store.create_item(
# don't use course_location as it may not be the repr
course.location, category, location=self.import_chapter_location, block_id=block_id
)
- if isinstance(course.location, CourseLocator):
+ if isinstance(course.id, CourseLocator):
self.course_locations[self.MONGO_COURSEID] = course.location.version_agnostic()
self.import_chapter_location = chapter.location.version_agnostic()
else:
- self.assertEqual(course.location.course_id, course_id)
+ self.assertEqual(course.id, course_key)
self.assertEqual(chapter.location, self.import_chapter_location)
+ def _course_key_from_string(self, string):
+ """
+ Get the course key for the given course string
+ """
+ return self.course_locations[string].course_key
+
def initdb(self, default):
"""
Initialize the database and create one test course in it
@@ -141,11 +155,17 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.store = MixedModuleStore(**self.options)
self.addCleanup(self.store.close_all_connections)
+ # convert to CourseKeys
self.course_locations = {
- course_id: generate_location(course_id)
+ course_id: SlashSeparatedCourseKey.from_deprecated_string(course_id)
for course_id in [self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
- self.fake_location = Location('i4x', 'foo', 'bar', 'vertical', 'baz')
+ # and then to the root UsageKey
+ self.course_locations = {
+ course_id: course_key.make_usage_key('course', course_key.run)
+ for course_id, course_key in self.course_locations.iteritems() # pylint: disable=maybe-no-member
+ }
+ self.fake_location = Location('foo', 'bar', 'slowly', 'vertical', 'baz')
self.import_chapter_location = self.course_locations[self.MONGO_COURSEID].replace(
category='chapter', name='Overview'
)
@@ -154,9 +174,9 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
)
# get Locators and set up the loc mapper if app is Locator based
if default == 'split':
- self.fake_location = loc_mapper().translate_location('foo/bar/2012_Fall', self.fake_location)
+ self.fake_location = loc_mapper().translate_location(self.fake_location)
- self._create_course(default, self.MONGO_COURSEID)
+ self._create_course(default, self.course_locations[self.MONGO_COURSEID].course_key)
@ddt.data('direct', 'split')
def test_get_modulestore_type(self, default_ms):
@@ -164,57 +184,54 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
Make sure we get back the store type we expect for given mappings
"""
self.initdb(default_ms)
- self.assertEqual(self.store.get_modulestore_type(self.XML_COURSEID1), XML_MODULESTORE_TYPE)
- self.assertEqual(self.store.get_modulestore_type(self.XML_COURSEID2), XML_MODULESTORE_TYPE)
+ self.assertEqual(self.store.get_modulestore_type(
+ self._course_key_from_string(self.XML_COURSEID1)), XML_MODULESTORE_TYPE
+ )
+ self.assertEqual(self.store.get_modulestore_type(
+ self._course_key_from_string(self.XML_COURSEID2)), XML_MODULESTORE_TYPE
+ )
mongo_ms_type = MONGO_MODULESTORE_TYPE if default_ms == 'direct' else SPLIT_MONGO_MODULESTORE_TYPE
- self.assertEqual(self.store.get_modulestore_type(self.MONGO_COURSEID), mongo_ms_type)
+ self.assertEqual(self.store.get_modulestore_type(
+ self._course_key_from_string(self.MONGO_COURSEID)), mongo_ms_type
+ )
# try an unknown mapping, it should be the 'default' store
- self.assertEqual(self.store.get_modulestore_type('foo/bar/2012_Fall'), mongo_ms_type)
+ self.assertEqual(self.store.get_modulestore_type(
+ SlashSeparatedCourseKey('foo', 'bar', '2012_Fall')), mongo_ms_type
+ )
@ddt.data('direct', 'split')
def test_has_item(self, default_ms):
self.initdb(default_ms)
- for course_id, course_locn in self.course_locations.iteritems():
- self.assertTrue(self.store.has_item(course_id, course_locn))
+ for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
+ self.assertTrue(self.store.has_item(course_locn))
# try negative cases
self.assertFalse(self.store.has_item(
- self.XML_COURSEID1,
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
))
- self.assertFalse(self.store.has_item(self.MONGO_COURSEID, self.fake_location))
+ self.assertFalse(self.store.has_item(self.fake_location))
@ddt.data('direct', 'split')
def test_get_item(self, default_ms):
self.initdb(default_ms)
- with self.assertRaises(NotImplementedError):
- self.store.get_item(self.fake_location)
-
- @ddt.data('direct', 'split')
- def test_get_instance(self, default_ms):
- self.initdb(default_ms)
- for course_id, course_locn in self.course_locations.iteritems():
- self.assertIsNotNone(self.store.get_instance(course_id, course_locn))
+ for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
+ self.assertIsNotNone(self.store.get_item(course_locn))
# try negative cases
with self.assertRaises(ItemNotFoundError):
- self.store.get_instance(
- self.XML_COURSEID1,
+ self.store.get_item(
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
)
with self.assertRaises(ItemNotFoundError):
- self.store.get_instance(self.MONGO_COURSEID, self.fake_location)
+ self.store.get_item(self.fake_location)
@ddt.data('direct', 'split')
def test_get_items(self, default_ms):
self.initdb(default_ms)
- for course_id, course_locn in self.course_locations.iteritems():
- if hasattr(course_locn, 'as_course_locator'):
- locn = course_locn.as_course_locator()
- else:
- locn = course_locn.replace(org=None, course=None, name=None)
+ for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
+ locn = course_locn.course_key
# NOTE: use get_course if you just want the course. get_items is expensive
- modules = self.store.get_items(locn, course_id, qualifiers={'category': 'course'})
+ modules = self.store.get_items(locn, category='course')
self.assertEqual(len(modules), 1)
self.assertEqual(modules[0].location, course_locn)
@@ -224,25 +241,19 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
Update should fail for r/o dbs and succeed for r/w ones
"""
self.initdb(default_ms)
- course_id = self.XML_COURSEID1
- course = self.store.get_course(course_id)
+ course = self.store.get_course(self.course_locations[self.XML_COURSEID1].course_key)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
- with self.assertRaises(NotImplementedError):
+ with self.assertRaises(AttributeError): # ensure it doesn't allow writing
self.store.update_item(course, None)
# now do it for a r/w db
- # get_course api's are inconsistent: one takes Locators the other an old style course id
- if hasattr(self.course_locations[self.MONGO_COURSEID], 'as_course_locator'):
- locn = self.course_locations[self.MONGO_COURSEID]
- else:
- locn = self.MONGO_COURSEID
- course = self.store.get_course(locn)
+ course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
self.store.update_item(course, None)
- course = self.store.get_course(locn)
+ course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertTrue(course.show_calculator)
@ddt.data('direct', 'split')
@@ -251,13 +262,13 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
Delete should reject on r/o db and work on r/w one
"""
self.initdb(default_ms)
- # r/o try deleting the course
- with self.assertRaises(NotImplementedError):
+ # r/o try deleting the course (is here to ensure it can't be deleted)
+ with self.assertRaises(AttributeError):
self.store.delete_item(self.xml_chapter_location)
self.store.delete_item(self.import_chapter_location, '**replace_user**')
# verify it's gone
with self.assertRaises(ItemNotFoundError):
- self.store.get_instance(self.MONGO_COURSEID, self.import_chapter_location)
+ self.store.get_item(self.import_chapter_location)
@ddt.data('direct', 'split')
def test_get_courses(self, default_ms):
@@ -281,9 +292,9 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.initdb('direct')
courses = self.store.modulestores['xml'].get_courses()
self.assertEqual(len(courses), 2)
- course_ids = [course.location.course_id for course in courses]
- self.assertIn(self.XML_COURSEID1, course_ids)
- self.assertIn(self.XML_COURSEID2, course_ids)
+ course_ids = [course.id for course in courses]
+ self.assertIn(self.course_locations[self.XML_COURSEID1].course_key, course_ids)
+ self.assertIn(self.course_locations[self.XML_COURSEID2].course_key, course_ids)
# this course is in the directory from which we loaded courses but not in the map
self.assertNotIn("edX/toy/TT_2012_Fall", course_ids)
@@ -293,35 +304,25 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
self.initdb('direct')
with self.assertRaises(NotImplementedError):
- self.store.create_course("org/course/run", store_name='xml')
+ self.store.create_course("org", "course/run", store_name='xml')
@ddt.data('direct', 'split')
def test_get_course(self, default_ms):
self.initdb(default_ms)
- for course_locn in self.course_locations.itervalues():
- if hasattr(course_locn, 'as_course_locator'):
- locn = course_locn.as_course_locator()
- else:
- locn = course_locn.course_id
+ for course_location in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
# NOTE: use get_course if you just want the course. get_items is expensive
- course = self.store.get_course(locn)
+ course = self.store.get_course(course_location.course_key)
self.assertIsNotNone(course)
- self.assertEqual(course.location, course_locn)
+ self.assertEqual(course.id, course_location.course_key)
@ddt.data('direct', 'split')
def test_get_parent_locations(self, default_ms):
self.initdb(default_ms)
- parents = self.store.get_parent_locations(
- self.import_chapter_location,
- self.MONGO_COURSEID
- )
+ parents = self.store.get_parent_locations(self.import_chapter_location)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0], self.course_locations[self.MONGO_COURSEID])
- parents = self.store.get_parent_locations(
- self.xml_chapter_location,
- self.XML_COURSEID1
- )
+ parents = self.store.get_parent_locations(self.xml_chapter_location)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0], self.course_locations[self.XML_COURSEID1])
@@ -329,33 +330,13 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
def test_get_orphans(self, default_ms):
self.initdb(default_ms)
# create an orphan
- if default_ms == 'split':
- course_id = self.course_locations[self.MONGO_COURSEID].as_course_locator()
- branch = course_id.branch
- else:
- course_id = self.MONGO_COURSEID
- branch = None
+ course_id = self.course_locations[self.MONGO_COURSEID].course_key
orphan = self.store.create_item(course_id, 'problem', block_id='orphan')
- found_orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID], branch)
+ found_orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
if default_ms == 'split':
self.assertEqual(found_orphans, [orphan.location.version_agnostic()])
else:
- self.assertEqual(found_orphans, [unicode(orphan.location)])
-
- @ddt.data('split')
- def test_create_item_from_course_id(self, default_ms):
- """
- Test code paths missed by the above:
- * passing an old-style course_id which has a loc map to split's create_item
- """
- self.initdb(default_ms)
- # create loc_map entry
- loc_mapper().translate_location(self.MONGO_COURSEID, generate_location(self.MONGO_COURSEID))
- orphan = self.store.create_item(self.MONGO_COURSEID, 'problem', block_id='orphan')
- self.assertEqual(
- orphan.location.version_agnostic().as_course_locator(),
- self.course_locations[self.MONGO_COURSEID].as_course_locator()
- )
+ self.assertEqual(found_orphans, [orphan.location.to_deprecated_string()])
@ddt.data('direct')
def test_create_item_from_parent_location(self, default_ms):
@@ -365,7 +346,7 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
self.initdb(default_ms)
self.store.create_item(self.course_locations[self.MONGO_COURSEID], 'problem', block_id='orphan')
- orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID], None)
+ orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertEqual(len(orphans), 0, "unexpected orphans: {}".format(orphans))
@ddt.data('direct')
@@ -376,11 +357,11 @@ class TestMixedModuleStore(LocMapperSetupSansDjango):
self.initdb(default_ms)
course_locations = self.store.get_courses_for_wiki('toy')
self.assertEqual(len(course_locations), 1)
- self.assertIn(Location('i4x', 'edX', 'toy', 'course', '2012_Fall'), course_locations)
+ self.assertIn(self.course_locations[self.XML_COURSEID1], course_locations)
course_locations = self.store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 1)
- self.assertIn(Location('i4x', 'edX', 'simple', 'course', '2012_Fall'), course_locations)
+ self.assertIn(self.course_locations[self.XML_COURSEID2], course_locations)
self.assertEqual(len(self.store.get_courses_for_wiki('edX.simple.2012_Fall')), 0)
self.assertEqual(len(self.store.get_courses_for_wiki('no_such_wiki')), 0)
@@ -413,13 +394,3 @@ def create_modulestore_instance(engine, doc_store_config, options, i18n_service=
doc_store_config=doc_store_config,
**options
)
-
-
-def generate_location(course_id):
- """
- Generate the locations for the given ids
- """
- course_dict = Location.parse_course_id(course_id)
- course_dict['tag'] = 'i4x'
- course_dict['category'] = 'course'
- return Location(course_dict)
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
index b737dd653bf..b53a081e50b 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
@@ -1,26 +1,70 @@
-from nose.tools import assert_equals, assert_raises # pylint: disable=E0611
+from nose.tools import assert_equals, assert_raises, assert_true, assert_false # pylint: disable=E0611
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.search import path_to_location
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
+
def check_path_to_location(modulestore):
"""
Make sure that path_to_location works: should be passed a modulestore
with the toy and simple courses loaded.
"""
+ course_id = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")
+
should_work = (
- ("i4x://edX/toy/video/Welcome",
- ("edX/toy/2012_Fall", "Overview", "Welcome", None)),
- ("i4x://edX/toy/chapter/Overview",
- ("edX/toy/2012_Fall", "Overview", None, None)),
+ (course_id.make_usage_key('video', 'Welcome'),
+ (course_id, "Overview", "Welcome", None)),
+ (course_id.make_usage_key('chapter', 'Overview'),
+ (course_id, "Overview", None, None)),
)
- course_id = "edX/toy/2012_Fall"
for location, expected in should_work:
- assert_equals(path_to_location(modulestore, course_id, location), expected)
+ assert_equals(path_to_location(modulestore, location), expected)
not_found = (
- "i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome"
+ course_id.make_usage_key('video', 'WelcomeX'),
+ course_id.make_usage_key('course', 'NotHome'),
)
for location in not_found:
- assert_raises(ItemNotFoundError, path_to_location, modulestore, course_id, location)
+ with assert_raises(ItemNotFoundError):
+ path_to_location(modulestore, location)
+
+
+def check_has_course_method(modulestore, locator, locator_key_fields):
+ error_message = "Called has_course with query {0} and ignore_case is {1}."
+
+ for ignore_case in [True, False]:
+
+ # should find the course with exact locator
+ assert_true(modulestore.has_course(locator, ignore_case))
+
+ for key_field in locator_key_fields:
+ locator_changes_that_should_not_be_found = [ # pylint: disable=invalid-name
+ # replace value for one of the keys
+ {key_field: 'fake'},
+ # add a character at the end
+ {key_field: getattr(locator, key_field) + 'X'},
+ # add a character in the beginning
+ {key_field: 'X' + getattr(locator, key_field)},
+ ]
+ for changes in locator_changes_that_should_not_be_found:
+ search_locator = locator.replace(**changes)
+ assert_false(
+ modulestore.has_course(search_locator),
+ error_message.format(search_locator, ignore_case)
+ )
+
+ # test case [in]sensitivity
+ locator_case_changes = [
+ {key_field: getattr(locator, key_field).upper()},
+ {key_field: getattr(locator, key_field).capitalize()},
+ {key_field: getattr(locator, key_field).capitalize().swapcase()},
+ ]
+ for changes in locator_case_changes:
+ search_locator = locator.replace(**changes)
+ assert_equals(
+ modulestore.has_course(search_locator, ignore_case),
+ ignore_case,
+ error_message.format(search_locator, ignore_case)
+ )
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
index 0663f9fdea9..28c521d35fb 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
@@ -1,28 +1,33 @@
-from pprint import pprint
# pylint: disable=E0611
from nose.tools import assert_equals, assert_raises, \
- assert_not_equals, assert_false
-from itertools import ifilter
+ assert_not_equals, assert_false, assert_true, assert_greater, assert_is_instance
# pylint: enable=E0611
import pymongo
import logging
from uuid import uuid4
+import unittest
+import bson.son
+from xblock.core import XBlock
-from xblock.fields import Scope
+from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xblock.runtime import KeyValueStore
from xblock.exceptions import InvalidScopeError
+from xblock.plugin import Plugin
from xmodule.tests import DATA_DIR
from xmodule.modulestore import Location, MONGO_MODULESTORE_TYPE
from xmodule.modulestore.mongo import MongoModuleStore, MongoKeyValueStore
from xmodule.modulestore.draft import DraftModuleStore
+from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
from xmodule.contentstore.mongo import MongoContentStore
from xmodule.modulestore.tests.test_modulestore import check_path_to_location
from nose.tools import assert_in
from xmodule.exceptions import NotFoundError
-from xmodule.modulestore.exceptions import InsufficientSpecificationError
+from git.test.lib.asserts import assert_not_none
+from xmodule.x_module import XModuleMixin
+
log = logging.getLogger(__name__)
@@ -35,7 +40,17 @@ DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
-class TestMongoModuleStore(object):
+class ReferenceTestXBlock(XBlock):
+ """
+ Test xblock type to test the reference field types
+ """
+ has_children = True
+ reference_link = Reference(default=None, scope=Scope.content)
+ reference_list = ReferenceList(scope=Scope.content)
+ reference_dict = ReferenceValueDict(scope=Scope.settings)
+
+
+class TestMongoModuleStore(unittest.TestCase):
'''Tests!'''
# Explicitly list the courses to load (don't want the big one)
courses = ['toy', 'simple', 'simple_with_draft', 'test_unicode']
@@ -46,6 +61,7 @@ class TestMongoModuleStore(object):
host=HOST,
port=PORT,
tz_aware=True,
+ document_class=bson.son.SON,
)
cls.connection.drop_database(DB)
@@ -57,6 +73,7 @@ class TestMongoModuleStore(object):
@classmethod
def teardownClass(cls):
+# cls.patcher.stop()
if cls.connection:
cls.connection.drop_database(DB)
cls.connection.close()
@@ -69,7 +86,10 @@ class TestMongoModuleStore(object):
'db': DB,
'collection': COLLECTION,
}
- store = MongoModuleStore(doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
+ store = MongoModuleStore(
+ doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS,
+ xblock_mixins=(XModuleMixin,)
+ )
# since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
# as well
content_store = MongoContentStore(HOST, DB)
@@ -103,25 +123,10 @@ class TestMongoModuleStore(object):
def tearDown(self):
pass
- def get_course_by_id(self, name):
- """
- Returns the first course with `id` of `name`, or `None` if there are none.
- """
- courses = self.store.get_courses()
- return next(ifilter(lambda x: x.id == name, courses), None)
-
- def course_with_id_exists(self, name):
- """
- Returns true iff there exists some course with `id` of `name`.
- """
- return (self.get_course_by_id(name) is not None)
-
def test_init(self):
- '''Make sure the db loads, and print all the locations in the db.
- Call this directly from failing tests to see what is loaded'''
+ '''Make sure the db loads'''
ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True}))
-
- pprint([Location(i['_id']).url() for i in ids])
+ assert_greater(len(ids), 12)
def test_mongo_modulestore_type(self):
store = MongoModuleStore(
@@ -134,53 +139,64 @@ class TestMongoModuleStore(object):
'''Make sure the course objects loaded properly'''
courses = self.store.get_courses()
assert_equals(len(courses), 5)
- assert self.course_with_id_exists('edX/simple/2012_Fall')
- assert self.course_with_id_exists('edX/simple_with_draft/2012_Fall')
- assert self.course_with_id_exists('edX/test_import_course/2012_Fall')
- assert self.course_with_id_exists('edX/test_unicode/2012_Fall')
- assert self.course_with_id_exists('edX/toy/2012_Fall')
+ course_ids = [course.id for course in courses]
+ for course_key in [
+
+ SlashSeparatedCourseKey(*fields)
+ for fields in [
+ ['edX', 'simple', '2012_Fall'], ['edX', 'simple_with_draft', '2012_Fall'],
+ ['edX', 'test_import_course', '2012_Fall'], ['edX', 'test_unicode', '2012_Fall'],
+ ['edX', 'toy', '2012_Fall']
+ ]
+ ]:
+ assert_in(course_key, course_ids)
+ course = self.store.get_course(course_key)
+ assert_not_none(course)
def test_loads(self):
- assert_not_equals(
- self.store.get_item("i4x://edX/toy/course/2012_Fall"),
- None)
+ assert_not_none(
+ self.store.get_item(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall'))
+ )
- assert_not_equals(
- self.store.get_item("i4x://edX/simple/course/2012_Fall"),
- None)
+ assert_not_none(
+ self.store.get_item(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall')),
+ )
- assert_not_equals(
- self.store.get_item("i4x://edX/toy/video/Welcome"),
- None)
+ assert_not_none(
+ self.store.get_item(Location('edX', 'toy', '2012_Fall', 'video', 'Welcome')),
+ )
def test_unicode_loads(self):
- assert_not_equals(
- self.store.get_item("i4x://edX/test_unicode/course/2012_Fall"),
- None)
+ """
+ Test that getting items from the test_unicode course works
+ """
+ assert_not_none(
+ self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'course', '2012_Fall')),
+ )
# All items with ascii-only filenames should load properly.
- assert_not_equals(
- self.store.get_item("i4x://edX/test_unicode/video/Welcome"),
- None)
- assert_not_equals(
- self.store.get_item("i4x://edX/test_unicode/video/Welcome"),
- None)
- assert_not_equals(
- self.store.get_item("i4x://edX/test_unicode/chapter/Overview"),
- None)
+ assert_not_none(
+ self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'video', 'Welcome')),
+ )
+ assert_not_none(
+ self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'video', 'Welcome')),
+ )
+ assert_not_none(
+ self.store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'chapter', 'Overview')),
+ )
def test_find_one(self):
- assert_not_equals(
- self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")),
- None)
+ assert_not_none(
+ self.store._find_one(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall')),
+ )
- assert_not_equals(
- self.store._find_one(Location("i4x://edX/simple/course/2012_Fall")),
- None)
+ assert_not_none(
+ self.store._find_one(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall')),
+ )
- assert_not_equals(
- self.store._find_one(Location("i4x://edX/toy/video/Welcome")),
- None)
+ assert_not_none(
+ self.store._find_one(Location('edX', 'toy', '2012_Fall', 'video', 'Welcome')),
+ )
def test_path_to_location(self):
'''Make sure that path_to_location works'''
@@ -209,7 +225,7 @@ class TestMongoModuleStore(object):
Assumes the information is desired for courses[4] ('toy' course).
"""
- course = self.get_course_by_id('edX/toy/2012_Fall')
+ course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
return course.tabs[index]['name']
# There was a bug where model.save was not getting called after the static tab name
@@ -224,29 +240,32 @@ class TestMongoModuleStore(object):
"""
Test getting, setting, and defaulting the locked attr and arbitrary attrs.
"""
- location = Location('i4x', 'edX', 'toy', 'course', '2012_Fall')
- course_content, __ = TestMongoModuleStore.content_store.get_all_content_for_course(location)
- assert len(course_content) > 0
+ location = Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall')
+ course_content, __ = TestMongoModuleStore.content_store.get_all_content_for_course(location.course_key)
+ assert_true(len(course_content) > 0)
# a bit overkill, could just do for content[0]
for content in course_content:
assert not content.get('locked', False)
- assert not TestMongoModuleStore.content_store.get_attr(content['_id'], 'locked', False)
- attrs = TestMongoModuleStore.content_store.get_attrs(content['_id'])
+ asset_key = AssetLocation._from_deprecated_son(content['_id'], location.run)
+ assert not TestMongoModuleStore.content_store.get_attr(asset_key, 'locked', False)
+ attrs = TestMongoModuleStore.content_store.get_attrs(asset_key)
assert_in('uploadDate', attrs)
assert not attrs.get('locked', False)
- TestMongoModuleStore.content_store.set_attr(content['_id'], 'locked', True)
- assert TestMongoModuleStore.content_store.get_attr(content['_id'], 'locked', False)
- attrs = TestMongoModuleStore.content_store.get_attrs(content['_id'])
+ TestMongoModuleStore.content_store.set_attr(asset_key, 'locked', True)
+ assert TestMongoModuleStore.content_store.get_attr(asset_key, 'locked', False)
+ attrs = TestMongoModuleStore.content_store.get_attrs(asset_key)
assert_in('locked', attrs)
assert attrs['locked'] is True
- TestMongoModuleStore.content_store.set_attrs(content['_id'], {'miscel': 99})
- assert_equals(TestMongoModuleStore.content_store.get_attr(content['_id'], 'miscel'), 99)
+ TestMongoModuleStore.content_store.set_attrs(asset_key, {'miscel': 99})
+ assert_equals(TestMongoModuleStore.content_store.get_attr(asset_key, 'miscel'), 99)
+
+ asset_key = AssetLocation._from_deprecated_son(course_content[0]['_id'], location.run)
assert_raises(
- AttributeError, TestMongoModuleStore.content_store.set_attr, course_content[0]['_id'],
+ AttributeError, TestMongoModuleStore.content_store.set_attr, asset_key,
'md5', 'ff1532598830e3feac91c2449eaa60d6'
)
assert_raises(
- AttributeError, TestMongoModuleStore.content_store.set_attrs, course_content[0]['_id'],
+ AttributeError, TestMongoModuleStore.content_store.set_attrs, asset_key,
{'foo': 9, 'md5': 'ff1532598830e3feac91c2449eaa60d6'}
)
assert_raises(
@@ -269,7 +288,7 @@ class TestMongoModuleStore(object):
{'displayname': 'hello'}
)
assert_raises(
- InsufficientSpecificationError, TestMongoModuleStore.content_store.set_attrs,
+ NotFoundError, TestMongoModuleStore.content_store.set_attrs,
Location('bogus', 'bogus', 'bogus', 'asset', None),
{'displayname': 'hello'}
)
@@ -281,13 +300,13 @@ class TestMongoModuleStore(object):
for course_number in self.courses:
course_locations = self.store.get_courses_for_wiki(course_number)
assert_equals(len(course_locations), 1)
- assert_equals(Location('i4x', 'edX', course_number, 'course', '2012_Fall'), course_locations[0])
+ assert_equals(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations[0])
course_locations = self.store.get_courses_for_wiki('no_such_wiki')
assert_equals(len(course_locations), 0)
# set toy course to share the wiki with simple course
- toy_course = self.store.get_course('edX/toy/2012_Fall')
+ toy_course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
toy_course.wiki_slug = 'simple'
self.store.update_item(toy_course)
@@ -299,17 +318,78 @@ class TestMongoModuleStore(object):
course_locations = self.store.get_courses_for_wiki('simple')
assert_equals(len(course_locations), 2)
for course_number in ['toy', 'simple']:
- assert_in(Location('i4x', 'edX', course_number, 'course', '2012_Fall'), course_locations)
+ assert_in(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations)
# configure simple course to use unique wiki_slug.
- simple_course = self.store.get_course('edX/simple/2012_Fall')
+ simple_course = self.store.get_course(SlashSeparatedCourseKey('edX', 'simple', '2012_Fall'))
simple_course.wiki_slug = 'edX.simple.2012_Fall'
self.store.update_item(simple_course)
# it should be retrievable with its new wiki_slug
course_locations = self.store.get_courses_for_wiki('edX.simple.2012_Fall')
assert_equals(len(course_locations), 1)
- assert_in(Location('i4x', 'edX', 'simple', 'course', '2012_Fall'), course_locations)
+ assert_in(Location('edX', 'simple', '2012_Fall', 'course', '2012_Fall'), course_locations)
+
+ @Plugin.register_temp_plugin(ReferenceTestXBlock, 'ref_test')
+ def test_reference_converters(self):
+ """
+ Test that references types get deserialized correctly
+ """
+ course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
+
+ def setup_test():
+ course = self.store.get_course(course_key)
+ # can't use item factory as it depends on django settings
+ p1ele = self.store.create_and_save_xmodule(course.id.make_usage_key('problem', 'p1'))
+ p2ele = self.store.create_and_save_xmodule(course.id.make_usage_key('problem', 'p2'))
+ self.refloc = course.id.make_usage_key('ref_test', 'ref_test')
+ self.store.create_and_save_xmodule(
+ self.refloc, fields={
+ 'reference_link': p1ele.location,
+ 'reference_list': [p1ele.location, p2ele.location],
+ 'reference_dict': {'p1': p1ele.location, 'p2': p2ele.location},
+ 'children': [p1ele.location, p2ele.location],
+ }
+ )
+ def check_xblock_fields():
+ def check_children(xblock):
+ for child in xblock.children:
+ assert_is_instance(child, Location)
+
+ course = self.store.get_course(course_key)
+ check_children(course)
+
+ refele = self.store.get_item(self.refloc)
+ check_children(refele)
+ assert_is_instance(refele.reference_link, Location)
+ assert_greater(len(refele.reference_list), 0)
+ for ref in refele.reference_list:
+ assert_is_instance(ref, Location)
+ assert_greater(len(refele.reference_dict), 0)
+ for ref in refele.reference_dict.itervalues():
+ assert_is_instance(ref, Location)
+
+ def check_mongo_fields():
+ def get_item(location):
+ return self.store._find_one(location)
+
+ def check_children(payload):
+ for child in payload['definition']['children']:
+ assert_is_instance(child, basestring)
+
+ refele = get_item(self.refloc)
+ check_children(refele)
+ assert_is_instance(refele['definition']['data']['reference_link'], basestring)
+ assert_greater(len(refele['definition']['data']['reference_list']), 0)
+ for ref in refele['definition']['data']['reference_list']:
+ assert_is_instance(ref, basestring)
+ assert_greater(len(refele['metadata']['reference_dict']), 0)
+ for ref in refele['metadata']['reference_dict'].itervalues():
+ assert_is_instance(ref, basestring)
+
+ setup_test()
+ check_xblock_fields()
+ check_mongo_fields()
class TestMongoKeyValueStore(object):
"""
@@ -318,8 +398,8 @@ class TestMongoKeyValueStore(object):
def setUp(self):
self.data = {'foo': 'foo_value'}
- self.location = Location('i4x://org/course/category/name@version')
- self.children = ['i4x://org/course/child/a', 'i4x://org/course/child/b']
+ self.course_id = SlashSeparatedCourseKey('org', 'course', 'run')
+ self.children = [self.course_id.make_usage_key('child', 'a'), self.course_id.make_usage_key('child', 'b')]
self.metadata = {'meta': 'meta_val'}
self.kvs = MongoKeyValueStore(self.data, self.children, self.metadata)
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_orphan.py b/common/lib/xmodule/xmodule/modulestore/tests/test_orphan.py
index 6202d9779de..5385eeda963 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_orphan.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_orphan.py
@@ -1,158 +1,52 @@
-import uuid
-import mock
-import unittest
-import random
-import datetime
+from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
-from xmodule.modulestore.inheritance import InheritanceMixin
-from xmodule.modulestore.mongo import MongoModuleStore
-from xmodule.modulestore.split_mongo import SplitMongoModuleStore
-from xmodule.modulestore import Location
-from xmodule.fields import Date
-from xmodule.modulestore.locator import BlockUsageLocator, CourseLocator
-
-class TestOrphan(unittest.TestCase):
+class TestOrphan(SplitWMongoCourseBoostrapper):
"""
Test the orphan finding code
"""
- # Snippet of what would be in the django settings envs file
- db_config = {
- 'host': 'localhost',
- 'db': 'test_xmodule',
- }
-
- modulestore_options = {
- 'default_class': 'xmodule.raw_module.RawDescriptor',
- 'fs_root': '',
- 'render_template': mock.Mock(return_value=""),
- 'xblock_mixins': (InheritanceMixin,)
- }
-
- split_package_id = 'test_org.test_course.runid'
-
- def setUp(self):
- self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
-
- self.userid = random.getrandbits(32)
- super(TestOrphan, self).setUp()
- self.split_mongo = SplitMongoModuleStore(
- self.db_config,
- **self.modulestore_options
- )
- self.addCleanup(self.tear_down_split)
- self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
- self.addCleanup(self.tear_down_mongo)
- self.course_location = None
- self._create_course()
-
- def tear_down_split(self):
- """
- Remove the test collections, close the db connection
- """
- split_db = self.split_mongo.db
- split_db.drop_collection(split_db.course_index)
- split_db.drop_collection(split_db.structures)
- split_db.drop_collection(split_db.definitions)
- split_db.connection.close()
-
- def tear_down_mongo(self):
- """
- Remove the test collections, close the db connection
- """
- split_db = self.split_mongo.db
- # old_mongo doesn't give a db attr, but all of the dbs are the same
- split_db.drop_collection(self.old_mongo.collection)
-
- def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
- """
- Create the item of the given category and block id in split and old mongo, add it to the optional
- parent. The parent category is only needed because old mongo requires it for the id.
- """
- location = Location('i4x', 'test_org', 'test_course', category, name)
- self.old_mongo.create_and_save_xmodule(location, data, metadata, runtime)
- if isinstance(data, basestring):
- fields = {'data': data}
- else:
- fields = data.copy()
- fields.update(metadata)
- if parent_name:
- # add child to parent in mongo
- parent_location = Location('i4x', 'test_org', 'test_course', parent_category, parent_name)
- parent = self.old_mongo.get_item(parent_location)
- parent.children.append(location.url())
- self.old_mongo.update_item(parent, self.userid)
- # create pointer for split
- course_or_parent_locator = BlockUsageLocator(
- package_id=self.split_package_id,
- branch='draft',
- block_id=parent_name
- )
- else:
- course_or_parent_locator = CourseLocator(
- package_id='test_org.test_course.runid',
- branch='draft',
- )
- self.split_mongo.create_item(course_or_parent_locator, category, self.userid, block_id=name, fields=fields)
-
def _create_course(self):
"""
* some detached items
* some attached children
* some orphans
"""
- date_proxy = Date()
- metadata = {
- 'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
- 'display_name': 'Migration test course',
- }
- data = {
- 'wiki_slug': 'test_course_slug'
- }
- fields = metadata.copy()
- fields.update(data)
- # split requires the course to be created separately from creating items
- self.split_mongo.create_course(
- self.split_package_id, 'test_org', self.userid, fields=fields, root_block_id='runid'
- )
- self.course_location = Location('i4x', 'test_org', 'test_course', 'course', 'runid')
- self.old_mongo.create_and_save_xmodule(self.course_location, data, metadata)
- runtime = self.old_mongo.get_item(self.course_location).runtime
+ super(TestOrphan, self)._create_course()
- self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', runtime)
- self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', runtime)
- self._create_item('chapter', 'OrphanChapter', {}, {'display_name': 'Orphan Chapter'}, None, None, runtime)
- self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', runtime)
- self._create_item('vertical', 'OrphanVert', {}, {'display_name': 'Orphan Vertical'}, None, None, runtime)
- self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', runtime)
- self._create_item('html', 'OrphanHtml', "<p>Hello</p>", {'display_name': 'Orphan html'}, None, None, runtime)
- self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, runtime)
- self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, runtime)
- self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, runtime)
+ self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid')
+ self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid')
+ self._create_item('chapter', 'OrphanChapter', {}, {'display_name': 'Orphan Chapter'}, None, None)
+ self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1')
+ self._create_item('vertical', 'OrphanVert', {}, {'display_name': 'Orphan Vertical'}, None, None)
+ self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1')
+ self._create_item('html', 'OrphanHtml', "<p>Hello</p>", {'display_name': 'Orphan html'}, None, None)
+ self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None)
+ self._create_item('about', 'overview', "<p>overview</p>", {}, None, None)
+ self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None)
def test_mongo_orphan(self):
"""
Test that old mongo finds the orphans
"""
- orphans = self.old_mongo.get_orphans(self.course_location, None)
+ orphans = self.old_mongo.get_orphans(self.old_course_key)
self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
- location = self.course_location.replace(category='chapter', name='OrphanChapter')
- self.assertIn(location.url(), orphans)
- location = self.course_location.replace(category='vertical', name='OrphanVert')
- self.assertIn(location.url(), orphans)
- location = self.course_location.replace(category='html', name='OrphanHtml')
- self.assertIn(location.url(), orphans)
+ location = self.old_course_key.make_usage_key('chapter', name='OrphanChapter')
+ self.assertIn(location.to_deprecated_string(), orphans)
+ location = self.old_course_key.make_usage_key('vertical', name='OrphanVert')
+ self.assertIn(location.to_deprecated_string(), orphans)
+ location = self.old_course_key.make_usage_key('html', 'OrphanHtml')
+ self.assertIn(location.to_deprecated_string(), orphans)
def test_split_orphan(self):
"""
- Test that old mongo finds the orphans
+ Test that split mongo finds the orphans
"""
- orphans = self.split_mongo.get_orphans(self.split_package_id, 'draft')
+ orphans = self.split_mongo.get_orphans(self.split_course_key)
self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
- location = BlockUsageLocator(package_id=self.split_package_id, branch='draft', block_id='OrphanChapter')
+ location = self.split_course_key.make_usage_key('chapter', 'OrphanChapter')
self.assertIn(location, orphans)
- location = BlockUsageLocator(package_id=self.split_package_id, branch='draft', block_id='OrphanVert')
+ location = self.split_course_key.make_usage_key('vertical', 'OrphanVert')
self.assertIn(location, orphans)
- location = BlockUsageLocator(package_id=self.split_package_id, branch='draft', block_id='OrphanHtml')
+ location = self.split_course_key.make_usage_key('html', 'OrphanHtml')
self.assertIn(location, orphans)
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py b/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py
index 9124eeffec8..19a9fb6249e 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py
@@ -1,103 +1,26 @@
"""
-Test the publish code (primary causing orphans)
+Test the publish code (mostly testing that publishing doesn't result in orphans)
"""
-import uuid
-import mock
-import unittest
-import datetime
-import random
-
-from xmodule.modulestore.inheritance import InheritanceMixin
-from xmodule.modulestore.mongo import MongoModuleStore, DraftMongoModuleStore
-from xmodule.modulestore import Location
-from xmodule.fields import Date
from xmodule.modulestore.exceptions import ItemNotFoundError
-from xmodule.modulestore.mongo.draft import DIRECT_ONLY_CATEGORIES
+from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
-class TestPublish(unittest.TestCase):
+class TestPublish(SplitWMongoCourseBoostrapper):
"""
Test the publish code (primary causing orphans)
"""
-
- # Snippet of what would be in the django settings envs file
- db_config = {
- 'host': 'localhost',
- 'db': 'test_xmodule',
- }
-
- modulestore_options = {
- 'default_class': 'xmodule.raw_module.RawDescriptor',
- 'fs_root': '',
- 'render_template': mock.Mock(return_value=""),
- 'xblock_mixins': (InheritanceMixin,)
- }
-
- def setUp(self):
- self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
-
- self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
- self.draft_mongo = DraftMongoModuleStore(self.db_config, **self.modulestore_options)
- self.addCleanup(self.tear_down_mongo)
- self.course_location = None
-
- def tear_down_mongo(self):
- # old_mongo doesn't give a db attr, but all of the dbs are the same and draft and pub use same collection
- dbref = self.old_mongo.collection.database
- dbref.drop_collection(self.old_mongo.collection)
- dbref.connection.close()
-
- def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
- """
- Create the item in either draft or direct based on category and attach to its parent.
- """
- location = self.course_location.replace(category=category, name=name)
- if category in DIRECT_ONLY_CATEGORIES:
- mongo = self.old_mongo
- else:
- mongo = self.draft_mongo
- mongo.create_and_save_xmodule(location, data, metadata, runtime)
- if isinstance(data, basestring):
- fields = {'data': data}
- else:
- fields = data.copy()
- fields.update(metadata)
- if parent_name:
- # add child to parent in mongo
- parent_location = self.course_location.replace(category=parent_category, name=parent_name)
- parent = self.draft_mongo.get_item(parent_location)
- parent.children.append(location.url())
- if parent_category in DIRECT_ONLY_CATEGORIES:
- mongo = self.old_mongo
- else:
- mongo = self.draft_mongo
- mongo.update_item(parent, '**replace_user**')
-
def _create_course(self):
"""
Create the course, publish all verticals
* some detached items
"""
- date_proxy = Date()
- metadata = {
- 'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
- 'display_name': 'Migration test course',
- }
- data = {
- 'wiki_slug': 'test_course_slug'
- }
- fields = metadata.copy()
- fields.update(data)
-
- self.course_location = Location('i4x', 'test_org', 'test_course', 'course', 'runid')
- self.old_mongo.create_and_save_xmodule(self.course_location, data, metadata)
- runtime = self.draft_mongo.get_item(self.course_location).runtime
+ super(TestPublish, self)._create_course(split=False)
- self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', runtime)
- self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', runtime)
- self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', runtime)
- self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', runtime)
- self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', runtime)
+ self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
+ self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False)
+ self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', split=False)
+ self._create_item('vertical', 'Vert2', {}, {'display_name': 'Vertical 2'}, 'chapter', 'Chapter1', split=False)
+ self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion1',
"discussion discussion_category=\"Lecture 1\" discussion_id=\"a08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 1\"/>\n",
@@ -107,9 +30,10 @@ class TestPublish(unittest.TestCase):
"display_name": "Lecture 1 Discussion",
"discussion_id": "a08bfd89b2aa40fa81f2c650a9332846"
},
- 'vertical', 'Vert1', runtime
+ 'vertical', 'Vert1',
+ split=False
)
- self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', runtime)
+ self._create_item('html', 'Html2', "<p>Hellow</p>", {'display_name': 'Hollow Html'}, 'vertical', 'Vert1', split=False)
self._create_item(
'discussion', 'Discussion2',
"discussion discussion_category=\"Lecture 2\" discussion_id=\"b08bfd89b2aa40fa81f2c650a9332846\" discussion_target=\"Lecture 2\"/>\n",
@@ -119,11 +43,12 @@ class TestPublish(unittest.TestCase):
"display_name": "Lecture 2 Discussion",
"discussion_id": "b08bfd89b2aa40fa81f2c650a9332846"
},
- 'vertical', 'Vert2', runtime
+ 'vertical', 'Vert2',
+ split=False
)
- self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, runtime)
- self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, runtime)
- self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, runtime)
+ self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, split=False)
+ self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, split=False)
+ self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, split=False)
def _xmodule_recurse(self, item, action):
"""
@@ -142,13 +67,11 @@ class TestPublish(unittest.TestCase):
To reproduce a bug (STUD-811) publish a vertical, convert to draft, delete a child, move a child, publish.
See if deleted and moved children still is connected or exists in db (bug was disconnected but existed)
"""
- self._create_course()
- userid = random.getrandbits(32)
- location = self.course_location.replace(category='vertical', name='Vert1')
+ location = self.old_course_key.make_usage_key('vertical', name='Vert1')
item = self.draft_mongo.get_item(location, 2)
self._xmodule_recurse(
item,
- lambda i: self.draft_mongo.publish(i.location, userid)
+ lambda i: self.draft_mongo.publish(i.location, self.userid)
)
# verify status
item = self.draft_mongo.get_item(location, 0)
@@ -164,26 +87,26 @@ class TestPublish(unittest.TestCase):
self.assertFalse(getattr(item, 'is_draft', False), "Published item doesn't say so")
# delete the discussion (which oddly is not in draft mode)
- location = self.course_location.replace(category='discussion', name='Discussion1')
+ location = self.old_course_key.make_usage_key('discussion', name='Discussion1')
self.draft_mongo.delete_item(location)
# remove pointer from draft vertical (verify presence first to ensure process is valid)
- self.assertIn(location.url(), draft_vert.children)
- draft_vert.children.remove(location.url())
+ self.assertIn(location, draft_vert.children)
+ draft_vert.children.remove(location)
# move the other child
- other_child_loc = self.course_location.replace(category='html', name='Html2')
- draft_vert.children.remove(other_child_loc.url())
- other_vert = self.draft_mongo.get_item(self.course_location.replace(category='vertical', name='Vert2'), 0)
- other_vert.children.append(other_child_loc.url())
- self.draft_mongo.update_item(draft_vert, '**replace_user**')
- self.draft_mongo.update_item(other_vert, '**replace_user**')
+ other_child_loc = self.old_course_key.make_usage_key('html', name='Html2')
+ draft_vert.children.remove(other_child_loc)
+ other_vert = self.draft_mongo.get_item(self.old_course_key.make_usage_key('vertical', name='Vert2'), 0)
+ other_vert.children.append(other_child_loc)
+ self.draft_mongo.update_item(draft_vert, self.userid)
+ self.draft_mongo.update_item(other_vert, self.userid)
# publish
self._xmodule_recurse(
draft_vert,
- lambda i: self.draft_mongo.publish(i.location, userid)
+ lambda i: self.draft_mongo.publish(i.location, self.userid)
)
item = self.old_mongo.get_item(draft_vert.location, 0)
- self.assertNotIn(location.url(), item.children)
+ self.assertNotIn(location, item.children)
with self.assertRaises(ItemNotFoundError):
self.draft_mongo.get_item(location)
- self.assertNotIn(other_child_loc.url(), item.children)
- self.assertTrue(self.draft_mongo.has_item(None, other_child_loc), "Oops, lost moved item")
+ self.assertNotIn(other_child_loc, item.children)
+ self.assertTrue(self.draft_mongo.has_item(other_child_loc), "Oops, lost moved item")
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py
index f57c347e112..a0b37622e31 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_migrator.py
@@ -1,81 +1,37 @@
"""
-Created on Sep 10, 2013
-
-@author: dmitchell
-
Tests for split_migrator
"""
-import unittest
import uuid
import random
import mock
-import datetime
-from xmodule.fields import Date
-from xmodule.modulestore import Location
-from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.loc_mapper_store import LocMapperStore
-from xmodule.modulestore.mongo.draft import DraftModuleStore
-from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
-from xmodule.modulestore.mongo.base import MongoModuleStore
from xmodule.modulestore.split_migrator import SplitMigrator
from xmodule.modulestore.mongo import draft
from xmodule.modulestore.tests import test_location_mapper
+from xmodule.modulestore.tests.test_split_w_old_mongo import SplitWMongoCourseBoostrapper
+from nose.tools import nottest
-class TestMigration(unittest.TestCase):
+@nottest
+class TestMigration(SplitWMongoCourseBoostrapper):
"""
Test the split migrator
"""
- # Snippet of what would be in the django settings envs file
- db_config = {
- 'host': 'localhost',
- 'db': 'test_xmodule',
- 'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]),
- }
-
- modulestore_options = {
- 'default_class': 'xmodule.raw_module.RawDescriptor',
- 'fs_root': '',
- 'render_template': mock.Mock(return_value=""),
- 'xblock_mixins': (InheritanceMixin,)
- }
-
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(), **self.db_config)
- self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
- self.draft_mongo = DraftModuleStore(self.db_config, **self.modulestore_options)
- self.split_mongo = SplitMongoModuleStore(
- doc_store_config=self.db_config,
- loc_mapper=self.loc_mapper,
- **self.modulestore_options
- )
+ self.split_mongo.loc_mapper = self.loc_mapper
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo, self.draft_mongo, self.loc_mapper)
- self.course_location = None
- self.create_source_course()
def tearDown(self):
dbref = self.loc_mapper.db
dbref.drop_collection(self.loc_mapper.location_map)
- split_db = self.split_mongo.db
- split_db.drop_collection(self.split_mongo.db_connection.course_index)
- split_db.drop_collection(self.split_mongo.db_connection.structures)
- split_db.drop_collection(self.split_mongo.db_connection.definitions)
- # old_mongo doesn't give a db attr, but all of the dbs are the same
- dbref.drop_collection(self.old_mongo.collection)
-
- dbref.connection.close()
-
super(TestMigration, self).tearDown()
- def _create_and_get_item(self, store, location, data, metadata, runtime=None):
- store.create_and_save_xmodule(location, data, metadata, runtime)
- return store.get_item(location)
-
- def create_source_course(self):
+ def _create_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
@@ -83,150 +39,138 @@ class TestMigration(unittest.TestCase):
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
- location = Location('i4x', 'test_org', 'test_course', 'course', 'runid')
- self.course_location = location
- date_proxy = Date()
- metadata = {
- 'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
- 'display_name': 'Migration test course',
- }
- data = {
- 'wiki_slug': 'test_course_slug'
- }
- course_root = self._create_and_get_item(self.old_mongo, location, data, metadata)
- runtime = course_root.runtime
+ super(TestMigration, self)._create_course(split=False)
+
# chapters
- location = location.replace(category='chapter', name=uuid.uuid4().hex)
- chapter1 = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Chapter 1'}, runtime)
- course_root.children.append(chapter1.location.url())
- location = location.replace(category='chapter', name=uuid.uuid4().hex)
- chapter2 = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Chapter 2'}, runtime)
- course_root.children.append(chapter2.location.url())
- self.old_mongo.update_item(course_root, '**replace_user**')
+ chapter1_name = uuid.uuid4().hex
+ self._create_item('chapter', chapter1_name, {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
+ chap2_loc = self.old_course_key.make_usage_key('chapter', uuid.uuid4().hex)
+ self._create_item(
+ chap2_loc.category, chap2_loc.name, {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False
+ )
# vertical in live only
- location = location.replace(category='vertical', name=uuid.uuid4().hex)
- live_vert = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Live vertical'}, runtime)
- chapter1.children.append(live_vert.location.url())
- self.create_random_units(self.old_mongo, live_vert)
- # vertical in both live and draft
- location = location.replace(category='vertical', name=uuid.uuid4().hex)
- both_vert = self._create_and_get_item(
- self.old_mongo, location, {}, {'display_name': 'Both vertical'}, runtime
+ live_vert_name = uuid.uuid4().hex
+ self._create_item(
+ 'vertical', live_vert_name, {}, {'display_name': 'Live vertical'}, 'chapter', chapter1_name,
+ draft=False, split=False
)
- draft_both = self._create_and_get_item(
- self.draft_mongo, location, {}, {'display_name': 'Both vertical renamed'}, runtime
+ self.create_random_units(False, self.old_course_key.make_usage_key('vertical', live_vert_name))
+ # vertical in both live and draft
+ both_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
+ self._create_item(
+ both_vert_loc.category, both_vert_loc.name, {}, {'display_name': 'Both vertical'}, 'chapter', chapter1_name,
+ draft=False, split=False
)
- chapter1.children.append(both_vert.location.url())
- self.create_random_units(self.old_mongo, both_vert, self.draft_mongo, draft_both)
+ self.create_random_units(False, both_vert_loc)
+ draft_both = self.draft_mongo.get_item(both_vert_loc)
+ draft_both.display_name = 'Both vertical renamed'
+ self.draft_mongo.update_item(draft_both)
+ self.create_random_units(True, both_vert_loc)
# vertical in draft only (x2)
- location = location.replace(category='vertical', name=uuid.uuid4().hex)
- draft_vert = self._create_and_get_item(
- self.draft_mongo,
- location, {}, {'display_name': 'Draft vertical'}, runtime)
- chapter1.children.append(draft_vert.location.url())
- self.create_random_units(self.draft_mongo, draft_vert)
- location = location.replace(category='vertical', name=uuid.uuid4().hex)
- draft_vert = self._create_and_get_item(
- self.draft_mongo,
- location, {}, {'display_name': 'Draft vertical2'}, runtime)
- chapter1.children.append(draft_vert.location.url())
- self.create_random_units(self.draft_mongo, draft_vert)
- # and finally one in live only (so published has to skip 2)
- location = location.replace(category='vertical', name=uuid.uuid4().hex)
- live_vert = self._create_and_get_item(
- self.old_mongo,
- location, {}, {'display_name': 'Live vertical end'}, runtime)
- chapter1.children.append(live_vert.location.url())
- self.create_random_units(self.old_mongo, live_vert)
-
- # update the chapter
- self.old_mongo.update_item(chapter1, '**replace_user**')
-
- # now the other one w/ the conditional
- # first create some show children
- indirect1 = self._create_and_get_item(
- self.old_mongo,
- location.replace(category='discussion', name=uuid.uuid4().hex),
- "", {'display_name': 'conditional show 1'}, runtime
+ draft_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
+ self._create_item(
+ draft_vert_loc.category, draft_vert_loc.name, {}, {'display_name': 'Draft vertical'}, 'chapter', chapter1_name,
+ draft=True, split=False
+ )
+ self.create_random_units(True, draft_vert_loc)
+ draft_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
+ self._create_item(
+ draft_vert_loc.category, draft_vert_loc.name, {}, {'display_name': 'Draft vertical2'}, 'chapter', chapter1_name,
+ draft=True, split=False
)
- indirect2 = self._create_and_get_item(
- self.old_mongo,
- location.replace(category='html', name=uuid.uuid4().hex),
- "", {'display_name': 'conditional show 2'}, runtime
+ self.create_random_units(True, draft_vert_loc)
+
+ # and finally one in live only (so published has to skip 2 preceding sibs)
+ live_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
+ self._create_item(
+ live_vert_loc.category, live_vert_loc.name, {}, {'display_name': 'Live vertical end'}, 'chapter', chapter1_name,
+ draft=False, split=False
)
- location = location.replace(category='conditional', name=uuid.uuid4().hex)
- metadata = {
- 'xml_attributes': {
- 'sources': [live_vert.location.url(), ],
- 'completed': True,
+ self.create_random_units(True, draft_vert_loc)
+
+ # now the other chapter w/ the conditional
+ # create pointers to children (before the children exist)
+ indirect1_loc = self.old_course_key.make_usage_key('discussion', uuid.uuid4().hex)
+ indirect2_loc = self.old_course_key.make_usage_key('html', uuid.uuid4().hex)
+ conditional_loc = self.old_course_key.make_usage_key('conditional', uuid.uuid4().hex)
+ self._create_item(
+ conditional_loc.category, conditional_loc.name,
+ {
+ 'show_tag_list': [indirect1_loc, indirect2_loc],
+ 'sources_list': [live_vert_loc, ],
+ },
+ {
+ 'xml_attributes': {
+ 'completed': True,
+ },
},
- }
- data = {
- 'show_tag_list': [indirect1.location.url(), indirect2.location.url()]
- }
- conditional = self._create_and_get_item(self.old_mongo, location, data, metadata, runtime)
- conditional.children = [indirect1.location.url(), indirect2.location.url()]
+ chap2_loc.category, chap2_loc.name,
+ draft=False, split=False
+ )
+ # create the children
+ self._create_item(
+ indirect1_loc.category, indirect1_loc.name, {'data': ""}, {'display_name': 'conditional show 1'},
+ conditional_loc.category, conditional_loc.name,
+ draft=False, split=False
+ )
+ self._create_item(
+ indirect2_loc.category, indirect2_loc.name, {'data': ""}, {'display_name': 'conditional show 2'},
+ conditional_loc.category, conditional_loc.name,
+ draft=False, split=False
+ )
+
# add direct children
- self.create_random_units(self.old_mongo, conditional)
- chapter2.children.append(conditional.location.url())
- self.old_mongo.update_item(chapter2, '**replace_user**')
+ self.create_random_units(False, conditional_loc)
# and the ancillary docs (not children)
- location = location.replace(category='static_tab', name=uuid.uuid4().hex)
- # the below automatically adds the tab to the course
- _tab = self._create_and_get_item(self.old_mongo, location, "", {'display_name': 'Tab uno'}, runtime)
-
- location = location.replace(category='about', name='overview')
- _overview = self._create_and_get_item(self.old_mongo, location, "<p>test</p>", {}, runtime)
- location = location.replace(category='course_info', name='updates')
- _overview = self._create_and_get_item(
- self.old_mongo,
- location, "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, runtime
+ self._create_item(
+ 'static_tab', uuid.uuid4().hex, {'data': ""}, {'display_name': 'Tab uno'},
+ None, None, draft=False, split=False
+ )
+ self._create_item(
+ 'about', 'overview', {'data': "<p>test</p>"}, {},
+ None, None, draft=False, split=False
+ )
+ self._create_item(
+ 'course_info', 'updates', {'data': "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>"}, {},
+ None, None, draft=False, split=False
)
- def create_random_units(self, store, parent, cc_store=None, cc_parent=None):
+ def create_random_units(self, draft, parent_loc):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
- :param cc_store: (optional) if given, make a small change and save also to this store but w/ same location
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
- location = parent.location.replace(
+ location = parent_loc.replace(
category=random.choice(['html', 'video', 'problem', 'discussion']),
name=uuid.uuid4().hex
)
metadata = {'display_name': str(uuid.uuid4()), 'graded': True}
data = {}
- element = self._create_and_get_item(store, location, data, metadata, parent.runtime)
- parent.children.append(element.location.url())
- if cc_store is not None:
- # change display_name and remove graded to test the delta
- element = self._create_and_get_item(
- cc_store, location, data, {'display_name': str(uuid.uuid4())}, parent.runtime
- )
- cc_parent.children.append(element.location.url())
- store.update_item(parent, '**replace_user**')
- if cc_store is not None:
- cc_store.update_item(cc_parent, '**replace_user**')
+ self._create_item(
+ location.category, location.name, data, metadata, parent_loc.category, parent_loc.name,
+ draft=draft, split=False
+ )
def compare_courses(self, presplit, published):
# descend via children to do comparison
- old_root = presplit.get_item(self.course_location, depth=None)
- new_root_locator = self.loc_mapper.translate_location(
- self.course_location.course_id, self.course_location, published, add_entry_if_missing=False
+ old_root = presplit.get_course(self.old_course_key)
+ new_root_locator = self.loc_mapper.translate_location_to_course_locator(
+ old_root.id, published
)
new_root = self.split_mongo.get_course(new_root_locator)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ['conditional', 'about', 'course_info', 'static_tab']:
- location = self.course_location.replace(name=None, category=category)
- for conditional in presplit.get_items(location):
+ for conditional in presplit.get_items(self.old_course_key, category=category):
locator = self.loc_mapper.translate_location(
- self.course_location.course_id,
- conditional.location, published, add_entry_if_missing=False
+ conditional.location,
+ published,
+ add_entry_if_missing=False
)
self.compare_dags(presplit, conditional, self.split_mongo.get_item(locator), published)
@@ -262,9 +206,10 @@ class TestMigration(unittest.TestCase):
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
- len(presplit_dag_root.get_children()), len(split_dag_root.get_children()),
- "{0.category} '{0.display_name}': children count {1} != {2}".format(
- presplit_dag_root, len(presplit_dag_root.get_children()), split_dag_root.children
+ # need get_children to filter out drafts
+ len(presplit_dag_root.get_children()), len(split_dag_root.children),
+ "{0.category} '{0.display_name}': children {1} != {2}".format(
+ presplit_dag_root, presplit_dag_root.children, split_dag_root.children
)
)
for pre_child, split_child in zip(presplit_dag_root.get_children(), split_dag_root.get_children()):
@@ -272,7 +217,7 @@ class TestMigration(unittest.TestCase):
def test_migrator(self):
user = mock.Mock(id=1)
- self.migrator.migrate_mongo_course(self.course_location, user)
+ self.migrator.migrate_mongo_course(self.old_course_key, user)
# now compare the migrated to the original course
self.compare_courses(self.old_mongo, True)
self.compare_courses(self.draft_mongo, False)
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
index 4ada0bfddad..84b346d94ec 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
@@ -17,8 +17,8 @@ from xmodule.modulestore.locator import CourseLocator, BlockUsageLocator, Versio
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.x_module import XModuleMixin
from xmodule.fields import Date, Timedelta
-from bson.objectid import ObjectId
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
+from xmodule.modulestore.tests.test_modulestore import check_has_course_method
class SplitModuleTest(unittest.TestCase):
@@ -56,6 +56,7 @@ class SplitModuleTest(unittest.TestCase):
COURSE_CONTENT = {
"testx.GreekHero": {
"org": "testx",
+ "offering": "GreekHero",
"root_block_id": "head12345",
"user_id": "test@edx.org",
"fields": {
@@ -185,7 +186,7 @@ class SplitModuleTest(unittest.TestCase):
}}
},
{"user_id": "testassist@edx.org",
- "update":
+ "update":
{"head12345": {
"end": _date_field.from_json("2013-06-13T04:30"),
"grading_policy": {
@@ -272,9 +273,10 @@ class SplitModuleTest(unittest.TestCase):
]
},
]
- },
+ },
"testx.wonderful": {
"org": "testx",
+ "offering": "wonderful",
"root_block_id": "head23456",
"user_id": "test@edx.org",
"fields": {
@@ -377,9 +379,10 @@ class SplitModuleTest(unittest.TestCase):
}
}
]
- },
+ },
"guestx.contender": {
"org": "guestx",
+ "offering": "contender",
"root_block_id": "head345679",
"user_id": "test@guestx.edu",
"fields": {
@@ -441,7 +444,7 @@ class SplitModuleTest(unittest.TestCase):
split_store = modulestore()
for course_id, course_spec in SplitModuleTest.COURSE_CONTENT.iteritems():
course = split_store.create_course(
- course_id, course_spec['org'], course_spec['user_id'],
+ course_spec['org'], course_spec['offering'], course_spec['user_id'],
fields=course_spec['fields'],
root_block_id=course_spec['root_block_id']
)
@@ -452,7 +455,7 @@ class SplitModuleTest(unittest.TestCase):
block = course
else:
block_usage = BlockUsageLocator.make_relative(course.location, block_id)
- block = split_store.get_instance(course.location.package_id, block_usage)
+ block = split_store.get_item(block_usage)
for key, value in fields.iteritems():
setattr(block, key, value)
# create new blocks into dag: parent must already exist; thus, order is important
@@ -464,7 +467,7 @@ class SplitModuleTest(unittest.TestCase):
parent = course
else:
block_usage = BlockUsageLocator.make_relative(course.location, spec['parent'])
- parent = split_store.get_instance(course.location.package_id, block_usage)
+ parent = split_store.get_item(block_usage)
block_id = LocalId(spec['id'])
child = split_store.create_xblock(
course.runtime, spec['category'], spec['fields'], block_id, parent_xblock=parent
@@ -472,8 +475,11 @@ class SplitModuleTest(unittest.TestCase):
new_ele_dict[spec['id']] = child
course = split_store.persist_xblock_dag(course, revision['user_id'])
# publish "testx.wonderful"
- to_publish = BlockUsageLocator(package_id="testx.wonderful", branch="draft", block_id="head23456")
- destination = CourseLocator(package_id="testx.wonderful", branch="published")
+ to_publish = BlockUsageLocator(
+ CourseLocator(org="testx", offering="wonderful", branch="draft"),
+ block_id="head23456"
+ )
+ destination = CourseLocator(org="testx", offering="wonderful", branch="published")
split_store.xblock_publish("test@edx.org", to_publish, destination, [to_publish.block_id], None)
def tearDown(self):
@@ -509,7 +515,7 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(len(courses), 3, "Wrong number of courses")
# check metadata -- NOTE no promised order
course = self.findByIdInResult(courses, "head12345")
- self.assertEqual(course.location.package_id, "testx.GreekHero")
+ self.assertEqual(course.location.org, "testx")
self.assertEqual(course.category, 'course', 'wrong category')
self.assertEqual(len(course.tabs), 6, "wrong number of tabs")
self.assertEqual(
@@ -532,7 +538,8 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(len(courses_published), 1, len(courses_published))
course = self.findByIdInResult(courses_published, "head23456")
self.assertIsNotNone(course, "published courses")
- self.assertEqual(course.location.package_id, "testx.wonderful")
+ self.assertEqual(course.location.course_key.org, "testx")
+ self.assertEqual(course.location.course_key.offering, "wonderful")
self.assertEqual(course.category, 'course', 'wrong category')
self.assertEqual(len(course.tabs), 4, "wrong number of tabs")
self.assertEqual(course.display_name, "The most wonderful course",
@@ -550,16 +557,27 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertIsNotNone(self.findByIdInResult(courses, "head12345"))
self.assertIsNotNone(self.findByIdInResult(courses, "head23456"))
+ def test_has_course(self):
+ '''
+ Test the various calling forms for has_course
+ '''
+
+ check_has_course_method(
+ modulestore(),
+ CourseLocator(org='testx', offering='wonderful', branch="draft"),
+ locator_key_fields=['org', 'offering']
+ )
+
def test_get_course(self):
'''
Test the various calling forms for get_course
'''
- locator = CourseLocator(package_id="testx.GreekHero", branch="draft")
+ locator = CourseLocator(org='testx', offering='GreekHero', branch="draft")
head_course = modulestore().get_course(locator)
self.assertNotEqual(head_course.location.version_guid, head_course.previous_version)
locator = CourseLocator(version_guid=head_course.previous_version)
course = modulestore().get_course(locator)
- self.assertIsNone(course.location.package_id)
+ self.assertIsNone(course.location.course_key.org)
self.assertEqual(course.location.version_guid, head_course.previous_version)
self.assertEqual(course.category, 'course')
self.assertEqual(len(course.tabs), 6)
@@ -572,9 +590,10 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(course.edited_by, "testassist@edx.org")
self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.55})
- locator = CourseLocator(package_id='testx.GreekHero', branch='draft')
+ locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
course = modulestore().get_course(locator)
- self.assertEqual(course.location.package_id, "testx.GreekHero")
+ self.assertEqual(course.location.course_key.org, "testx")
+ self.assertEqual(course.location.course_key.offering, "GreekHero")
self.assertEqual(course.category, 'course')
self.assertEqual(len(course.tabs), 6)
self.assertEqual(course.display_name, "The Ancient Greek Hero")
@@ -584,29 +603,28 @@ class SplitModuleCourseTests(SplitModuleTest):
self.assertEqual(course.edited_by, "testassist@edx.org")
self.assertDictEqual(course.grade_cutoffs, {"Pass": 0.45})
- locator = CourseLocator(package_id='testx.wonderful', branch='published')
+ locator = CourseLocator(org='testx', offering='wonderful', branch='published')
course = modulestore().get_course(locator)
published_version = course.location.version_guid
- locator = CourseLocator(package_id='testx.wonderful', branch='draft')
+ locator = CourseLocator(org='testx', offering='wonderful', branch='draft')
course = modulestore().get_course(locator)
self.assertNotEqual(course.location.version_guid, published_version)
def test_get_course_negative(self):
# Now negative testing
- self.assertRaises(InsufficientSpecificationError,
- modulestore().get_course, CourseLocator(package_id='edu.meh.blah'))
- self.assertRaises(ItemNotFoundError,
- modulestore().get_course, CourseLocator(package_id='nosuchthing', branch='draft'))
- self.assertRaises(ItemNotFoundError,
- modulestore().get_course,
- CourseLocator(package_id='testx.GreekHero', branch='published'))
+ with self.assertRaises(InsufficientSpecificationError):
+ modulestore().get_course(CourseLocator(org='edu', offering='meh.blah'))
+ with self.assertRaises(ItemNotFoundError):
+ modulestore().get_course(CourseLocator(org='edu', offering='nosuchthing', branch='draft'))
+ with self.assertRaises(ItemNotFoundError):
+ modulestore().get_course(CourseLocator(org='testx', offering='GreekHero', branch='published'))
def test_cache(self):
"""
Test that the mechanics of caching work.
"""
- locator = CourseLocator(package_id='testx.GreekHero', branch='draft')
+ locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
course = modulestore().get_course(locator)
block_map = modulestore().cache_items(course.system, course.children, depth=3)
self.assertIn('chapter1', block_map)
@@ -616,7 +634,7 @@ class SplitModuleCourseTests(SplitModuleTest):
"""
get_course_successors(course_locator, version_history_depth=1)
"""
- locator = CourseLocator(package_id='testx.GreekHero', branch='draft')
+ locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
course = modulestore().get_course(locator)
versions = [course.location.version_guid, course.previous_version]
locator = CourseLocator(version_guid=course.previous_version)
@@ -626,7 +644,7 @@ class SplitModuleCourseTests(SplitModuleTest):
locator = CourseLocator(version_guid=course.previous_version)
result = modulestore().get_course_successors(locator)
self.assertIsInstance(result, VersionTree)
- self.assertIsNone(result.locator.package_id)
+ self.assertIsNone(result.locator.org)
self.assertEqual(result.locator.version_guid, versions[-1])
self.assertEqual(len(result.children), 1)
self.assertEqual(result.children[0].locator.version_guid, versions[-2])
@@ -652,87 +670,85 @@ class SplitModuleItemTests(SplitModuleTest):
'''
has_item(BlockUsageLocator)
'''
- package_id = 'testx.GreekHero'
- locator = CourseLocator(package_id=package_id, branch='draft')
- course = modulestore().get_course(locator)
+ org = 'testx'
+ offering = 'GreekHero'
+ course_locator = CourseLocator(org=org, offering=offering, branch='draft')
+ course = modulestore().get_course(course_locator)
previous_version = course.previous_version
# positive tests of various forms
- locator = BlockUsageLocator(version_guid=previous_version, block_id='head12345')
+ locator = BlockUsageLocator(CourseLocator(version_guid=previous_version), block_id='head12345')
self.assertTrue(
- modulestore().has_item(package_id, locator), "couldn't find in %s" % previous_version
+ modulestore().has_item(locator), "couldn't find in %s" % previous_version
)
- locator = BlockUsageLocator(package_id='testx.GreekHero', block_id='head12345', branch='draft')
+ locator = BlockUsageLocator(course_locator, block_id='head12345')
self.assertTrue(
- modulestore().has_item(locator.package_id, locator),
+ modulestore().has_item(locator),
)
self.assertFalse(
- modulestore().has_item(locator.package_id, BlockUsageLocator(
- package_id=locator.package_id,
- branch='published',
- block_id=locator.block_id)),
+ modulestore().has_item(
+ BlockUsageLocator(
+ locator.course_key.for_branch('published'),
+ block_id=locator.block_id
+ )
+ ),
"found in published head"
)
# not a course obj
- locator = BlockUsageLocator(package_id='testx.GreekHero', block_id='chapter1', branch='draft')
+ locator = BlockUsageLocator(course_locator, block_id='chapter1')
self.assertTrue(
- modulestore().has_item(locator.package_id, locator),
+ modulestore().has_item(locator),
"couldn't find chapter1"
)
# in published course
- locator = BlockUsageLocator(package_id="testx.wonderful", block_id="head23456", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org="testx", offering="wonderful", branch='draft'),
+ block_id="head23456"
+ )
self.assertTrue(
modulestore().has_item(
- locator.package_id,
- BlockUsageLocator(package_id=locator.package_id, block_id=locator.block_id, branch='published')
+ BlockUsageLocator(locator.course_key.for_branch("published"), block_id=locator.block_id)
)
)
- locator.branch = 'published'
- self.assertTrue(modulestore().has_item(package_id, locator), "couldn't find in published")
+ locator = locator.for_branch('published')
+ self.assertTrue(modulestore().has_item(locator), "couldn't find in published")
def test_negative_has_item(self):
# negative tests--not found
# no such course or block
- package_id = 'testx.GreekHero'
- locator = BlockUsageLocator(package_id="doesnotexist", block_id="head23456", branch='draft')
- self.assertFalse(modulestore().has_item(package_id, locator))
- locator = BlockUsageLocator(package_id="testx.wonderful", block_id="doesnotexist", branch='draft')
- self.assertFalse(modulestore().has_item(package_id, locator))
-
- # negative tests--insufficient specification
- self.assertRaises(InsufficientSpecificationError, BlockUsageLocator)
-
- locator = CourseLocator(package_id=package_id, branch='draft')
- course = modulestore().get_course(locator)
- previous_version = course.previous_version
-
- with self.assertRaises(InsufficientSpecificationError):
- modulestore().has_item(None, BlockUsageLocator(version_guid=previous_version))
- with self.assertRaises(InsufficientSpecificationError):
- modulestore().has_item(None, BlockUsageLocator(package_id='testx.GreekHero'))
+ locator = BlockUsageLocator(
+ CourseLocator(org="foo", offering="doesnotexist", branch='draft'),
+ block_id="head23456"
+ )
+ self.assertFalse(modulestore().has_item(locator))
+ locator = BlockUsageLocator(
+ CourseLocator(org="testx", offering="wonderful", branch='draft'),
+ block_id="doesnotexist"
+ )
+ self.assertFalse(modulestore().has_item(locator))
def test_get_item(self):
'''
get_item(blocklocator)
'''
- locator = CourseLocator(package_id="testx.GreekHero", branch='draft')
- course = modulestore().get_course(locator)
+ hero_locator = CourseLocator(org="testx", offering="GreekHero", branch='draft')
+ course = modulestore().get_course(hero_locator)
previous_version = course.previous_version
# positive tests of various forms
- locator = BlockUsageLocator(version_guid=previous_version, block_id='head12345')
+ locator = BlockUsageLocator(CourseLocator(version_guid=previous_version), block_id='head12345')
block = modulestore().get_item(locator)
self.assertIsInstance(block, CourseDescriptor)
- # get_instance just redirects to get_item, ignores package_id
- self.assertIsInstance(modulestore().get_instance("package_id", locator), CourseDescriptor)
+ self.assertIsInstance(modulestore().get_item(locator), CourseDescriptor)
def verify_greek_hero(block):
"""
Check contents of block
"""
- self.assertEqual(block.location.package_id, "testx.GreekHero")
+ self.assertEqual(block.location.org, "testx")
+ self.assertEqual(block.location.offering, "GreekHero")
self.assertEqual(len(block.tabs), 6, "wrong number of tabs")
self.assertEqual(block.display_name, "The Ancient Greek Hero")
self.assertEqual(block.advertised_start, "Fall 2013")
@@ -743,18 +759,17 @@ class SplitModuleItemTests(SplitModuleTest):
block.grade_cutoffs, {"Pass": 0.45},
)
- locator = BlockUsageLocator(package_id='testx.GreekHero', block_id='head12345', branch='draft')
+ locator = BlockUsageLocator(hero_locator, block_id='head12345')
verify_greek_hero(modulestore().get_item(locator))
- # get_instance just redirects to get_item, ignores package_id
- verify_greek_hero(modulestore().get_instance("package_id", locator))
# try to look up other branches
- self.assertRaises(ItemNotFoundError,
- modulestore().get_item,
- BlockUsageLocator(package_id=locator.as_course_locator(),
- block_id=locator.block_id,
- branch='published'))
- locator.branch = 'draft'
+ with self.assertRaises(ItemNotFoundError):
+ modulestore().get_item(
+ BlockUsageLocator(
+ hero_locator.for_branch("published"),
+ block_id=locator.block_id,
+ )
+ )
self.assertIsInstance(
modulestore().get_item(locator),
CourseDescriptor
@@ -762,15 +777,19 @@ class SplitModuleItemTests(SplitModuleTest):
def test_get_non_root(self):
# not a course obj
- locator = BlockUsageLocator(package_id='testx.GreekHero', block_id='chapter1', branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'chapter1'
+ )
block = modulestore().get_item(locator)
- self.assertEqual(block.location.package_id, "testx.GreekHero")
+ self.assertEqual(block.location.package_id, "testx+GreekHero")
self.assertEqual(block.category, 'chapter')
self.assertEqual(block.display_name, "Hercules")
self.assertEqual(block.edited_by, "testassist@edx.org")
# in published course
- locator = BlockUsageLocator(package_id="testx.wonderful", block_id="head23456", branch='published')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='wonderful', branch='published'), 'head23456'
+ )
self.assertIsInstance(
modulestore().get_item(locator),
CourseDescriptor
@@ -778,19 +797,17 @@ class SplitModuleItemTests(SplitModuleTest):
# negative tests--not found
# no such course or block
- locator = BlockUsageLocator(package_id="doesnotexist", block_id="head23456", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='doesnotexist', offering='doesnotexist', branch='draft'), 'head23456'
+ )
with self.assertRaises(ItemNotFoundError):
modulestore().get_item(locator)
- locator = BlockUsageLocator(package_id="testx.wonderful", block_id="doesnotexist", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='wonderful', branch='draft'), 'doesnotexist'
+ )
with self.assertRaises(ItemNotFoundError):
modulestore().get_item(locator)
- # negative tests--insufficient specification
- with self.assertRaises(InsufficientSpecificationError):
- modulestore().get_item(BlockUsageLocator(version_guid=ObjectId()))
- with self.assertRaises(InsufficientSpecificationError):
- modulestore().get_item(BlockUsageLocator(package_id='testx.GreekHero', branch='draft'))
-
# pylint: disable=W0212
def test_matching(self):
'''
@@ -800,66 +817,65 @@ class SplitModuleItemTests(SplitModuleTest):
self.assertFalse(modulestore()._value_matches('help', 'Help'))
self.assertTrue(modulestore()._value_matches(['distract', 'help', 'notme'], 'help'))
self.assertFalse(modulestore()._value_matches(['distract', 'Help', 'notme'], 'help'))
- self.assertFalse(modulestore()._value_matches({'field': ['distract', 'Help', 'notme']}, {'field': 'help'}))
- self.assertFalse(modulestore()._value_matches(['distract', 'Help', 'notme'], {'field': 'help'}))
- self.assertTrue(modulestore()._value_matches(
+ self.assertFalse(modulestore()._block_matches({'field': ['distract', 'Help', 'notme']}, {'field': 'help'}))
+ self.assertTrue(modulestore()._block_matches(
{'field': ['distract', 'help', 'notme'],
'irrelevant': 2},
{'field': 'help'}))
- self.assertTrue(modulestore()._value_matches('I need some help', {'$regex': 'help'}))
- self.assertTrue(modulestore()._value_matches(['I need some help', 'today'], {'$regex': 'help'}))
- self.assertFalse(modulestore()._value_matches('I need some help', {'$regex': 'Help'}))
- self.assertFalse(modulestore()._value_matches(['I need some help', 'today'], {'$regex': 'Help'}))
+ self.assertTrue(modulestore()._value_matches('I need some help', re.compile(r'help')))
+ self.assertTrue(modulestore()._value_matches(['I need some help', 'today'], re.compile(r'help')))
+ self.assertFalse(modulestore()._value_matches('I need some help', re.compile(r'Help')))
+ self.assertTrue(modulestore()._value_matches(['I need some help', 'today'], re.compile(r'Help', re.IGNORECASE)))
self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1}))
- self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'c': None}))
- self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1, 'c': None}))
self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 2}))
self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'c': 1}))
self.assertFalse(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': 1, 'c': 1}))
+ self.assertTrue(modulestore()._block_matches({'a': 1, 'b': 2}, {'a': lambda i: 0 < i < 2}))
def test_get_items(self):
'''
get_items(locator, qualifiers, [branch])
'''
- locator = CourseLocator(package_id="testx.GreekHero", branch='draft')
+ locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
# get all modules
matches = modulestore().get_items(locator)
self.assertEqual(len(matches), 6)
- matches = modulestore().get_items(locator, qualifiers={})
+ matches = modulestore().get_items(locator)
self.assertEqual(len(matches), 6)
- matches = modulestore().get_items(locator, qualifiers={'category': 'chapter'})
+ matches = modulestore().get_items(locator, category='chapter')
self.assertEqual(len(matches), 3)
- matches = modulestore().get_items(locator, qualifiers={'category': 'garbage'})
+ matches = modulestore().get_items(locator, category='garbage')
self.assertEqual(len(matches), 0)
matches = modulestore().get_items(
locator,
- qualifiers=
- {
- 'category': 'chapter',
- 'fields': {'display_name': {'$regex': 'Hera'}}
- }
+ category='chapter',
+ settings={'display_name': re.compile(r'Hera')},
)
self.assertEqual(len(matches), 2)
- matches = modulestore().get_items(locator, qualifiers={'fields': {'children': 'chapter2'}})
+ matches = modulestore().get_items(locator, children='chapter2')
self.assertEqual(len(matches), 1)
self.assertEqual(matches[0].location.block_id, 'head12345')
def test_get_parents(self):
'''
- get_parent_locations(locator, [block_id], [branch]): [BlockUsageLocator]
+ get_parent_locations(locator): [BlockUsageLocator]
'''
- locator = BlockUsageLocator(package_id="testx.GreekHero", branch='draft', block_id='chapter1')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'),
+ block_id='chapter1'
+ )
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0].block_id, 'head12345')
- self.assertEqual(parents[0].package_id, "testx.GreekHero")
- locator.block_id = 'chapter2'
+ self.assertEqual(parents[0].org, "testx")
+ self.assertEqual(parents[0].offering, "GreekHero")
+ locator = locator.course_key.make_usage_key('Chapter', 'chapter2')
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0].block_id, 'head12345')
- locator.block_id = 'nosuchblock'
+ locator = locator.course_key.make_usage_key('garbage', 'nosuchblock')
parents = modulestore().get_parent_locations(locator)
self.assertEqual(len(parents), 0)
@@ -867,7 +883,9 @@ class SplitModuleItemTests(SplitModuleTest):
"""
Test the existing get_children method on xdescriptors
"""
- locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="head12345", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'head12345'
+ )
block = modulestore().get_item(locator)
children = block.get_children()
expected_ids = [
@@ -909,7 +927,7 @@ class TestItemCrud(SplitModuleTest):
create_item(course_or_parent_locator, category, user, definition_locator=None, fields): new_desciptor
"""
# grab link to course to ensure new versioning works
- locator = CourseLocator(package_id="testx.GreekHero", branch='draft')
+ locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
premod_course = modulestore().get_course(locator)
premod_history = modulestore().get_course_history_info(premod_course.location)
# add minimal one w/o a parent
@@ -919,7 +937,7 @@ class TestItemCrud(SplitModuleTest):
fields={'display_name': 'new sequential'}
)
# check that course version changed and course's previous is the other one
- self.assertEqual(new_module.location.package_id, "testx.GreekHero")
+ self.assertEqual(new_module.location.offering, "GreekHero")
self.assertNotEqual(new_module.location.version_guid, premod_course.location.version_guid)
self.assertIsNone(locator.version_guid, "Version inadvertently filled in")
current_course = modulestore().get_course(locator)
@@ -935,7 +953,7 @@ class TestItemCrud(SplitModuleTest):
self.assertEqual(new_module.display_name, 'new sequential')
# check that block does not exist in previous version
locator = BlockUsageLocator(
- version_guid=premod_course.location.version_guid,
+ CourseLocator(version_guid=premod_course.location.version_guid),
block_id=new_module.location.block_id
)
self.assertRaises(ItemNotFoundError, modulestore().get_item, locator)
@@ -944,11 +962,16 @@ class TestItemCrud(SplitModuleTest):
"""
Test create_item w/ specifying the parent of the new item
"""
- locator = BlockUsageLocator(package_id="testx.GreekHero", branch='draft', block_id='chapter2')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'),
+ block_id='chapter2'
+ )
original = modulestore().get_item(locator)
- locator = BlockUsageLocator(package_id="testx.wonderful", block_id="head23456", branch='draft')
- premod_course = modulestore().get_course(locator)
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='wonderful', branch='draft'), 'head23456'
+ )
+ premod_course = modulestore().get_course(locator.course_key)
category = 'chapter'
new_module = modulestore().create_item(
locator, category, 'user123',
@@ -967,10 +990,15 @@ class TestItemCrud(SplitModuleTest):
a definition id and new def data that it branches the definition in the db.
Actually, this tries to test all create_item features not tested above.
"""
- locator = BlockUsageLocator(package_id="testx.GreekHero", branch='draft', block_id='problem1')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'),
+ block_id='problem1'
+ )
original = modulestore().get_item(locator)
- locator = BlockUsageLocator(package_id="guestx.contender", block_id="head345679", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='guestx', offering='contender', branch='draft'), 'head345679'
+ )
category = 'problem'
new_payload = "<problem>empty</problem>"
new_module = modulestore().create_item(
@@ -1002,8 +1030,9 @@ class TestItemCrud(SplitModuleTest):
"""
Check that using odd characters in block id don't break ability to add and retrieve block.
"""
- parent_locator = BlockUsageLocator(package_id="guestx.contender", block_id="head345679", branch='draft')
- chapter_locator = BlockUsageLocator(package_id="guestx.contender", block_id="foo.bar_-~:0", branch='draft')
+ course_key = CourseLocator(org='guestx', offering='contender', branch='draft')
+ parent_locator = BlockUsageLocator(course_key, block_id="head345679")
+ chapter_locator = BlockUsageLocator(course_key, block_id="foo.bar_-~:0")
modulestore().create_item(
parent_locator, 'chapter', 'anotheruser',
block_id=chapter_locator.block_id,
@@ -1014,7 +1043,7 @@ class TestItemCrud(SplitModuleTest):
self.assertEqual(new_module.location.block_id, "foo.bar_-~:0") # hardcode to ensure BUL init didn't change
# now try making that a parent of something
new_payload = "<problem>empty</problem>"
- problem_locator = BlockUsageLocator(package_id="guestx.contender", block_id="prob.bar_-~:99a", branch='draft')
+ problem_locator = BlockUsageLocator(course_key, block_id="prob.bar_-~:99a")
modulestore().create_item(
chapter_locator, 'problem', 'anotheruser',
block_id=problem_locator.block_id,
@@ -1032,15 +1061,13 @@ class TestItemCrud(SplitModuleTest):
"""
# start transaction w/ simple creation
user = random.getrandbits(32)
- new_course = modulestore().create_course('test_org.test_transaction', 'test_org', user)
- new_course_locator = new_course.location.as_course_locator()
+ new_course = modulestore().create_course('test_org', 'test_transaction', user)
+ new_course_locator = new_course.id
index_history_info = modulestore().get_course_history_info(new_course.location)
course_block_prev_version = new_course.previous_version
course_block_update_version = new_course.update_version
self.assertIsNotNone(new_course_locator.version_guid, "Want to test a definite version")
- versionless_course_locator = CourseLocator(
- package_id=new_course_locator.package_id, branch=new_course_locator.branch
- )
+ versionless_course_locator = new_course_locator.version_agnostic()
# positive simple case: no force, add chapter
new_ele = modulestore().create_item(
@@ -1093,9 +1120,8 @@ class TestItemCrud(SplitModuleTest):
# add new child to old parent in continued (leave off version_guid)
course_module_locator = BlockUsageLocator(
- package_id=new_course.location.package_id,
+ new_course.location.course_key.version_agnostic(),
block_id=new_course.location.block_id,
- branch=new_course.location.branch
)
new_ele = modulestore().create_item(
course_module_locator, 'chapter', user,
@@ -1115,7 +1141,10 @@ class TestItemCrud(SplitModuleTest):
"""
test updating an items metadata ensuring the definition doesn't version but the course does if it should
"""
- locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="problem3_2", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org="testx", offering="GreekHero", branch='draft'),
+ block_id="problem3_2"
+ )
problem = modulestore().get_item(locator)
pre_def_id = problem.definition_locator.definition_id
pre_version_guid = problem.location.version_guid
@@ -1132,13 +1161,13 @@ class TestItemCrud(SplitModuleTest):
self.assertEqual(updated_problem.max_attempts, 4)
# refetch to ensure original didn't change
original_location = BlockUsageLocator(
- version_guid=pre_version_guid,
+ CourseLocator(version_guid=pre_version_guid),
block_id=problem.location.block_id
)
problem = modulestore().get_item(original_location)
self.assertNotEqual(problem.max_attempts, 4, "original changed")
- current_course = modulestore().get_course(locator)
+ current_course = modulestore().get_course(locator.course_key)
self.assertEqual(updated_problem.location.version_guid, current_course.location.version_guid)
history_info = modulestore().get_course_history_info(current_course.location)
@@ -1149,7 +1178,9 @@ class TestItemCrud(SplitModuleTest):
"""
test updating an item's children ensuring the definition doesn't version but the course does if it should
"""
- locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="chapter3", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'chapter3'
+ )
block = modulestore().get_item(locator)
pre_def_id = block.definition_locator.definition_id
pre_version_guid = block.location.version_guid
@@ -1164,10 +1195,9 @@ class TestItemCrud(SplitModuleTest):
self.assertNotEqual(updated_problem.location.version_guid, pre_version_guid)
self.assertEqual(updated_problem.children, block.children)
self.assertNotIn(moved_child, updated_problem.children)
- locator.block_id = "chapter1"
+ locator = locator.course_key.make_usage_key('Chapter', "chapter1")
other_block = modulestore().get_item(locator)
other_block.children.append(moved_child)
- other_block.save() # decache model changes
other_updated = modulestore().update_item(other_block, '**replace_user**')
self.assertIn(moved_child, other_updated.children)
@@ -1175,7 +1205,9 @@ class TestItemCrud(SplitModuleTest):
"""
test updating an item's definition: ensure it gets versioned as well as the course getting versioned
"""
- locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="head12345", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'head12345'
+ )
block = modulestore().get_item(locator)
pre_def_id = block.definition_locator.definition_id
pre_version_guid = block.location.version_guid
@@ -1192,10 +1224,16 @@ class TestItemCrud(SplitModuleTest):
"""
Test updating metadata, children, and definition in a single call ensuring all the versioning occurs
"""
- locator = BlockUsageLocator(package_id="testx.GreekHero", branch='draft', block_id='problem1')
+ locator = BlockUsageLocator(
+ CourseLocator('testx', 'GreekHero', branch='draft'),
+ block_id='problem1'
+ )
original = modulestore().get_item(locator)
# first add 2 children to the course for the update to manipulate
- locator = BlockUsageLocator(package_id="guestx.contender", block_id="head345679", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator('guestx', 'contender', branch='draft'),
+ block_id="head345679"
+ )
category = 'problem'
new_payload = "<problem>empty</problem>"
modulestore().create_item(
@@ -1231,33 +1269,28 @@ class TestItemCrud(SplitModuleTest):
def test_delete_item(self):
course = self.create_course_for_deletion()
- self.assertRaises(ValueError,
- modulestore().delete_item,
- course.location,
- 'deleting_user')
- reusable_location = BlockUsageLocator(
- package_id=course.location.package_id,
- block_id=course.location.block_id,
- branch='draft')
+ with self.assertRaises(ValueError):
+ modulestore().delete_item(course.location, 'deleting_user')
+ reusable_location = course.id.version_agnostic().for_branch('draft')
# delete a leaf
- problems = modulestore().get_items(reusable_location, {'category': 'problem'})
+ problems = modulestore().get_items(reusable_location, category='problem')
locn_to_del = problems[0].location
new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user', delete_children=False)
- deleted = BlockUsageLocator(package_id=reusable_location.package_id,
- branch=reusable_location.branch,
- block_id=locn_to_del.block_id)
- self.assertFalse(modulestore().has_item(reusable_location.package_id, deleted))
- self.assertRaises(VersionConflictError, modulestore().has_item, reusable_location.package_id, locn_to_del)
+ deleted = locn_to_del.version_agnostic()
+ self.assertFalse(modulestore().has_item(deleted))
+ with self.assertRaises(VersionConflictError):
+ modulestore().has_item(locn_to_del)
+
locator = BlockUsageLocator(
- version_guid=locn_to_del.version_guid,
+ CourseLocator(version_guid=locn_to_del.version_guid),
block_id=locn_to_del.block_id
)
- self.assertTrue(modulestore().has_item(reusable_location.package_id, locator))
+ self.assertTrue(modulestore().has_item(locator))
self.assertNotEqual(new_course_loc.version_guid, course.location.version_guid)
# delete a subtree
- nodes = modulestore().get_items(reusable_location, {'category': 'chapter'})
+ nodes = modulestore().get_items(reusable_location, category='chapter')
new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user', delete_children=True)
# check subtree
@@ -1267,15 +1300,23 @@ class TestItemCrud(SplitModuleTest):
"""
if node:
node_loc = node.location
- self.assertFalse(modulestore().has_item(reusable_location.package_id,
- BlockUsageLocator(
- package_id=node_loc.package_id,
- branch=node_loc.branch,
- block_id=node.location.block_id)))
+ self.assertFalse(
+ modulestore().has_item(
+ BlockUsageLocator(
+ CourseLocator(
+ org=node_loc.org,
+ offering=node_loc.offering,
+ branch=node_loc.branch,
+ ),
+ block_id=node_loc.block_id
+ )
+ )
+ )
locator = BlockUsageLocator(
- version_guid=node.location.version_guid,
- block_id=node.location.block_id)
- self.assertTrue(modulestore().has_item(reusable_location.package_id, locator))
+ CourseLocator(version_guid=node.location.version_guid),
+ block_id=node.location.block_id
+ )
+ self.assertTrue(modulestore().has_item(locator))
if node.has_children:
for sub in node.get_children():
check_subtree(sub)
@@ -1285,11 +1326,11 @@ class TestItemCrud(SplitModuleTest):
"""
Create a course we can delete
"""
- course = modulestore().create_course('nihilx.deletion', 'nihilx', 'deleting_user')
+ course = modulestore().create_course('nihilx', 'deletion', 'deleting_user')
root = BlockUsageLocator(
- package_id=course.location.package_id,
+ course.id.version_agnostic().for_branch('draft'),
block_id=course.location.block_id,
- branch='draft')
+ )
for _ in range(4):
self.create_subtree_for_deletion(root, ['chapter', 'vertical', 'problem'])
return modulestore().get_item(root)
@@ -1300,8 +1341,8 @@ class TestItemCrud(SplitModuleTest):
"""
if not category_queue:
return
- node = modulestore().create_item(parent, category_queue[0], 'deleting_user')
- node_loc = BlockUsageLocator(parent.as_course_locator(), block_id=node.location.block_id)
+ node = modulestore().create_item(parent.version_agnostic(), category_queue[0], 'deleting_user')
+ node_loc = BlockUsageLocator(parent.course_key, block_id=node.location.block_id)
for _ in range(4):
self.create_subtree_for_deletion(node_loc, category_queue[1:])
@@ -1315,7 +1356,7 @@ class TestCourseCreation(SplitModuleTest):
The simplest case but probing all expected results from it.
"""
# Oddly getting differences of 200nsec
- new_course = modulestore().create_course('test_org.test_course', 'test_org', 'create_user')
+ new_course = modulestore().create_course('test_org', 'test_course', 'create_user')
new_locator = new_course.location
# check index entry
index_info = modulestore().get_course_index_info(new_locator)
@@ -1340,13 +1381,13 @@ class TestCourseCreation(SplitModuleTest):
"""
Test making a course which points to an existing draft and published but not making any changes to either.
"""
- original_locator = CourseLocator(package_id="testx.wonderful", branch='draft')
+ original_locator = CourseLocator(org='testx', offering='wonderful', branch='draft')
original_index = modulestore().get_course_index_info(original_locator)
new_draft = modulestore().create_course(
'best', 'leech', 'leech_master',
versions_dict=original_index['versions'])
new_draft_locator = new_draft.location
- self.assertRegexpMatches(new_draft_locator.package_id, 'best')
+ self.assertRegexpMatches(new_draft_locator.org, 'best')
# the edited_by and other meta fields on the new course will be the original author not this one
self.assertEqual(new_draft.edited_by, 'test@edx.org')
self.assertEqual(new_draft_locator.version_guid, original_index['versions']['draft'])
@@ -1354,7 +1395,7 @@ class TestCourseCreation(SplitModuleTest):
new_index = modulestore().get_course_index_info(new_draft_locator)
self.assertEqual(new_index['edited_by'], 'leech_master')
- new_published_locator = CourseLocator(package_id=new_draft_locator.package_id, branch='published')
+ new_published_locator = new_draft_locator.course_key.for_branch("published")
new_published = modulestore().get_course(new_published_locator)
self.assertEqual(new_published.edited_by, 'test@edx.org')
self.assertEqual(new_published.location.version_guid, original_index['versions']['published'])
@@ -1365,7 +1406,7 @@ class TestCourseCreation(SplitModuleTest):
new_draft.location, 'chapter', 'leech_master',
fields={'display_name': 'new chapter'}
)
- new_draft_locator.version_guid = None
+ new_draft_locator = new_draft_locator.course_key.version_agnostic()
new_index = modulestore().get_course_index_info(new_draft_locator)
self.assertNotEqual(new_index['versions']['draft'], original_index['versions']['draft'])
new_draft = modulestore().get_course(new_draft_locator)
@@ -1377,18 +1418,12 @@ class TestCourseCreation(SplitModuleTest):
original_course = modulestore().get_course(original_locator)
self.assertEqual(original_course.location.version_guid, original_index['versions']['draft'])
- self.assertFalse(
- modulestore().has_item(new_draft_locator.package_id, BlockUsageLocator(
- original_locator,
- block_id=new_item.location.block_id
- ))
- )
def test_derived_course(self):
"""
Create a new course which overrides metadata and course_data
"""
- original_locator = CourseLocator(package_id="guestx.contender", branch='draft')
+ original_locator = CourseLocator(org='guestx', offering='contender', branch='draft')
original = modulestore().get_course(original_locator)
original_index = modulestore().get_course_index_info(original_locator)
fields = {}
@@ -1410,7 +1445,7 @@ class TestCourseCreation(SplitModuleTest):
fields=fields
)
new_draft_locator = new_draft.location
- self.assertRegexpMatches(new_draft_locator.package_id, 'counter')
+ self.assertRegexpMatches(new_draft_locator.org, 'counter')
# the edited_by and other meta fields on the new course will be the original author not this one
self.assertEqual(new_draft.edited_by, 'leech_master')
self.assertNotEqual(new_draft_locator.version_guid, original_index['versions']['draft'])
@@ -1425,19 +1460,12 @@ class TestCourseCreation(SplitModuleTest):
def test_update_course_index(self):
"""
- Test changing the org, pretty id, etc of a course. Test that it doesn't allow changing the id, etc.
+ Test the versions pointers. NOTE: you can change the org, offering, or other things, but
+ it's not clear how you'd find them again or associate them w/ existing student history since
+ we use course_key so many places as immutable.
"""
- locator = CourseLocator(package_id="testx.GreekHero", branch='draft')
- course_info = modulestore().get_course_index_info(locator)
- course_info['org'] = 'funkyU'
- modulestore().update_course_index(course_info)
- course_info = modulestore().get_course_index_info(locator)
- self.assertEqual(course_info['org'], 'funkyU')
-
- course_info['org'] = 'moreFunky'
- modulestore().update_course_index(course_info)
+ locator = CourseLocator(org='testx', offering='GreekHero', branch='draft')
course_info = modulestore().get_course_index_info(locator)
- self.assertEqual(course_info['org'], 'moreFunky')
# an allowed but not necessarily recommended way to revert the draft version
head_course = modulestore().get_course(locator)
@@ -1450,7 +1478,7 @@ class TestCourseCreation(SplitModuleTest):
# an allowed but not recommended way to publish a course
versions['published'] = versions['draft']
modulestore().update_course_index(course_info)
- course = modulestore().get_course(CourseLocator(package_id=locator.package_id, branch="published"))
+ course = modulestore().get_course(locator.for_branch("published"))
self.assertEqual(course.location.version_guid, versions['draft'])
def test_create_with_root(self):
@@ -1459,7 +1487,7 @@ class TestCourseCreation(SplitModuleTest):
"""
user = random.getrandbits(32)
new_course = modulestore().create_course(
- 'test_org.test_transaction', 'test_org', user,
+ 'test_org', 'test_transaction', user,
root_block_id='top', root_category='chapter'
)
self.assertEqual(new_course.location.block_id, 'top')
@@ -1480,7 +1508,8 @@ class TestCourseCreation(SplitModuleTest):
user = random.getrandbits(32)
courses = modulestore().get_courses()
with self.assertRaises(DuplicateCourseError):
- modulestore().create_course(courses[0].location.package_id, 'org', 'pretty', user)
+ dupe_course_key = courses[0].location.course_key
+ modulestore().create_course(dupe_course_key.org, dupe_course_key.offering, user)
class TestInheritance(SplitModuleTest):
@@ -1493,11 +1522,15 @@ class TestInheritance(SplitModuleTest):
"""
# Note, not testing value where defined (course) b/c there's no
# defined accessor for it on CourseDescriptor.
- locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="problem3_2", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'problem3_2'
+ )
node = modulestore().get_item(locator)
# inherited
self.assertEqual(node.graceperiod, datetime.timedelta(hours=2))
- locator = BlockUsageLocator(package_id="testx.GreekHero", block_id="problem1", branch='draft')
+ locator = BlockUsageLocator(
+ CourseLocator(org='testx', offering='GreekHero', branch='draft'), 'problem1'
+ )
node = modulestore().get_item(locator)
# overridden
self.assertEqual(node.graceperiod, datetime.timedelta(hours=4))
@@ -1518,8 +1551,8 @@ class TestPublish(SplitModuleTest):
"""
Test the standard patterns: publish to new branch, revise and publish
"""
- source_course = CourseLocator(package_id="testx.GreekHero", branch='draft')
- dest_course = CourseLocator(package_id="testx.GreekHero", branch="published")
+ source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
+ dest_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
modulestore().xblock_publish(self.user, source_course, dest_course, ["head12345"], ["chapter2", "chapter3"])
expected = ["head12345", "chapter1"]
self._check_course(
@@ -1560,13 +1593,13 @@ class TestPublish(SplitModuleTest):
"""
Test the exceptions which preclude successful publication
"""
- source_course = CourseLocator(package_id="testx.GreekHero", branch='draft')
+ source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
# destination does not exist
- destination_course = CourseLocator(package_id="Unknown", branch="published")
+ destination_course = CourseLocator(org='fake', offering='Unknown', branch="published")
with self.assertRaises(ItemNotFoundError):
modulestore().xblock_publish(self.user, source_course, destination_course, ["chapter3"], None)
# publishing into a new branch w/o publishing the root
- destination_course = CourseLocator(package_id="testx.GreekHero", branch="published")
+ destination_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
with self.assertRaises(ItemNotFoundError):
modulestore().xblock_publish(self.user, source_course, destination_course, ["chapter3"], None)
# publishing a subdag w/o the parent already in course
@@ -1578,8 +1611,8 @@ class TestPublish(SplitModuleTest):
"""
Test publishing moves and deletes.
"""
- source_course = CourseLocator(package_id="testx.GreekHero", branch='draft')
- dest_course = CourseLocator(package_id="testx.GreekHero", branch="published")
+ source_course = CourseLocator(org='testx', offering='GreekHero', branch='draft')
+ dest_course = CourseLocator(org='testx', offering='GreekHero', branch="published")
modulestore().xblock_publish(self.user, source_course, dest_course, ["head12345"], ["chapter2"])
expected = ["head12345", "chapter1", "chapter3", "problem1", "problem3_2"]
self._check_course(source_course, dest_course, expected, ["chapter2"])
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
new file mode 100644
index 00000000000..d4377b5bd8d
--- /dev/null
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
@@ -0,0 +1,137 @@
+import unittest
+import mock
+import datetime
+import uuid
+import random
+
+from xmodule.modulestore.inheritance import InheritanceMixin
+from xmodule.modulestore.locator import CourseLocator, BlockUsageLocator
+from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
+from xmodule.modulestore.mongo import MongoModuleStore, DraftMongoModuleStore
+from xmodule.modulestore.mongo.draft import DIRECT_ONLY_CATEGORIES
+
+
+class SplitWMongoCourseBoostrapper(unittest.TestCase):
+ """
+ Helper for tests which need to construct split mongo & old mongo based courses to get interesting internal structure.
+ Override _create_course and after invoking the super() _create_course, have it call _create_item for
+ each xblock you want in the course.
+ This class ensures the db gets created, opened, and cleaned up in addition to creating the course
+
+ Defines the following attrs on self:
+ * userid: a random non-registered mock user id
+ * split_mongo: a pointer to the split mongo instance
+ * old_mongo: a pointer to the old_mongo instance
+ * draft_mongo: a pointer to the old draft instance
+ * split_course_key (CourseLocator): of the new course
+ * old_course_key: the SlashSpecifiedCourseKey for the course
+ """
+ # Snippet of what would be in the django settings envs file
+ db_config = {
+ 'host': 'localhost',
+ 'db': 'test_xmodule',
+ }
+
+ modulestore_options = {
+ 'default_class': 'xmodule.raw_module.RawDescriptor',
+ 'fs_root': '',
+ 'render_template': mock.Mock(return_value=""),
+ 'xblock_mixins': (InheritanceMixin,)
+ }
+
+ split_course_key = CourseLocator('test_org', 'test_course.runid', branch='draft')
+
+ def setUp(self):
+ self.db_config['collection'] = 'modulestore{0}'.format(uuid.uuid4().hex[:5])
+
+ self.userid = random.getrandbits(32)
+ super(SplitWMongoCourseBoostrapper, self).setUp()
+ self.split_mongo = SplitMongoModuleStore(
+ self.db_config,
+ **self.modulestore_options
+ )
+ self.addCleanup(self.split_mongo.db.connection.close)
+ self.addCleanup(self.tear_down_split)
+ self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
+ self.draft_mongo = DraftMongoModuleStore(self.db_config, **self.modulestore_options)
+ self.addCleanup(self.tear_down_mongo)
+ self.old_course_key = None
+ self.runtime = None
+ self._create_course()
+
+ def tear_down_split(self):
+ """
+ Remove the test collections, close the db connection
+ """
+ split_db = self.split_mongo.db
+ split_db.drop_collection(split_db.course_index)
+ split_db.drop_collection(split_db.structures)
+ split_db.drop_collection(split_db.definitions)
+
+ def tear_down_mongo(self):
+ """
+ Remove the test collections, close the db connection
+ """
+ split_db = self.split_mongo.db
+ # old_mongo doesn't give a db attr, but all of the dbs are the same
+ split_db.drop_collection(self.old_mongo.collection)
+
+ def _create_item(self, category, name, data, metadata, parent_category, parent_name, draft=True, split=True):
+ """
+ Create the item of the given category and block id in split and old mongo, add it to the optional
+ parent. The parent category is only needed because old mongo requires it for the id.
+ """
+ location = self.old_course_key.make_usage_key(category, name)
+ if not draft or category in DIRECT_ONLY_CATEGORIES:
+ mongo = self.old_mongo
+ else:
+ mongo = self.draft_mongo
+ mongo.create_and_save_xmodule(location, data, metadata, self.runtime)
+ if isinstance(data, basestring):
+ fields = {'data': data}
+ else:
+ fields = data.copy()
+ fields.update(metadata)
+ if parent_name:
+ # add child to parent in mongo
+ parent_location = self.old_course_key.make_usage_key(parent_category, parent_name)
+ if not draft or parent_category in DIRECT_ONLY_CATEGORIES:
+ mongo = self.old_mongo
+ else:
+ mongo = self.draft_mongo
+ parent = mongo.get_item(parent_location)
+ parent.children.append(location)
+ mongo.update_item(parent, self.userid)
+ # create pointer for split
+ course_or_parent_locator = BlockUsageLocator(
+ course_key=self.split_course_key,
+ block_id=parent_name
+ )
+ else:
+ course_or_parent_locator = self.split_course_key
+ if split:
+ self.split_mongo.create_item(course_or_parent_locator, category, self.userid, block_id=name, fields=fields)
+
+ def _create_course(self, split=True):
+ """
+ * some detached items
+ * some attached children
+ * some orphans
+ """
+ metadata = {
+ 'start': datetime.datetime(2000, 3, 13, 4),
+ 'display_name': 'Migration test course',
+ }
+ data = {
+ 'wiki_slug': 'test_course_slug'
+ }
+ fields = metadata.copy()
+ fields.update(data)
+ if split:
+ # split requires the course to be created separately from creating items
+ self.split_mongo.create_course(
+ self.split_course_key.org, self.split_course_key.offering, self.userid, fields=fields, root_block_id='runid'
+ )
+ old_course = self.old_mongo.create_course(self.split_course_key.org, 'test_course/runid', fields=fields)
+ self.old_course_key = old_course.id
+ self.runtime = old_course.runtime
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py b/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
index 8cbdbbeffe8..e0facc92ee0 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
@@ -7,12 +7,13 @@ import unittest
from glob import glob
from mock import patch
-from xmodule.course_module import CourseDescriptor
from xmodule.modulestore.xml import XMLModuleStore
from xmodule.modulestore import Location, XML_MODULESTORE_TYPE
from .test_modulestore import check_path_to_location
from xmodule.tests import DATA_DIR
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
+from xmodule.modulestore.tests.test_modulestore import check_has_course_method
def glob_tildes_at_end(path):
@@ -58,22 +59,16 @@ class TestXMLModuleStore(unittest.TestCase):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'], load_error_modules=False)
# Look up the errors during load. There should be none.
- location = CourseDescriptor.id_to_location("edX/toy/2012_Fall")
- errors = modulestore.get_item_errors(location)
+ errors = modulestore.get_course_errors(SlashSeparatedCourseKey("edX", "toy", "2012_Fall"))
assert errors == []
@patch("xmodule.modulestore.xml.glob.glob", side_effect=glob_tildes_at_end)
def test_tilde_files_ignored(self, _fake_glob):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['tilde'], load_error_modules=False)
- course_module = modulestore.modules['edX/tilde/2012_Fall']
- about_location = Location({
- 'tag': 'i4x',
- 'org': 'edX',
- 'course': 'tilde',
- 'category': 'about',
- 'name': 'index',
- })
- about_module = course_module[about_location]
+ about_location = SlashSeparatedCourseKey('edX', 'tilde', '2012_Fall').make_usage_key(
+ 'about', 'index',
+ )
+ about_module = modulestore.get_item(about_location)
self.assertIn("GREEN", about_module.data)
self.assertNotIn("RED", about_module.data)
@@ -85,13 +80,13 @@ class TestXMLModuleStore(unittest.TestCase):
for course in store.get_courses():
course_locations = store.get_courses_for_wiki(course.wiki_slug)
self.assertEqual(len(course_locations), 1)
- self.assertIn(Location('i4x', 'edX', course.location.course, 'course', '2012_Fall'), course_locations)
+ self.assertIn(course.location, course_locations)
course_locations = store.get_courses_for_wiki('no_such_wiki')
self.assertEqual(len(course_locations), 0)
# now set toy course to share the wiki with simple course
- toy_course = store.get_course('edX/toy/2012_Fall')
+ toy_course = store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
toy_course.wiki_slug = 'simple'
course_locations = store.get_courses_for_wiki('toy')
@@ -100,4 +95,14 @@ class TestXMLModuleStore(unittest.TestCase):
course_locations = store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 2)
for course_number in ['toy', 'simple']:
- self.assertIn(Location('i4x', 'edX', course_number, 'course', '2012_Fall'), course_locations)
+ self.assertIn(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations)
+
+ def test_has_course(self):
+ """
+ Test the has_course method
+ """
+ check_has_course_method(
+ XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple']),
+ SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'),
+ locator_key_fields=SlashSeparatedCourseKey.KEY_FIELDS
+ )
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py b/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
index 9e21a1a6364..d39c81b43bc 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
@@ -1,7 +1,6 @@
"""
Tests for XML importer.
"""
-from unittest import TestCase
import mock
from xblock.core import XBlock
from xblock.fields import String, Scope, ScopeIds
@@ -9,7 +8,93 @@ from xblock.runtime import Runtime, KvsFieldData, DictKeyValueStore
from xmodule.x_module import XModuleMixin
from xmodule.modulestore import Location
from xmodule.modulestore.inheritance import InheritanceMixin
-from xmodule.modulestore.xml_importer import remap_namespace
+from xmodule.modulestore.xml_importer import import_module
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
+from xmodule.tests import DATA_DIR
+from uuid import uuid4
+import unittest
+import importlib
+
+
+class ModuleStoreNoSettings(unittest.TestCase):
+ """
+ A mixin to create a mongo modulestore that avoids settings
+ """
+ HOST = 'localhost'
+ PORT = 27017
+ DB = 'test_mongo_%s' % uuid4().hex[:5]
+ COLLECTION = 'modulestore'
+ FS_ROOT = DATA_DIR
+ DEFAULT_CLASS = 'xmodule.modulestore.tests.test_xml_importer.StubXBlock'
+ RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
+
+ modulestore_options = {
+ 'default_class': DEFAULT_CLASS,
+ 'fs_root': DATA_DIR,
+ 'render_template': RENDER_TEMPLATE,
+ }
+ DOC_STORE_CONFIG = {
+ 'host': HOST,
+ 'db': DB,
+ 'collection': COLLECTION,
+ }
+ MODULESTORE = {
+ 'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+ 'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
+ 'OPTIONS': modulestore_options
+ }
+
+ modulestore = None
+
+ def cleanup_modulestore(self):
+ """
+ cleanup
+ """
+ if modulestore:
+ connection = self.modulestore.database.connection
+ connection.drop_database(self.modulestore.database)
+ connection.close()
+
+ def setUp(self):
+ """
+ Add cleanups
+ """
+ self.addCleanup(self.cleanup_modulestore)
+ super(ModuleStoreNoSettings, self).setUp()
+
+
+#===========================================
+def modulestore():
+ """
+ Mock the django dependent global modulestore function to disentangle tests from django
+ """
+ def load_function(engine_path):
+ """
+ Load the given engine
+ """
+ module_path, _, name = engine_path.rpartition('.')
+ return getattr(importlib.import_module(module_path), name)
+
+ if ModuleStoreNoSettings.modulestore is None:
+ class_ = load_function(ModuleStoreNoSettings.MODULESTORE['ENGINE'])
+
+ options = {}
+
+ options.update(ModuleStoreNoSettings.MODULESTORE['OPTIONS'])
+ options['render_template'] = render_to_template_mock
+
+ # pylint: disable=W0142
+ ModuleStoreNoSettings.modulestore = class_(
+ ModuleStoreNoSettings.MODULESTORE['DOC_STORE_CONFIG'],
+ **options
+ )
+
+ return ModuleStoreNoSettings.modulestore
+
+
+# pylint: disable=W0613
+def render_to_template_mock(*args):
+ pass
class StubXBlock(XBlock, XModuleMixin, InheritanceMixin):
@@ -29,7 +114,7 @@ class StubXBlock(XBlock, XModuleMixin, InheritanceMixin):
)
-class RemapNamespaceTest(TestCase):
+class RemapNamespaceTest(ModuleStoreNoSettings):
"""
Test that remapping the namespace from import to the actual course location.
"""
@@ -42,81 +127,99 @@ class RemapNamespaceTest(TestCase):
self.field_data = KvsFieldData(kvs=DictKeyValueStore())
self.scope_ids = ScopeIds('Bob', 'stubxblock', '123', 'import')
self.xblock = StubXBlock(self.runtime, self.field_data, self.scope_ids)
+ super(RemapNamespaceTest, self).setUp()
def test_remap_namespace_native_xblock(self):
# Set the XBlock's location
- self.xblock.location = Location("i4x://import/org/run/stubxblock")
+ self.xblock.location = Location("org", "import", "run", "category", "stubxblock")
# Explicitly set the content and settings fields
self.xblock.test_content_field = "Explicitly set"
self.xblock.test_settings_field = "Explicitly set"
self.xblock.save()
- # Remap the namespace
- target_location_namespace = Location("i4x://course/org/run/stubxblock")
- remap_namespace(self.xblock, target_location_namespace)
+ # Move to different runtime w/ different course id
+ target_location_namespace = SlashSeparatedCourseKey("org", "course", "run")
+ new_version = import_module(
+ self.xblock,
+ modulestore(),
+ self.xblock.location.course_key,
+ target_location_namespace,
+ do_import_static=False
+ )
# Check the XBlock's location
- self.assertEqual(self.xblock.location, target_location_namespace)
+ self.assertEqual(new_version.location.course_key, target_location_namespace)
# Check the values of the fields.
# The content and settings fields should be preserved
- self.assertEqual(self.xblock.test_content_field, 'Explicitly set')
- self.assertEqual(self.xblock.test_settings_field, 'Explicitly set')
+ self.assertEqual(new_version.test_content_field, 'Explicitly set')
+ self.assertEqual(new_version.test_settings_field, 'Explicitly set')
# Expect that these fields are marked explicitly set
self.assertIn(
'test_content_field',
- self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.content)
+ new_version.get_explicitly_set_fields_by_scope(scope=Scope.content)
)
self.assertIn(
'test_settings_field',
- self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.settings)
+ new_version.get_explicitly_set_fields_by_scope(scope=Scope.settings)
)
def test_remap_namespace_native_xblock_default_values(self):
# Set the XBlock's location
- self.xblock.location = Location("i4x://import/org/run/stubxblock")
+ self.xblock.location = Location("org", "import", "run", "category", "stubxblock")
# Do NOT set any values, so the fields should use the defaults
self.xblock.save()
# Remap the namespace
- target_location_namespace = Location("i4x://course/org/run/stubxblock")
- remap_namespace(self.xblock, target_location_namespace)
+ target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
+ new_version = import_module(
+ self.xblock,
+ modulestore(),
+ self.xblock.location.course_key,
+ target_location_namespace.course_key,
+ do_import_static=False
+ )
# Check the values of the fields.
# The content and settings fields should be the default values
- self.assertEqual(self.xblock.test_content_field, 'default value')
- self.assertEqual(self.xblock.test_settings_field, 'default value')
+ self.assertEqual(new_version.test_content_field, 'default value')
+ self.assertEqual(new_version.test_settings_field, 'default value')
# The fields should NOT appear in the explicitly set fields
self.assertNotIn(
'test_content_field',
- self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.content)
+ new_version.get_explicitly_set_fields_by_scope(scope=Scope.content)
)
self.assertNotIn(
'test_settings_field',
- self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.settings)
+ new_version.get_explicitly_set_fields_by_scope(scope=Scope.settings)
)
def test_remap_namespace_native_xblock_inherited_values(self):
# Set the XBlock's location
- self.xblock.location = Location("i4x://import/org/run/stubxblock")
+ self.xblock.location = Location("org", "import", "run", "category", "stubxblock")
self.xblock.save()
# Remap the namespace
- target_location_namespace = Location("i4x://course/org/run/stubxblock")
- remap_namespace(self.xblock, target_location_namespace)
+ target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
+ new_version = import_module(
+ self.xblock,
+ modulestore(),
+ self.xblock.location.course_key,
+ target_location_namespace.course_key,
+ do_import_static=False
+ )
# Inherited fields should NOT be explicitly set
self.assertNotIn(
- 'start', self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.settings)
+ 'start', new_version.get_explicitly_set_fields_by_scope(scope=Scope.settings)
)
self.assertNotIn(
- 'graded', self.xblock.get_explicitly_set_fields_by_scope(scope=Scope.settings)
+ 'graded', new_version.get_explicitly_set_fields_by_scope(scope=Scope.settings)
)
-
diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py
index 304f5f1c0c8..160097428f3 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml.py
@@ -16,21 +16,23 @@ from path import path
from xmodule.error_module import ErrorDescriptor
from xmodule.errortracker import make_error_tracker, exc_info_to_str
-from xmodule.course_module import CourseDescriptor
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.x_module import XMLParsingSystem, policy_key
from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS
from xmodule.tabs import CourseTabList
+from xmodule.modulestore.keys import UsageKey
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
-from xblock.fields import ScopeIds
from xblock.field_data import DictFieldData
-from xblock.runtime import DictKeyValueStore, IdReader, IdGenerator
+from xblock.runtime import DictKeyValueStore, IdGenerator
from . import ModuleStoreReadBase, Location, XML_MODULESTORE_TYPE
from .exceptions import ItemNotFoundError
from .inheritance import compute_inherited_metadata, inheriting_field_data
+from xblock.fields import ScopeIds, Reference, ReferenceList, ReferenceValueDict
+
edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
remove_comments=True, remove_blank_text=True)
@@ -51,7 +53,7 @@ def clean_out_mako_templating(xml_string):
class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
def __init__(self, xmlstore, course_id, course_dir,
error_tracker, parent_tracker,
- load_error_modules=True, id_reader=None, **kwargs):
+ load_error_modules=True, **kwargs):
"""
A class that handles loading from xml. Does some munging to ensure that
all elements have unique slugs.
@@ -60,13 +62,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
"""
self.unnamed = defaultdict(int) # category -> num of new url_names for that category
self.used_names = defaultdict(set) # category -> set of used url_names
- course_id_dict = Location.parse_course_id(course_id)
- self.org = course_id_dict['org']
- self.course = course_id_dict['course']
- self.url_name = course_id_dict['name']
- if id_reader is None:
- id_reader = LocationReader()
- id_generator = CourseLocationGenerator(self.org, self.course)
+ id_generator = CourseLocationGenerator(course_id)
# cdodge: adding the course_id as passed in for later reference rather than having to recomine the org/course/url_name
self.course_id = course_id
@@ -178,7 +174,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
self,
id_generator,
)
- except Exception as err:
+ except Exception as err: # pylint: disable=broad-except
if not self.load_error_modules:
raise
@@ -224,9 +220,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
# TODO (vshnayder): we are somewhat architecturally confused in the loading code:
# load_item should actually be get_instance, because it expects the course-specific
# policy to be loaded. For now, just add the course_id here...
- def load_item(location):
+ def load_item(usage_key):
"""Return the XBlock for the specified location"""
- return xmlstore.get_instance(course_id, Location(location))
+ return xmlstore.get_item(usage_key)
resources_fs = OSFS(xmlstore.data_dir / course_dir)
@@ -236,7 +232,6 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
render_template=render_template,
error_tracker=error_tracker,
process_xml=process_xml,
- id_reader=id_reader,
**kwargs
)
@@ -247,37 +242,53 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
block.children.append(child_block.scope_ids.usage_id)
-class LocationReader(IdReader):
- """
- IdReader for definition and usage ids that are Locations
- """
- def get_definition_id(self, usage_id):
- return usage_id
-
- def get_block_type(self, def_id):
- location = def_id
- return location.category
-
-
class CourseLocationGenerator(IdGenerator):
"""
IdGenerator for Location-based definition ids and usage ids
based within a course
"""
- def __init__(self, org, course):
- self.org = org
- self.course = course
+ def __init__(self, course_id):
+ self.course_id = course_id
self.autogen_ids = itertools.count(0)
def create_usage(self, def_id):
- return Location(def_id)
+ return def_id
def create_definition(self, block_type, slug=None):
assert block_type is not None
if slug is None:
slug = 'autogen_{}_{}'.format(block_type, self.autogen_ids.next())
- location = Location('i4x', self.org, self.course, block_type, slug)
- return location
+ return self.course_id.make_usage_key(block_type, slug)
+
+
+def _make_usage_key(course_key, value):
+ """
+ Makes value into a UsageKey inside the specified course.
+ If value is already a UsageKey, returns that.
+ """
+ if isinstance(value, UsageKey):
+ return value
+ return course_key.make_usage_key_from_deprecated_string(value)
+
+
+def _convert_reference_fields_to_keys(xblock): # pylint: disable=invalid-name
+ """
+ Find all fields of type reference and convert the payload into UsageKeys
+ """
+ course_key = xblock.scope_ids.usage_id.course_key
+
+ for field in xblock.fields.itervalues():
+ if field.is_set_on(xblock):
+ field_value = getattr(xblock, field.name)
+ if isinstance(field, Reference):
+ setattr(xblock, field.name, _make_usage_key(course_key, field_value))
+ elif isinstance(field, ReferenceList):
+ setattr(xblock, field.name, [_make_usage_key(course_key, ele) for ele in field_value])
+ elif isinstance(field, ReferenceValueDict):
+ for key, subvalue in field_value.iteritems():
+ assert isinstance(subvalue, basestring)
+ field_value[key] = _make_usage_key(course_key, subvalue)
+ setattr(xblock, field.name, field_value)
def create_block_from_xml(xml_data, system, id_generator):
@@ -309,6 +320,9 @@ def create_block_from_xml(xml_data, system, id_generator):
scope_ids = ScopeIds(None, block_type, def_id, usage_id)
xblock = xblock_class.parse_xml(node, system, scope_ids, id_generator)
+
+ _convert_reference_fields_to_keys(xblock)
+
return xblock
@@ -327,8 +341,8 @@ class ParentTracker(object):
child and parent must be :class:`.Location` instances.
"""
- s = self._parents.setdefault(child, set())
- s.add(parent)
+ setp = self._parents.setdefault(child, set())
+ setp.add(parent)
def is_known(self, child):
"""
@@ -359,13 +373,14 @@ class XMLModuleStore(ModuleStoreReadBase):
"""
Initialize an XMLModuleStore from data_dir
- data_dir: path to data directory containing the course directories
+ Args:
+ data_dir (str): path to data directory containing the course directories
- default_class: dot-separated string defining the default descriptor
- class to use if none is specified in entry_points
+ default_class (str): dot-separated string defining the default descriptor
+ class to use if none is specified in entry_points
- course_dirs or course_ids: If specified, the list of course_dirs or course_ids to load. Otherwise,
- load all courses. Note, providing both
+ course_dirs or course_ids (list of str): If specified, the list of course_dirs or course_ids to load. Otherwise,
+ load all courses. Note, providing both
"""
super(XMLModuleStore, self).__init__(**kwargs)
@@ -374,6 +389,9 @@ class XMLModuleStore(ModuleStoreReadBase):
self.courses = {} # course_dir -> XBlock for the course
self.errored_courses = {} # course_dir -> errorlog, for dirs that failed to load
+ if course_ids is not None:
+ course_ids = [SlashSeparatedCourseKey.from_deprecated_string(course_id) for course_id in course_ids]
+
self.load_error_modules = load_error_modules
if default_class is None:
@@ -415,9 +433,9 @@ class XMLModuleStore(ModuleStoreReadBase):
course_descriptor = None
try:
course_descriptor = self.load_course(course_dir, course_ids, errorlog.tracker)
- except Exception as e:
+ except Exception as exc: # pylint: disable=broad-except
msg = "ERROR: Failed to load course '{0}': {1}".format(
- course_dir.encode("utf-8"), unicode(e)
+ course_dir.encode("utf-8"), unicode(exc)
)
log.exception(msg)
errorlog.tracker(msg)
@@ -430,7 +448,7 @@ class XMLModuleStore(ModuleStoreReadBase):
self.errored_courses[course_dir] = errorlog
else:
self.courses[course_dir] = course_descriptor
- self._location_errors[course_descriptor.scope_ids.usage_id] = errorlog
+ self._course_errors[course_descriptor.id] = errorlog
self.parent_trackers[course_descriptor.id].make_known(course_descriptor.scope_ids.usage_id)
def __unicode__(self):
@@ -521,7 +539,7 @@ class XMLModuleStore(ModuleStoreReadBase):
raise ValueError("Can't load a course without a 'url_name' "
"(or 'name') set. Set url_name.")
- course_id = CourseDescriptor.make_id(org, course, url_name)
+ course_id = SlashSeparatedCourseKey(org, course, url_name)
if course_ids is not None and course_id not in course_ids:
return None
@@ -670,42 +688,20 @@ class XMLModuleStore(ModuleStoreReadBase):
module.save()
self.modules[course_descriptor.id][module.scope_ids.usage_id] = module
- except Exception, e:
+ except Exception as exc: # pylint: disable=broad-except
logging.exception("Failed to load %s. Skipping... \
- Exception: %s", filepath, unicode(e))
- system.error_tracker("ERROR: " + unicode(e))
-
- def get_instance(self, course_id, location, depth=0):
- """
- Returns an XBlock instance for the item at
- location, with the policy for course_id. (In case two xml
- dirs have different content at the same location, return the
- one for this course_id.)
+ Exception: %s", filepath, unicode(exc))
+ system.error_tracker("ERROR: " + unicode(exc))
- If any segment of the location is None except revision, raises
- xmodule.modulestore.exceptions.InsufficientSpecificationError
-
- If no object is found at that location, raises
- xmodule.modulestore.exceptions.ItemNotFoundError
-
- location: Something that can be passed to Location
- """
- location = Location(location)
- try:
- return self.modules[course_id][location]
- except KeyError:
- raise ItemNotFoundError(location)
-
- def has_item(self, course_id, location):
+ def has_item(self, usage_key):
"""
Returns True if location exists in this ModuleStore.
"""
- location = Location(location)
- return location in self.modules[course_id]
+ return usage_key in self.modules[usage_key.course_key]
- def get_item(self, location, depth=0):
+ def get_item(self, usage_key, depth=0):
"""
- Returns an XBlock instance for the item at location.
+ Returns an XBlock instance for the item for this UsageKey.
If any segment of the location is None except revision, raises
xmodule.modulestore.exceptions.InsufficientSpecificationError
@@ -713,26 +709,56 @@ class XMLModuleStore(ModuleStoreReadBase):
If no object is found at that location, raises
xmodule.modulestore.exceptions.ItemNotFoundError
- location: Something that can be passed to Location
+ usage_key: a UsageKey that matches the module we are looking for.
+ """
+ try:
+ return self.modules[usage_key.course_key][usage_key]
+ except KeyError:
+ raise ItemNotFoundError(usage_key)
+
+ def get_items(self, course_id, settings=None, content=None, **kwargs):
+ """
+ Returns:
+ list of XModuleDescriptor instances for the matching items within the course with
+ the given course_id
+
+ NOTE: don't use this to look for courses
+ as the course_id is required. Use get_courses.
+
+ Args:
+ course_id (CourseKey): the course identifier
+ settings (dict): fields to look for which have settings scope. Follows same syntax
+ and rules as kwargs below
+ content (dict): fields to look for which have content scope. Follows same syntax and
+ rules as kwargs below.
+ kwargs (key=value): what to look for within the course.
+ Common qualifiers are ``category`` or any field name. if the target field is a list,
+ then it searches for the given value in the list not list equivalence.
+ Substring matching pass a regex object.
+ For this modulestore, ``name`` is another commonly provided key (Location based stores)
+ (but not revision!)
+ For this modulestore,
+ you can search dates by providing either a datetime for == (probably
+ useless) or a tuple (">"|"<" datetime) for after or before, etc.
"""
- raise NotImplementedError("XMLModuleStores can't guarantee that definitions"
- " are unique. Use get_instance.")
-
- def get_items(self, location, course_id=None, depth=0, qualifiers=None):
items = []
- def _add_get_items(self, location, modules):
- for mod_loc, module in modules.iteritems():
- # Locations match if each value in `location` is None or if the value from `location`
- # matches the value from `mod_loc`
- if all(goal is None or goal == value for goal, value in zip(location, mod_loc)):
- items.append(module)
+ category = kwargs.pop('category', None)
+ name = kwargs.pop('name', None)
+
+ def _block_matches_all(mod_loc, module):
+ if category and mod_loc.category != category:
+ return False
+ if name and mod_loc.name != name:
+ return False
+ return all(
+ self._block_matches(module, fields or {})
+ for fields in [settings, content, kwargs]
+ )
- if course_id is None:
- for _, modules in self.modules.iteritems():
- _add_get_items(self, location, modules)
- else:
- _add_get_items(self, location, self.modules[course_id])
+ for mod_loc, module in self.modules[course_id].iteritems():
+ if _block_matches_all(mod_loc, module):
+ items.append(module)
return items
@@ -750,7 +776,7 @@ class XMLModuleStore(ModuleStoreReadBase):
"""
return dict((k, self.errored_courses[k].errors) for k in self.errored_courses)
- def get_orphans(self, course_location, _branch):
+ def get_orphans(self, course_key):
"""
Get all of the xblocks in the given course which have no parents and are not of types which are
usually orphaned. NOTE: may include xblocks which still have references via xblocks which don't
@@ -759,28 +785,17 @@ class XMLModuleStore(ModuleStoreReadBase):
# here just to quell the abstractmethod. someone could write the impl if needed
raise NotImplementedError
- def update_item(self, xblock, user, **kwargs):
- """
- Set the data in the item specified by the location to
- data
-
- location: Something that can be passed to Location
- data: A nested dictionary of problem data
- """
- raise NotImplementedError("XMLModuleStores are read-only")
-
- def get_parent_locations(self, location, course_id):
+ def get_parent_locations(self, location):
'''Find all locations that are the parents of this location in this
course. Needed for path_to_location().
returns an iterable of things that can be passed to Location. This may
be empty if there are no parents.
'''
- location = Location.ensure_fully_specified(location)
- if not self.parent_trackers[course_id].is_known(location):
- raise ItemNotFoundError("{0} not in {1}".format(location, course_id))
+ if not self.parent_trackers[location.course_key].is_known(location):
+ raise ItemNotFoundError("{0} not in {1}".format(location, location.course_key))
- return self.parent_trackers[course_id].parents(location)
+ return self.parent_trackers[location.course_key].parents(location)
def get_modulestore_type(self, course_id):
"""
diff --git a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
index b442e2ef965..c4ef90154f5 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
@@ -32,7 +32,7 @@ class EdxJSONEncoder(json.JSONEncoder):
"""
def default(self, obj):
if isinstance(obj, Location):
- return obj.url()
+ return obj.to_deprecated_string()
elif isinstance(obj, datetime.datetime):
if obj.tzinfo is not None:
if obj.utcoffset() is None:
@@ -45,24 +45,23 @@ class EdxJSONEncoder(json.JSONEncoder):
return super(EdxJSONEncoder, self).default(obj)
-def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
+def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, draft_modulestore=None):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
- `course_location`: The `Location` of the `CourseModuleDescriptor` to export
+ `course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
`draft_modulestore`: An optional `DraftModuleStore` that contains draft content, which will be exported
alongside the public content in the course.
"""
- course_id = course_location.course_id
- course = modulestore.get_course(course_id)
+ course = modulestore.get_course(course_key)
- fs = OSFS(root_dir)
- export_fs = course.runtime.export_fs = fs.makeopendir(course_dir)
+ fsm = OSFS(root_dir)
+ export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element('unknown')
course.add_xml_to_node(root)
@@ -74,22 +73,22 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
policies_dir = export_fs.makeopendir('policies')
if contentstore:
contentstore.export_all_for_course(
- course_location,
+ course_key,
root_dir + '/' + course_dir + '/static/',
root_dir + '/' + course_dir + '/policies/assets.json',
)
# export the static tabs
- export_extra_content(export_fs, modulestore, course_id, course_location, 'static_tab', 'tabs', '.html')
+ export_extra_content(export_fs, modulestore, course_key, 'static_tab', 'tabs', '.html')
# export the custom tags
- export_extra_content(export_fs, modulestore, course_id, course_location, 'custom_tag_template', 'custom_tags')
+ export_extra_content(export_fs, modulestore, course_key, 'custom_tag_template', 'custom_tags')
# export the course updates
- export_extra_content(export_fs, modulestore, course_id, course_location, 'course_info', 'info', '.html')
+ export_extra_content(export_fs, modulestore, course_key, 'course_info', 'info', '.html')
# export the 'about' data (e.g. overview, etc.)
- export_extra_content(export_fs, modulestore, course_id, course_location, 'about', 'about', '.html')
+ export_extra_content(export_fs, modulestore, course_key, 'about', 'about', '.html')
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
@@ -106,18 +105,17 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
- draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course,
- 'vertical', None, 'draft'])
+ draft_verticals = draft_modulestore.get_items(course_key, category='vertical')
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
- parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
+ parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug('parent_locs = {0}'.format(parent_locs))
- draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url()
- sequential = modulestore.get_item(Location(parent_locs[0]))
- index = sequential.children.index(draft_vertical.location.url())
+ draft_vertical.xml_attributes['parent_sequential_url'] = parent_locs[0].to_deprecated_string()
+ sequential = modulestore.get_item(parent_locs[0])
+ index = sequential.children.index(draft_vertical.location)
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element('unknown')
@@ -138,9 +136,8 @@ def _export_field_content(xblock_item, item_dir):
field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder))
-def export_extra_content(export_fs, modulestore, course_id, course_location, category_type, dirname, file_suffix=''):
- query_loc = Location('i4x', course_location.org, course_location.course, category_type, None)
- items = modulestore.get_items(query_loc, course_id)
+def export_extra_content(export_fs, modulestore, course_key, category_type, dirname, file_suffix=''):
+ items = modulestore.get_items(course_key, category=category_type)
if len(items) > 0:
item_dir = export_fs.makeopendir(dirname)
diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py
index 2902d3f6f81..488a4f8650b 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py
@@ -5,22 +5,23 @@ from path import path
import json
from .xml import XMLModuleStore, ImportSystem, ParentTracker
-from xmodule.modulestore import Location
from xblock.runtime import KvsFieldData, DictKeyValueStore
from xmodule.x_module import XModuleDescriptor
+from xmodule.modulestore.keys import UsageKey
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xmodule.contentstore.content import StaticContent
from .inheritance import own_metadata
from xmodule.errortracker import make_error_tracker
from .store_utilities import rewrite_nonportable_content_links
import xblock
+from xmodule.tabs import CourseTabList
log = logging.getLogger(__name__)
def import_static_content(
- modules, course_loc, course_data_path, static_content_store,
- target_location_namespace, subpath='static', verbose=False):
+ course_data_path, static_content_store,
+ target_course_id, subpath='static', verbose=False):
remap_dict = {}
@@ -65,12 +66,9 @@ def import_static_content(
fullname_with_subpath = content_path.replace(static_dir, '')
if fullname_with_subpath.startswith('/'):
fullname_with_subpath = fullname_with_subpath[1:]
- content_loc = StaticContent.compute_location(
- target_location_namespace.org, target_location_namespace.course,
- fullname_with_subpath
- )
+ asset_key = StaticContent.compute_location(target_course_id, fullname_with_subpath)
- policy_ele = policy.get(content_loc.name, {})
+ policy_ele = policy.get(asset_key.path, {})
displayname = policy_ele.get('displayname', filename)
locked = policy_ele.get('locked', False)
mime_type = policy_ele.get('contentType')
@@ -79,7 +77,7 @@ def import_static_content(
if not mime_type or mime_type not in mimetypes_list:
mime_type = mimetypes.guess_type(filename)[0] # Assign guessed mimetype
content = StaticContent(
- content_loc, displayname, mime_type, data,
+ asset_key, displayname, mime_type, data,
import_path=fullname_with_subpath, locked=locked
)
@@ -99,7 +97,7 @@ def import_static_content(
# store the remapping information which will be needed
# to subsitute in the module data
- remap_dict[fullname_with_subpath] = content_loc.name
+ remap_dict[fullname_with_subpath] = asset_key
return remap_dict
@@ -108,8 +106,8 @@ def import_from_xml(
store, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None,
- target_location_namespace=None, verbose=False, draft_store=None,
- do_import_static=True):
+ target_course_id=None, verbose=False, draft_store=None,
+ do_import_static=True, create_new_course=False):
"""
Import the specified xml data_dir into the "store" modulestore,
using org and course as the location org and course.
@@ -117,8 +115,7 @@ def import_from_xml(
course_dirs: If specified, the list of course_dirs to load. Otherwise, load
all course dirs
- target_location_namespace is the namespace [passed as Location]
- (i.e. {tag},{org},{course}) that all modules in the should be remapped to
+ target_course_id is the CourseKey that all modules should be remapped to
after import off disk. We do this remapping as a post-processing step
because there's logic in the importing which expects a 'url_name' as an
identifier to where things are on disk
@@ -132,6 +129,9 @@ def import_from_xml(
time the course is loaded. Static content for some courses may also be
served directly by nginx, instead of going through django.
+ : create_new_course:
+ If True, then courses whose ids already exist in the store are not imported.
+ The check for existing courses is case-insensitive.
"""
xml_module_store = XMLModuleStore(
@@ -143,6 +143,12 @@ def import_from_xml(
xblock_select=store.xblock_select,
)
+ # If we're going to remap the course_id, then we can only do that with
+ # a single course
+
+ if target_course_id:
+ assert(len(xml_module_store.modules) == 1)
+
# NOTE: the XmlModuleStore does not implement get_items()
# which would be a preferable means to enumerate the entire collection
# of course modules. It will be left as a TBD to implement that
@@ -150,21 +156,28 @@ def import_from_xml(
course_items = []
for course_id in xml_module_store.modules.keys():
- if target_location_namespace is not None:
- pseudo_course_id = u'{0.org}/{0.course}'.format(target_location_namespace)
+ if target_course_id is not None:
+ dest_course_id = target_course_id
else:
- course_id_components = Location.parse_course_id(course_id)
- pseudo_course_id = u'{org}/{course}'.format(**course_id_components)
+ dest_course_id = course_id
+
+ if create_new_course:
+ if store.has_course(dest_course_id, ignore_case=True):
+ log.debug(
+ "Skipping import of course with id, {0},"
+ "since it collides with an existing one".format(dest_course_id)
+ )
+ continue
+ else:
+ store.create_course(dest_course_id.org, dest_course_id.offering)
try:
# turn off all write signalling while importing as this
# is a high volume operation on stores that need it
- if (hasattr(store, 'ignore_write_events_on_courses') and
- pseudo_course_id not in store.ignore_write_events_on_courses):
- store.ignore_write_events_on_courses.append(pseudo_course_id)
+ if hasattr(store, 'ignore_write_events_on_courses'):
+ store.ignore_write_events_on_courses.add(dest_course_id)
course_data_path = None
- course_location = None
if verbose:
log.debug("Scanning {0} for course module...".format(course_id))
@@ -175,39 +188,10 @@ def import_from_xml(
for module in xml_module_store.modules[course_id].itervalues():
if module.scope_ids.block_type == 'course':
course_data_path = path(data_dir) / module.data_dir
- course_location = module.location
- course_prefix = u'{0.org}/{0.course}'.format(course_location)
-
- # Check to see if a course with the same
- # pseudo_course_id, but different run exists in
- # the passed store to avoid broken courses
- courses = store.get_courses()
- bad_run = False
- for course in courses:
- if course.location.course_id.startswith(course_prefix):
- log.debug('Import is overwriting existing course')
- # Importing over existing course, check
- # that runs match or fail
- if course.location.name != module.location.name:
- log.error(
- 'A course with ID %s exists, and this '
- 'course has the same organization and '
- 'course number, but a different term that '
- 'is fully identified as %s.',
- course.location.course_id,
- module.location.course_id
- )
- bad_run = True
- break
- if bad_run:
- # Skip this course, but keep trying to import courses
- continue
-
- log.debug('======> IMPORTING course to location {loc}'.format(
- loc=course_location
- ))
- module = remap_namespace(module, target_location_namespace)
+ log.debug(u'======> IMPORTING course {course_id}'.format(
+ course_id=course_id,
+ ))
if not do_import_static:
# for old-style xblock where this was actually linked to kvs
@@ -219,6 +203,35 @@ def import_from_xml(
log.debug('course data_dir={0}'.format(module.data_dir))
+ course = import_module(
+ module, store,
+ course_id,
+ dest_course_id,
+ do_import_static=do_import_static
+ )
+
+ for entry in course.pdf_textbooks:
+ for chapter in entry.get('chapters', []):
+ if StaticContent.is_c4x_path(chapter.get('url', '')):
+ asset_key = StaticContent.get_location_from_path(chapter['url'])
+ chapter['url'] = StaticContent.get_static_path_from_location(asset_key)
+
+ # Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
+ # If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
+ # values then remap it so that the wiki does not point to the old wiki.
+ if course_id != course.id:
+ original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
+ course_id.org,
+ course_id.course,
+ course_id.run
+ )
+ if course.wiki_slug == original_unique_wiki_slug or course.wiki_slug == course_id.course:
+ course.wiki_slug = u'{0}.{1}.{2}'.format(
+ course.id.org,
+ course.id.course,
+ course.id.run,
+ )
+
# cdodge: more hacks (what else). Seems like we have a
# problem when importing a course (like 6.002) which
# does not have any tabs defined in the policy file.
@@ -227,36 +240,19 @@ def import_from_xml(
# the LMS barfs because it expects that -- if there are
# *any* tabs -- then there at least needs to be
# some predefined ones
- if module.tabs is None or len(module.tabs) == 0:
- module.tabs = [
- {"type": "courseware"},
- {"type": "course_info", "name": "Course Info"},
- {"type": "discussion", "name": "Discussion"},
- {"type": "wiki", "name": "Wiki"},
- # note, add 'progress' when we can support it on Edge
- ]
-
- import_module(
- module, store, course_data_path, static_content_store,
- course_location,
- target_location_namespace or course_location,
- do_import_static=do_import_static
- )
+ if course.tabs is None or len(course.tabs) == 0:
+ CourseTabList.initialize_default(course)
- course_items.append(module)
+ store.update_item(course)
+
+ course_items.append(course)
# then import all the static content
if static_content_store is not None and do_import_static:
- if target_location_namespace is not None:
- _namespace_rename = target_location_namespace
- else:
- _namespace_rename = course_location
-
# first pass to find everything in /static/
import_static_content(
- xml_module_store.modules[course_id], course_location,
course_data_path, static_content_store,
- _namespace_rename, subpath='static', verbose=verbose
+ dest_course_id, subpath='static', verbose=verbose
)
elif verbose and not do_import_static:
@@ -277,15 +273,9 @@ def import_from_xml(
simport = 'static_import'
if os.path.exists(course_data_path / simport):
- if target_location_namespace is not None:
- _namespace_rename = target_location_namespace
- else:
- _namespace_rename = course_location
-
import_static_content(
- xml_module_store.modules[course_id], course_location,
course_data_path, static_content_store,
- _namespace_rename, subpath=simport, verbose=verbose
+ dest_course_id, subpath=simport, verbose=verbose
)
# finally loop through all the modules
@@ -295,20 +285,17 @@ def import_from_xml(
# of the loop so just skip over it in the inner loop
continue
- # remap module to the new namespace
- if target_location_namespace is not None:
- module = remap_namespace(module, target_location_namespace)
-
if verbose:
log.debug('importing module location {loc}'.format(
loc=module.location
))
import_module(
- module, store, course_data_path, static_content_store,
- course_location,
- target_location_namespace if target_location_namespace else course_location,
- do_import_static=do_import_static
+ module, store,
+ course_id,
+ dest_course_id,
+ do_import_static=do_import_static,
+ system=course.runtime
)
# now import any 'draft' items
@@ -319,51 +306,93 @@ def import_from_xml(
draft_store,
course_data_path,
static_content_store,
- course_location,
- target_location_namespace if target_location_namespace else course_location
+ course_id,
+ dest_course_id,
+ course.runtime
)
finally:
# turn back on all write signalling on stores that need it
if (hasattr(store, 'ignore_write_events_on_courses') and
- pseudo_course_id in store.ignore_write_events_on_courses):
- store.ignore_write_events_on_courses.remove(pseudo_course_id)
- store.refresh_cached_metadata_inheritance_tree(
- target_location_namespace if target_location_namespace is not None else course_location
- )
+ dest_course_id in store.ignore_write_events_on_courses):
+ store.ignore_write_events_on_courses.remove(dest_course_id)
+ store.refresh_cached_metadata_inheritance_tree(dest_course_id)
return xml_module_store, course_items
def import_module(
- module, store, course_data_path, static_content_store,
- source_course_location, dest_course_location, allow_not_found=False,
- do_import_static=True):
+ module, store,
+ source_course_id, dest_course_id,
+ do_import_static=True, system=None):
- logging.debug(u'processing import of module {}...'.format(module.location.url()))
+ logging.debug(u'processing import of module {}...'.format(module.location.to_deprecated_string()))
if do_import_static and 'data' in module.fields and isinstance(module.fields['data'], xblock.fields.String):
# we want to convert all 'non-portable' links in the module_data
# (if it is a string) to portable strings (e.g. /static/)
module.data = rewrite_nonportable_content_links(
- source_course_location.course_id,
- dest_course_location.course_id, module.data
+ source_course_id,
+ dest_course_id,
+ module.data
)
- # remove any export/import only xml_attributes
- # which are used to wire together draft imports
- if 'parent_sequential_url' in getattr(module, 'xml_attributes', []):
- del module.xml_attributes['parent_sequential_url']
- if 'index_in_children_list' in getattr(module, 'xml_attributes', []):
- del module.xml_attributes['index_in_children_list']
+ # Move the module to a new course
+ new_usage_key = module.scope_ids.usage_id.map_into_course(dest_course_id)
+ if new_usage_key.category == 'course':
+ new_usage_key = new_usage_key.replace(name=dest_course_id.run)
+ new_module = store.create_xmodule(new_usage_key, system=system)
- store.update_item(module, '**replace_user**', allow_not_found=allow_not_found)
+ def _convert_reference_fields_to_new_namespace(reference):
+ """
+ Convert a reference to the new namespace, but only
+ if the original namespace matched the original course.
+
+ Otherwise, returns the input value.
+ """
+ assert isinstance(reference, UsageKey)
+ if source_course_id == reference.course_key:
+ return reference.map_into_course(dest_course_id)
+ else:
+ return reference
+
+ for field_name, field in module.fields.iteritems():
+ if field.is_set_on(module):
+ if isinstance(field, Reference):
+ new_ref = _convert_reference_fields_to_new_namespace(getattr(module, field_name))
+ setattr(new_module, field_name, new_ref)
+ elif isinstance(field, ReferenceList):
+ references = getattr(module, field_name)
+ new_references = [_convert_reference_fields_to_new_namespace(reference) for reference in references]
+ setattr(new_module, field_name, new_references)
+ elif isinstance(field, ReferenceValueDict):
+ reference_dict = getattr(module, field_name)
+ new_reference_dict = {
+ key: _convert_reference_fields_to_new_namespace(reference)
+ for key, reference
+ in reference_dict.items()
+ }
+ setattr(new_module, field_name, new_reference_dict)
+ elif field_name == 'xml_attributes':
+ value = getattr(module, field_name)
+ # remove any export/import only xml_attributes
+ # which are used to wire together draft imports
+ if 'parent_sequential_url' in value:
+ del value['parent_sequential_url']
+
+ if 'index_in_children_list' in value:
+ del value['index_in_children_list']
+ setattr(new_module, field_name, value)
+ else:
+ setattr(new_module, field_name, getattr(module, field_name))
+ store.update_item(new_module, '**replace_user**', allow_not_found=True)
+ return new_module
def import_course_draft(
xml_module_store, store, draft_store, course_data_path,
- static_content_store, source_location_namespace,
- target_location_namespace):
+ static_content_store, source_course_id,
+ target_course_id, mongo_runtime):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current
@@ -388,7 +417,7 @@ def import_course_draft(
draft_course_dir = draft_dir.replace(data_dir, '', 1)
system = ImportSystem(
xmlstore=xml_module_store,
- course_id=target_location_namespace.course_id,
+ course_id=target_course_id,
course_dir=draft_course_dir,
error_tracker=errorlog.tracker,
parent_tracker=ParentTracker(),
@@ -458,14 +487,13 @@ def import_course_draft(
else:
drafts[index] = [descriptor]
- except Exception, e:
- logging.exception('There was an error. {err}'.format(
- err=unicode(e)
- ))
+ except Exception:
+ logging.exception('Error while parsing course xml.')
# For each index_in_children_list key, there is a list of vertical descriptors.
for key in sorted(drafts.iterkeys()):
for descriptor in drafts[key]:
+ course_key = descriptor.location.course_key
try:
def _import_module(module):
# Update the module's location to "draft" revision
@@ -482,141 +510,29 @@ def import_course_draft(
sequential_url = module.xml_attributes['parent_sequential_url']
index = int(module.xml_attributes['index_in_children_list'])
- seq_location = Location(sequential_url)
+ seq_location = course_key.make_usage_key_from_deprecated_string(sequential_url)
# IMPORTANT: Be sure to update the sequential
# in the NEW namespace
- seq_location = seq_location.replace(
- org=target_location_namespace.org,
- course=target_location_namespace.course
- )
+ seq_location = seq_location.map_into_course(target_course_id)
sequential = store.get_item(seq_location, depth=0)
- if non_draft_location.url() not in sequential.children:
- sequential.children.insert(index, non_draft_location.url())
+ if non_draft_location not in sequential.children:
+ sequential.children.insert(index, non_draft_location)
store.update_item(sequential, '**replace_user**')
import_module(
- module, draft_store, course_data_path,
- static_content_store, source_location_namespace,
- target_location_namespace, allow_not_found=True
+ module, draft_store,
+ source_course_id,
+ target_course_id, system=mongo_runtime
)
for child in module.get_children():
_import_module(child)
_import_module(descriptor)
- except Exception, e:
- logging.exception('There was an error. {err}'.format(
- err=unicode(e)
- ))
-
-
-def remap_namespace(module, target_location_namespace):
- if target_location_namespace is None:
- return module
-
- original_location = module.location
-
- # This looks a bit wonky as we need to also change the 'name' of the
- # imported course to be what the caller passed in
- if module.location.category != 'course':
- _update_module_location(
- module,
- module.location.replace(
- tag=target_location_namespace.tag,
- org=target_location_namespace.org,
- course=target_location_namespace.course
- )
- )
-
- else:
- #
- # module is a course module
- #
- module.location = module.location.replace(
- tag=target_location_namespace.tag,
- org=target_location_namespace.org,
- course=target_location_namespace.course,
- name=target_location_namespace.name
- )
- # There is more re-namespacing work we have to do when
- # importing course modules
-
- # remap pdf_textbook urls to portable static URLs
- for entry in module.pdf_textbooks:
- for chapter in entry.get('chapters', []):
- if StaticContent.is_c4x_path(chapter.get('url', '')):
- chapter_loc = StaticContent.get_location_from_path(chapter['url'])
- chapter['url'] = StaticContent.get_static_path_from_location(
- chapter_loc
- )
-
- # Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
- # If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
- # values then remap it so that the wiki does not point to the old wiki.
- if original_location.course_id != target_location_namespace.course_id:
- original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
- original_location.org,
- original_location.course,
- original_location.name
- )
- if module.wiki_slug == original_unique_wiki_slug or module.wiki_slug == original_location.course:
- module.wiki_slug = u'{0}.{1}.{2}'.format(
- target_location_namespace.org,
- target_location_namespace.course,
- target_location_namespace.name,
- )
-
- module.save()
-
- all_fields = module.get_explicitly_set_fields_by_scope(Scope.content)
- all_fields.update(module.get_explicitly_set_fields_by_scope(Scope.settings))
- if hasattr(module, 'children'):
- all_fields['children'] = module.children
-
- def convert_ref(reference):
- """
- Convert a reference to the new namespace, but only
- if the original namespace matched the original course.
-
- Otherwise, returns the input value.
- """
- new_ref = reference
- ref = Location(reference)
- in_original_namespace = (original_location.tag == ref.tag and
- original_location.org == ref.org and
- original_location.course == ref.course)
- if in_original_namespace:
- new_ref = ref.replace(
- tag=target_location_namespace.tag,
- org=target_location_namespace.org,
- course=target_location_namespace.course
- ).url()
- return new_ref
-
- for field_name in all_fields:
- field_object = module.fields.get(field_name)
- if isinstance(field_object, Reference):
- new_ref = convert_ref(getattr(module, field_name))
- setattr(module, field_name, new_ref)
- module.save()
- elif isinstance(field_object, ReferenceList):
- references = getattr(module, field_name)
- new_references = [convert_ref(reference) for reference in references]
- setattr(module, field_name, new_references)
- module.save()
- elif isinstance(field_object, ReferenceValueDict):
- reference_dict = getattr(module, field_name)
- new_reference_dict = {
- key: convert_ref(reference)
- for key, reference
- in reference_dict.items()
- }
- setattr(module, field_name, new_reference_dict)
- module.save()
-
- return module
+ except Exception:
+ logging.exception('There while importing draft descriptor %s', descriptor)
def allowed_metadata_by_category(category):
@@ -648,7 +564,7 @@ def check_module_metadata_editability(module):
print(
": found non-editable metadata on {url}. "
"These metadata keys are not supported = {keys}".format(
- url=module.location.url(), keys=illegal_keys
+ url=module.location.to_deprecated_string(), keys=illegal_keys
)
)
@@ -676,7 +592,7 @@ def validate_category_hierarchy(
parents.append(module)
for parent in parents:
- for child_loc in [Location(child) for child in parent.children]:
+ for child_loc in parent.children:
if child_loc.category != expected_child_category:
err_cnt += 1
print(
@@ -767,7 +683,7 @@ def perform_xlint(
warn_cnt += _warn_cnt
# first count all errors and warnings as part of the XMLModuleStore import
- for err_log in module_store._location_errors.itervalues():
+ for err_log in module_store._course_errors.itervalues():
for err_log_entry in err_log.errors:
msg = err_log_entry[0]
if msg.startswith('ERROR:'):
@@ -815,12 +731,7 @@ def perform_xlint(
)
# check for a presence of a course marketing video
- location_elements = Location.parse_course_id(course_id)
- location_elements['tag'] = 'i4x'
- location_elements['category'] = 'about'
- location_elements['name'] = 'video'
- loc = Location(location_elements)
- if loc not in module_store.modules[course_id]:
+ if not module_store.has_item(course_id.make_usage_key('about', 'video')):
print(
"WARN: Missing course marketing video. It is recommended "
"that every course have a marketing video."
diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py
index 74125147402..8eaff34b64d 100644
--- a/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py
+++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py
@@ -412,7 +412,7 @@ class CombinedOpenEndedV1Module():
:param message: A message to put in the log.
:return: None
"""
- info_message = "Combined open ended user state for user {0} in location {1} was invalid. It has been reset, and you now have a new attempt. {2}".format(self.system.anonymous_student_id, self.location.url(), message)
+ info_message = "Combined open ended user state for user {0} in location {1} was invalid. It has been reset, and you now have a new attempt. {2}".format(self.system.anonymous_student_id, self.location.to_deprecated_string(), message)
self.current_task_number = 0
self.student_attempts = 0
self.old_task_states.append(self.task_states)
@@ -800,7 +800,7 @@ class CombinedOpenEndedV1Module():
success = False
allowed_to_submit = True
try:
- response = self.peer_gs.get_data_for_location(self.location.url(), student_id)
+ response = self.peer_gs.get_data_for_location(self.location.to_deprecated_string(), student_id)
count_graded = response['count_graded']
count_required = response['count_required']
student_sub_count = response['student_sub_count']
diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_rubric.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_rubric.py
index 7391dea5abd..0a4641f66d6 100644
--- a/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_rubric.py
+++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_rubric.py
@@ -96,7 +96,7 @@ class CombinedOpenEndedRubric(object):
if not success:
#This is a staff_facing_error
error_message = "Could not parse rubric : {0} for location {1}. Contact the learning sciences group for assistance.".format(
- rubric_string, location.url())
+ rubric_string, location.to_deprecated_string())
log.error(error_message)
raise RubricParsingError(error_message)
diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py
index 3d5b993c887..6ea21c1fef6 100644
--- a/common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py
+++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py
@@ -105,7 +105,7 @@ class OpenEndedModule(openendedchild.OpenEndedChild):
# NOTE: self.system.location is valid because the capa_module
# __init__ adds it (easiest way to get problem location into
# response types)
- except TypeError, ValueError:
+ except (TypeError, ValueError):
# This is a dev_facing_error
log.exception(
"Grader payload from external open ended grading server is not a json object! Object: {0}".format(
@@ -116,7 +116,7 @@ class OpenEndedModule(openendedchild.OpenEndedChild):
parsed_grader_payload.update({
'location': self.location_string,
- 'course_id': system.course_id,
+ 'course_id': system.course_id.to_deprecated_string(),
'prompt': prompt_string,
'rubric': rubric_string,
'initial_display': self.initial_display,
diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/openendedchild.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/openendedchild.py
index b68d8ad72a5..1951bce153e 100644
--- a/common/lib/xmodule/xmodule/open_ended_grading_classes/openendedchild.py
+++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/openendedchild.py
@@ -157,7 +157,7 @@ class OpenEndedChild(object):
self.location_string = location
try:
- self.location_string = self.location_string.url()
+ self.location_string = self.location_string.to_deprecated_string()
except:
pass
diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/peer_grading_service.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/peer_grading_service.py
index 875cc5f6800..dd9a74293a6 100644
--- a/common/lib/xmodule/xmodule/open_ended_grading_classes/peer_grading_service.py
+++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/peer_grading_service.py
@@ -87,6 +87,11 @@ class PeerGradingService(GradingService):
def get_problem_list(self, course_id, grader_id):
params = {'course_id': course_id, 'student_id': grader_id}
result = self.get(self.get_problem_list_url, params)
+
+ if 'problem_list' in result:
+ for problem in result['problem_list']:
+ problem['location'] = course_id.make_usage_key_from_deprecated_string(problem['location'])
+
self._record_result('get_problem_list', result)
dog_stats_api.histogram(
self._metric_name('get_problem_list.result.length'),
diff --git a/common/lib/xmodule/xmodule/peer_grading_module.py b/common/lib/xmodule/xmodule/peer_grading_module.py
index 191c05b46f9..c9fe618b109 100644
--- a/common/lib/xmodule/xmodule/peer_grading_module.py
+++ b/common/lib/xmodule/xmodule/peer_grading_module.py
@@ -8,7 +8,6 @@ from xblock.fields import Dict, String, Scope, Boolean, Float, Reference
from xmodule.capa_module import ComplexEncoder
from xmodule.fields import Date, Timedelta
-from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError, NoPathToItem
from xmodule.raw_module import RawDescriptor
from xmodule.timeinfo import TimeInfo
@@ -261,7 +260,7 @@ class PeerGradingModule(PeerGradingFields, XModule):
if not success:
log.exception(
"No instance data found and could not get data from controller for loc {0} student {1}".format(
- self.system.location.url(), self.system.anonymous_student_id
+ self.system.location.to_deprecated_string(), self.system.anonymous_student_id
))
return None
count_graded = response['count_graded']
@@ -563,7 +562,7 @@ class PeerGradingModule(PeerGradingFields, XModule):
good_problem_list = []
for problem in problem_list:
- problem_location = Location(problem['location'])
+ problem_location = problem['location']
try:
descriptor = self._find_corresponding_module_for_location(problem_location)
except (NoPathToItem, ItemNotFoundError):
@@ -588,7 +587,6 @@ class PeerGradingModule(PeerGradingFields, XModule):
ajax_url = self.ajax_url
html = self.system.render_template('peer_grading/peer_grading.html', {
- 'course_id': self.course_id,
'ajax_url': ajax_url,
'success': success,
'problem_list': good_problem_list,
@@ -611,10 +609,10 @@ class PeerGradingModule(PeerGradingFields, XModule):
log.error(
"Peer grading problem in peer_grading_module called with no get parameters, but use_for_single_location is False.")
return {'html': "", 'success': False}
- problem_location = Location(self.link_to_location)
+ problem_location = self.link_to_location
elif data.get('location') is not None:
- problem_location = Location(data.get('location'))
+ problem_location = self.course_id.make_usage_key_from_deprecated_string(data.get('location'))
module = self._find_corresponding_module_for_location(problem_location)
diff --git a/common/lib/xmodule/xmodule/seq_module.py b/common/lib/xmodule/xmodule/seq_module.py
index 73d1e9d59bc..6cb25884a41 100644
--- a/common/lib/xmodule/xmodule/seq_module.py
+++ b/common/lib/xmodule/xmodule/seq_module.py
@@ -96,7 +96,7 @@ class SequenceModule(SequenceFields, XModule):
'progress_status': Progress.to_js_status_str(progress),
'progress_detail': Progress.to_js_detail_str(progress),
'type': child.get_icon_class(),
- 'id': child.id,
+ 'id': child.scope_ids.usage_id.to_deprecated_string(),
}
if childinfo['title'] == '':
childinfo['title'] = child.display_name_with_default
@@ -104,7 +104,7 @@ class SequenceModule(SequenceFields, XModule):
params = {'items': contents,
'element_id': self.location.html_id(),
- 'item_id': self.id,
+ 'item_id': self.location.to_deprecated_string(),
'position': self.position,
'tag': self.location.category,
'ajax_url': self.system.ajax_url,
diff --git a/common/lib/xmodule/xmodule/split_test_module.py b/common/lib/xmodule/xmodule/split_test_module.py
index cf088a255ac..e813448cbdc 100644
--- a/common/lib/xmodule/xmodule/split_test_module.py
+++ b/common/lib/xmodule/xmodule/split_test_module.py
@@ -82,7 +82,7 @@ class SplitTestModule(SplitTestFields, XModule):
# we've picked a choice. Use self.descriptor.get_children() instead.
for child in self.descriptor.get_children():
- if child.location.url() == location:
+ if child.location == location:
return child
return None
@@ -182,7 +182,7 @@ class SplitTestModule(SplitTestFields, XModule):
fragment.add_frag_resources(rendered_child)
contents.append({
- 'id': child.id,
+ 'id': child.location.to_deprecated_string(),
'content': rendered_child.content
})
@@ -252,7 +252,11 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor):
def definition_to_xml(self, resource_fs):
xml_object = etree.Element('split_test')
- xml_object.set('group_id_to_child', json.dumps(self.group_id_to_child))
+ renderable_groups = {}
+ # json.dumps doesn't know how to handle Location objects
+ for group in self.group_id_to_child:
+ renderable_groups[group] = self.group_id_to_child[group].to_deprecated_string()
+ xml_object.set('group_id_to_child', json.dumps(renderable_groups))
xml_object.set('user_partition_id', str(self.user_partition_id))
for child in self.get_children():
self.runtime.add_block_as_child_node(child, xml_object)
diff --git a/common/lib/xmodule/xmodule/tabs.py b/common/lib/xmodule/xmodule/tabs.py
index cd1f5101db0..1b94f3ee021 100644
--- a/common/lib/xmodule/xmodule/tabs.py
+++ b/common/lib/xmodule/xmodule/tabs.py
@@ -456,7 +456,7 @@ class StaticTab(CourseTab):
super(StaticTab, self).__init__(
name=tab_dict['name'] if tab_dict else name,
tab_id='static_tab_{0}'.format(self.url_slug),
- link_func=lambda course, reverse_func: reverse_func(self.type, args=[course.id, self.url_slug]),
+ link_func=lambda course, reverse_func: reverse_func(self.type, args=[course.id.to_deprecated_string(), self.url_slug]),
)
def __getitem__(self, key):
@@ -537,7 +537,7 @@ class TextbookTabs(TextbookTabsBase):
yield SingleTextbookTab(
name=textbook.title,
tab_id='textbook/{0}'.format(index),
- link_func=lambda course, reverse_func: reverse_func('book', args=[course.id, index]),
+ link_func=lambda course, reverse_func: reverse_func('book', args=[course.id.to_deprecated_string(), index]),
)
@@ -557,7 +557,7 @@ class PDFTextbookTabs(TextbookTabsBase):
yield SingleTextbookTab(
name=textbook['tab_title'],
tab_id='pdftextbook/{0}'.format(index),
- link_func=lambda course, reverse_func: reverse_func('pdf_book', args=[course.id, index]),
+ link_func=lambda course, reverse_func: reverse_func('pdf_book', args=[course.id.to_deprecated_string(), index]),
)
@@ -577,7 +577,7 @@ class HtmlTextbookTabs(TextbookTabsBase):
yield SingleTextbookTab(
name=textbook['tab_title'],
tab_id='htmltextbook/{0}'.format(index),
- link_func=lambda course, reverse_func: reverse_func('html_book', args=[course.id, index]),
+ link_func=lambda course, reverse_func: reverse_func('html_book', args=[course.id.to_deprecated_string(), index]),
)
@@ -884,7 +884,7 @@ def link_reverse_func(reverse_name):
Returns a function that takes in a course and reverse_url_func,
and calls the reverse_url_func with the given reverse_name and course' ID.
"""
- return lambda course, reverse_url_func: reverse_url_func(reverse_name, args=[course.id])
+ return lambda course, reverse_url_func: reverse_url_func(reverse_name, args=[course.id.to_deprecated_string()])
def link_value_func(value):
diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 70ae00abb2f..e967e8872c5 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -16,12 +16,13 @@ from mock import Mock
from path import path
from xblock.field_data import DictFieldData
+from xblock.fields import ScopeIds
from xmodule.x_module import ModuleSystem, XModuleDescriptor, XModuleMixin
from xmodule.modulestore.inheritance import InheritanceMixin
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.error_module import ErrorDescriptor
-from xmodule.modulestore.xml import LocationReader
MODULE_DIR = path(__file__).dirname()
@@ -45,13 +46,21 @@ class TestModuleSystem(ModuleSystem): # pylint: disable=abstract-method
ModuleSystem for testing
"""
def handler_url(self, block, handler, suffix='', query='', thirdparty=False):
- return str(block.scope_ids.usage_id) + '/' + handler + '/' + suffix + '?' + query
+ return '{usage_id}/{handler}{suffix}?{query}'.format(
+ usage_id=block.scope_ids.usage_id.to_deprecated_string(),
+ handler=handler,
+ suffix=suffix,
+ query=query,
+ )
def local_resource_url(self, block, uri):
- return 'resource/' + str(block.scope_ids.block_type) + '/' + uri
+ return 'resource/{usage_id}/{uri}'.format(
+ usage_id=block.scope_ids.usage_id.to_deprecated_string(),
+ uri=uri,
+ )
-def get_test_system(course_id=''):
+def get_test_system(course_id=SlashSeparatedCourseKey('org', 'course', 'run')):
"""
Construct a test ModuleSystem instance.
@@ -96,7 +105,6 @@ def get_test_descriptor_system():
render_template=mock_render_template,
mixins=(InheritanceMixin, XModuleMixin),
field_data=DictFieldData({}),
- id_reader=LocationReader(),
)
@@ -131,12 +139,15 @@ class LogicTest(unittest.TestCase):
url_name = ''
category = 'test'
- self.system = get_test_system(course_id='test/course/id')
+ self.system = get_test_system()
self.descriptor = EmptyClass()
self.xmodule_class = self.descriptor_class.module_class
+ usage_key = self.system.course_id.make_usage_key(self.descriptor.category, 'test_loc')
+ # ScopeIds has 4 fields: user_id, block_type, def_id, usage_id
+ scope_ids = ScopeIds(1, self.descriptor.category, usage_key, usage_key)
self.xmodule = self.xmodule_class(
- self.descriptor, self.system, DictFieldData(self.raw_field_data), Mock()
+ self.descriptor, self.system, DictFieldData(self.raw_field_data), scope_ids
)
def ajax_request(self, dispatch, data):
diff --git a/common/lib/xmodule/xmodule/tests/test_annotatable_module.py b/common/lib/xmodule/xmodule/tests/test_annotatable_module.py
index 24076500c11..aea1672b4e4 100644
--- a/common/lib/xmodule/xmodule/tests/test_annotatable_module.py
+++ b/common/lib/xmodule/xmodule/tests/test_annotatable_module.py
@@ -35,7 +35,7 @@ class AnnotatableModuleTestCase(unittest.TestCase):
Mock(),
get_test_system(),
DictFieldData({'data': self.sample_xml}),
- ScopeIds(None, None, None, None)
+ ScopeIds(None, None, None, Location('org', 'course', 'run', 'category', 'name', None))
)
def test_annotation_data_attr(self):
diff --git a/common/lib/xmodule/xmodule/tests/test_capa_module.py b/common/lib/xmodule/xmodule/tests/test_capa_module.py
index 074439d13aa..2e6c92d3885 100644
--- a/common/lib/xmodule/xmodule/tests/test_capa_module.py
+++ b/common/lib/xmodule/xmodule/tests/test_capa_module.py
@@ -101,8 +101,14 @@ class CapaFactory(object):
attempts: also added to instance state. Will be converted to an int.
"""
- location = Location(["i4x", "edX", "capa_test", "problem",
- "SampleProblem{0}".format(cls.next_num())])
+ location = Location(
+ "edX",
+ "capa_test",
+ "2012_Fall",
+ "problem",
+ "SampleProblem{0}".format(cls.next_num()),
+ None
+ )
if xml is None:
xml = cls.sample_problem_xml
field_data = {'data': xml}
diff --git a/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py b/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py
index 3eb5c914fe6..c343bded18d 100644
--- a/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py
+++ b/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py
@@ -12,7 +12,7 @@ import unittest
from datetime import datetime
from lxml import etree
-from mock import Mock, MagicMock, ANY, patch
+from mock import Mock, MagicMock, patch
from pytz import UTC
from webob.multidict import MultiDict
@@ -20,7 +20,6 @@ from xmodule.open_ended_grading_classes.openendedchild import OpenEndedChild
from xmodule.open_ended_grading_classes.open_ended_module import OpenEndedModule
from xmodule.open_ended_grading_classes.self_assessment_module import SelfAssessmentModule
from xmodule.open_ended_grading_classes.combined_open_ended_modulev1 import CombinedOpenEndedV1Module
-from xmodule.open_ended_grading_classes.grading_service_module import GradingServiceError
from xmodule.combined_open_ended_module import CombinedOpenEndedModule
from xmodule.modulestore import Location
from xmodule.tests import get_test_system, test_util_open_ended
@@ -48,8 +47,7 @@ class OpenEndedChildTest(unittest.TestCase):
"""
Test the open ended child class
"""
- location = Location(["i4x", "edX", "sa_test", "selfassessment",
- "SampleQuestion"])
+ location = Location("edX", "sa_test", "2012_Fall", "selfassessment", "SampleQuestion")
metadata = json.dumps({'attempts': '10'})
prompt = etree.XML("<prompt>This is a question prompt</prompt>")
@@ -173,8 +171,7 @@ class OpenEndedModuleTest(unittest.TestCase):
"""
Test the open ended module class
"""
- location = Location(["i4x", "edX", "sa_test", "selfassessment",
- "SampleQuestion"])
+ location = Location("edX", "sa_test", "2012_Fall", "selfassessment", "SampleQuestion")
metadata = json.dumps({'attempts': '10'})
prompt = etree.XML("<prompt>This is a question prompt</prompt>")
@@ -446,8 +443,7 @@ class CombinedOpenEndedModuleTest(unittest.TestCase):
"""
Unit tests for the combined open ended xmodule
"""
- location = Location(["i4x", "edX", "open_ended", "combinedopenended",
- "SampleQuestion"])
+ location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion")
definition_template = """
<combinedopenended attempts="10000">
{rubric}
@@ -517,6 +513,9 @@ class CombinedOpenEndedModuleTest(unittest.TestCase):
descriptor = Mock(data=full_definition)
test_system = get_test_system()
test_system.open_ended_grading_interface = None
+ usage_key = test_system.course_id.make_usage_key('combinedopenended', 'test_loc')
+ # ScopeIds has 4 fields: user_id, block_type, def_id, usage_id
+ scope_ids = ScopeIds(1, 'combinedopenended', usage_key, usage_key)
combinedoe_container = CombinedOpenEndedModule(
descriptor=descriptor,
runtime=test_system,
@@ -524,7 +523,7 @@ class CombinedOpenEndedModuleTest(unittest.TestCase):
'data': full_definition,
'weight': '1',
}),
- scope_ids=ScopeIds(None, None, None, None),
+ scope_ids=scope_ids,
)
def setUp(self):
@@ -799,8 +798,7 @@ class CombinedOpenEndedModuleConsistencyTest(unittest.TestCase):
# location, definition_template, prompt, rubric, max_score, metadata, oeparam, task_xml1, task_xml2
# All these variables are used to construct the xmodule descriptor.
- location = Location(["i4x", "edX", "open_ended", "combinedopenended",
- "SampleQuestion"])
+ location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion")
definition_template = """
<combinedopenended attempts="10000">
{rubric}
@@ -871,6 +869,9 @@ class CombinedOpenEndedModuleConsistencyTest(unittest.TestCase):
descriptor = Mock(data=full_definition)
test_system = get_test_system()
test_system.open_ended_grading_interface = None
+ usage_key = test_system.course_id.make_usage_key('combinedopenended', 'test_loc')
+ # ScopeIds has 4 fields: user_id, block_type, def_id, usage_id
+ scope_ids = ScopeIds(1, 'combinedopenended', usage_key, usage_key)
combinedoe_container = CombinedOpenEndedModule(
descriptor=descriptor,
runtime=test_system,
@@ -878,7 +879,7 @@ class CombinedOpenEndedModuleConsistencyTest(unittest.TestCase):
'data': full_definition,
'weight': '1',
}),
- scope_ids=ScopeIds(None, None, None, None),
+ scope_ids=scope_ids,
)
def setUp(self):
@@ -964,7 +965,7 @@ class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore):
"""
Test the student flow in the combined open ended xmodule
"""
- problem_location = Location(["i4x", "edX", "open_ended", "combinedopenended", "SampleQuestion"])
+ problem_location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion")
answer = "blah blah"
assessment = [0, 1]
hint = "blah"
@@ -999,7 +1000,7 @@ class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore):
return result
def _module(self):
- return self.get_module_from_location(self.problem_location, COURSE)
+ return self.get_module_from_location(self.problem_location)
def test_open_ended_load_and_save(self):
"""
@@ -1212,7 +1213,7 @@ class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore):
"""
Test if student is able to reset the problem
"""
- problem_location = Location(["i4x", "edX", "open_ended", "combinedopenended", "SampleQuestion1Attempt"])
+ problem_location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion1Attempt")
answer = "blah blah"
assessment = [0, 1]
hint = "blah"
@@ -1241,7 +1242,7 @@ class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore):
return result
def _module(self):
- return self.get_module_from_location(self.problem_location, COURSE)
+ return self.get_module_from_location(self.problem_location)
def test_reset_fail(self):
"""
@@ -1283,12 +1284,13 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
"""
Test if student is able to upload images properly.
"""
- problem_location = Location(["i4x", "edX", "open_ended", "combinedopenended", "SampleQuestionImageUpload"])
+ problem_location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestionImageUpload")
answer_text = "Hello, this is my amazing answer."
file_text = "Hello, this is my amazing file."
file_name = "Student file 1"
answer_link = "http://www.edx.org"
autolink_tag = '<a target="_blank" href='
+ autolink_tag_swapped = '<a href='
def get_module_system(self, descriptor):
test_system = get_test_system()
@@ -1306,7 +1308,7 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
"""
Test to see if a student submission without a file attached fails.
"""
- module = self.get_module_from_location(self.problem_location, COURSE)
+ module = self.get_module_from_location(self.problem_location)
# Simulate a student saving an answer
response = module.handle_ajax("save_answer", {"student_answer": self.answer_text})
@@ -1326,7 +1328,7 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
"""
Test to see if a student submission with a file is handled properly.
"""
- module = self.get_module_from_location(self.problem_location, COURSE)
+ module = self.get_module_from_location(self.problem_location)
# Simulate a student saving an answer with a file
response = module.handle_ajax("save_answer", {
@@ -1338,13 +1340,14 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
response = json.loads(response)
self.assertTrue(response['success'])
self.assertIn(self.file_name, response['student_response'])
- self.assertIn(self.autolink_tag, response['student_response'])
+ self.assertTrue(self.autolink_tag in response['student_response'] or
+ self.autolink_tag_swapped in response['student_response'])
def test_link_submission_success(self):
"""
Students can submit links instead of files. Check that the link is properly handled.
"""
- module = self.get_module_from_location(self.problem_location, COURSE)
+ module = self.get_module_from_location(self.problem_location)
# Simulate a student saving an answer with a link.
response = module.handle_ajax("save_answer", {
@@ -1355,7 +1358,8 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
self.assertTrue(response['success'])
self.assertIn(self.answer_link, response['student_response'])
- self.assertIn(self.autolink_tag, response['student_response'])
+ self.assertTrue(self.autolink_tag in response['student_response'] or
+ self.autolink_tag_swapped in response['student_response'])
class OpenEndedModuleUtilTest(unittest.TestCase):
@@ -1369,7 +1373,7 @@ class OpenEndedModuleUtilTest(unittest.TestCase):
embed_dirty = u'<embed height="200" id="cats" onhover="eval()" src="http://example.com/lolcats.swf" width="200"/>'
embed_clean = u'<embed width="200" height="200" id="cats" src="http://example.com/lolcats.swf">'
iframe_dirty = u'<iframe class="cats" height="200" onerror="eval()" src="http://example.com/lolcats" width="200"/>'
- iframe_clean = u'<iframe height="200" class="cats" width="200" src="http://example.com/lolcats"></iframe>'
+ iframe_clean = ur'<iframe (height="200" ?|class="cats" ?|width="200" ?|src="http://example.com/lolcats" ?)+></iframe>'
text = u'I am a \u201c\xfcber student\u201d'
text_lessthan_noencd = u'This used to be broken < by the other parser. 3>5'
@@ -1402,7 +1406,7 @@ class OpenEndedModuleUtilTest(unittest.TestCase):
"""
Basic test for passing through iframe, but stripping bad attr
"""
- self.assertEqual(OpenEndedChild.sanitize_html(self.iframe_dirty), self.iframe_clean)
+ self.assertRegexpMatches(OpenEndedChild.sanitize_html(self.iframe_dirty), self.iframe_clean)
def test_text(self):
"""
diff --git a/common/lib/xmodule/xmodule/tests/test_conditional.py b/common/lib/xmodule/xmodule/tests/test_conditional.py
index 0487194dbc7..1189e8fd564 100644
--- a/common/lib/xmodule/xmodule/tests/test_conditional.py
+++ b/common/lib/xmodule/xmodule/tests/test_conditional.py
@@ -1,4 +1,3 @@
-from ast import literal_eval
import json
import unittest
@@ -8,7 +7,7 @@ from mock import Mock, patch
from xblock.field_data import DictFieldData
from xblock.fields import ScopeIds
from xmodule.error_module import NonStaffErrorDescriptor
-from xmodule.modulestore import Location
+from xmodule.modulestore.locations import SlashSeparatedCourseKey, Location
from xmodule.modulestore.xml import ImportSystem, XMLModuleStore, CourseLocationGenerator
from xmodule.conditional_module import ConditionalDescriptor
from xmodule.tests import DATA_DIR, get_test_system, get_test_descriptor_system
@@ -54,13 +53,13 @@ class ConditionalFactory(object):
descriptor_system = get_test_descriptor_system()
# construct source descriptor and module:
- source_location = Location(["i4x", "edX", "conditional_test", "problem", "SampleProblem"])
+ source_location = Location("edX", "conditional_test", "test_run", "problem", "SampleProblem", None)
if source_is_error_module:
# Make an error descriptor and module
source_descriptor = NonStaffErrorDescriptor.from_xml(
'some random xml data',
system,
- id_generator=CourseLocationGenerator(source_location.org, source_location.course),
+ id_generator=CourseLocationGenerator(SlashSeparatedCourseKey('edX', 'conditional_test', 'test_run')),
error_msg='random error message'
)
else:
@@ -78,15 +77,19 @@ class ConditionalFactory(object):
child_descriptor.runtime = descriptor_system
child_descriptor.xmodule_runtime = get_test_system()
child_descriptor.render = lambda view, context=None: descriptor_system.render(child_descriptor, view, context)
+ child_descriptor.location = source_location.replace(category='html', name='child')
- descriptor_system.load_item = {'child': child_descriptor, 'source': source_descriptor}.get
+ descriptor_system.load_item = {
+ child_descriptor.location: child_descriptor,
+ source_location: source_descriptor
+ }.get
# construct conditional module:
- cond_location = Location(["i4x", "edX", "conditional_test", "conditional", "SampleConditional"])
+ cond_location = Location("edX", "conditional_test", "test_run", "conditional", "SampleConditional", None)
field_data = DictFieldData({
'data': '<conditional/>',
'xml_attributes': {'attempted': 'true'},
- 'children': ['child'],
+ 'children': [child_descriptor.location],
})
cond_descriptor = ConditionalDescriptor(
@@ -130,7 +133,6 @@ class ConditionalModuleBasicTest(unittest.TestCase):
expected = modules['cond_module'].xmodule_runtime.render_template('conditional_ajax.html', {
'ajax_url': modules['cond_module'].xmodule_runtime.ajax_url,
'element_id': u'i4x-edX-conditional_test-conditional-SampleConditional',
- 'id': u'i4x://edX/conditional_test/conditional/SampleConditional',
'depends': u'i4x-edX-conditional_test-problem-SampleProblem',
})
self.assertEquals(expected, html)
@@ -198,14 +200,14 @@ class ConditionalModuleXmlTest(unittest.TestCase):
def inner_get_module(descriptor):
if isinstance(descriptor, Location):
location = descriptor
- descriptor = self.modulestore.get_instance(course.id, location, depth=None)
+ descriptor = self.modulestore.get_item(location, depth=None)
descriptor.xmodule_runtime = get_test_system()
descriptor.xmodule_runtime.get_module = inner_get_module
return descriptor
# edx - HarvardX
# cond_test - ER22x
- location = Location(["i4x", "HarvardX", "ER22x", "conditional", "condone"])
+ location = Location("HarvardX", "ER22x", "2013_Spring", "conditional", "condone")
def replace_urls(text, staticfiles_prefix=None, replace_prefix='/static/', course_namespace=None):
return text
@@ -224,9 +226,8 @@ class ConditionalModuleXmlTest(unittest.TestCase):
'conditional_ajax.html',
{
# Test ajax url is just usage-id / handler_name
- 'ajax_url': 'i4x://HarvardX/ER22x/conditional/condone/xmodule_handler',
+ 'ajax_url': '{}/xmodule_handler'.format(location.to_deprecated_string()),
'element_id': u'i4x-HarvardX-ER22x-conditional-condone',
- 'id': u'i4x://HarvardX/ER22x/conditional/condone',
'depends': u'i4x-HarvardX-ER22x-problem-choiceprob'
}
)
@@ -242,7 +243,7 @@ class ConditionalModuleXmlTest(unittest.TestCase):
self.assertFalse(any(['This is a secret' in item for item in html]))
# Now change state of the capa problem to make it completed
- inner_module = inner_get_module(Location('i4x://HarvardX/ER22x/problem/choiceprob'))
+ inner_module = inner_get_module(location.replace(category="problem", name='choiceprob'))
inner_module.attempts = 1
# Save our modifications to the underlying KeyValueStore so they can be persisted
inner_module.save()
diff --git a/common/lib/xmodule/xmodule/tests/test_content.py b/common/lib/xmodule/xmodule/tests/test_content.py
index 6f5cf8ab8c4..4ce25269720 100644
--- a/common/lib/xmodule/xmodule/tests/test_content.py
+++ b/common/lib/xmodule/xmodule/tests/test_content.py
@@ -1,7 +1,7 @@
import unittest
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.content import ContentStore
-from xmodule.modulestore import Location
+from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation
class Content:
@@ -21,18 +21,28 @@ class ContentTest(unittest.TestCase):
self.assertIsNone(content.thumbnail_location)
def test_static_url_generation_from_courseid(self):
- url = StaticContent.convert_legacy_static_url_with_course_id('images_course_image.jpg', 'foo/bar/bz')
+ course_key = SlashSeparatedCourseKey('foo', 'bar', 'bz')
+ url = StaticContent.convert_legacy_static_url_with_course_id('images_course_image.jpg', course_key)
self.assertEqual(url, '/c4x/foo/bar/asset/images_course_image.jpg')
def test_generate_thumbnail_image(self):
contentStore = ContentStore()
- content = Content(Location(u'c4x', u'mitX', u'800', u'asset', u'monsters__.jpg'), None)
+ content = Content(AssetLocation(u'mitX', u'800', u'ignore_run', u'asset', u'monsters__.jpg'), None)
(thumbnail_content, thumbnail_file_location) = contentStore.generate_thumbnail(content)
self.assertIsNone(thumbnail_content)
- self.assertEqual(Location(u'c4x', u'mitX', u'800', u'thumbnail', u'monsters__.jpg'), thumbnail_file_location)
+ self.assertEqual(AssetLocation(u'mitX', u'800', u'ignore_run', u'thumbnail', u'monsters__.jpg'), thumbnail_file_location)
def test_compute_location(self):
# We had a bug that __ got converted into a single _. Make sure that substitution of INVALID_CHARS (like space)
# still happen.
- asset_location = StaticContent.compute_location('mitX', '400', 'subs__1eo_jXvZnE .srt.sjson')
- self.assertEqual(Location(u'c4x', u'mitX', u'400', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
+ asset_location = StaticContent.compute_location(
+ SlashSeparatedCourseKey('mitX', '400', 'ignore'), 'subs__1eo_jXvZnE .srt.sjson'
+ )
+ self.assertEqual(AssetLocation(u'mitX', u'400', u'ignore', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
+
+ def test_get_location_from_path(self):
+ asset_location = StaticContent.get_location_from_path(u'/c4x/foo/bar/asset/images_course_image.jpg')
+ self.assertEqual(
+ AssetLocation(u'foo', u'bar', None, u'asset', u'images_course_image.jpg', None),
+ asset_location
+ )
diff --git a/common/lib/xmodule/xmodule/tests/test_course_module.py b/common/lib/xmodule/xmodule/tests/test_course_module.py
index 7a60e29e370..6f1ffda2dab 100644
--- a/common/lib/xmodule/xmodule/tests/test_course_module.py
+++ b/common/lib/xmodule/xmodule/tests/test_course_module.py
@@ -8,7 +8,8 @@ from mock import Mock, patch
from xblock.runtime import KvsFieldData, DictKeyValueStore
import xmodule.course_module
-from xmodule.modulestore.xml import ImportSystem, XMLModuleStore, LocationReader
+from xmodule.modulestore.xml import ImportSystem, XMLModuleStore
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
from django.utils.timezone import UTC
@@ -32,7 +33,7 @@ class DummySystem(ImportSystem):
xmlstore = XMLModuleStore("data_dir", course_dirs=[],
load_error_modules=load_error_modules)
- course_id = "/".join([ORG, COURSE, 'test_run'])
+ course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
parent_tracker = Mock()
@@ -45,7 +46,6 @@ class DummySystem(ImportSystem):
parent_tracker=parent_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
- id_reader=LocationReader(),
)
diff --git a/common/lib/xmodule/xmodule/tests/test_delay_between_attempts.py b/common/lib/xmodule/xmodule/tests/test_delay_between_attempts.py
index aa478c0a174..8235db22801 100644
--- a/common/lib/xmodule/xmodule/tests/test_delay_between_attempts.py
+++ b/common/lib/xmodule/xmodule/tests/test_delay_between_attempts.py
@@ -84,8 +84,7 @@ class CapaFactoryWithDelay(object):
"""
Optional parameters here are cut down to what we actually use vs. the regular CapaFactory.
"""
- location = Location(["i4x", "edX", "capa_test", "problem",
- "SampleProblem{0}".format(cls.next_num())])
+ location = Location("edX", "capa_test", "run", "problem", "SampleProblem{0}".format(cls.next_num()))
field_data = {'data': cls.sample_problem_xml}
if max_attempts is not None:
diff --git a/common/lib/xmodule/xmodule/tests/test_editing_module.py b/common/lib/xmodule/xmodule/tests/test_editing_module.py
index 36f028e8a4f..86e0568d0ad 100644
--- a/common/lib/xmodule/xmodule/tests/test_editing_module.py
+++ b/common/lib/xmodule/xmodule/tests/test_editing_module.py
@@ -5,6 +5,7 @@ import logging
from mock import Mock
from pkg_resources import resource_string
+from xmodule.modulestore.locations import Location
from xmodule.editing_module import TabsEditingDescriptor
from xblock.field_data import DictFieldData
from xblock.fields import ScopeIds
@@ -46,7 +47,7 @@ class TabsEditingDescriptorTestCase(unittest.TestCase):
TabsEditingDescriptor.tabs = self.tabs
self.descriptor = system.construct_xblock_from_class(
TabsEditingDescriptor,
- scope_ids=ScopeIds(None, None, None, None),
+ scope_ids=ScopeIds(None, None, None, Location('org', 'course', 'run', 'category', 'name', 'revision')),
field_data=DictFieldData({}),
)
diff --git a/common/lib/xmodule/xmodule/tests/test_error_module.py b/common/lib/xmodule/xmodule/tests/test_error_module.py
index be88575eb19..9050889e521 100644
--- a/common/lib/xmodule/xmodule/tests/test_error_module.py
+++ b/common/lib/xmodule/xmodule/tests/test_error_module.py
@@ -4,8 +4,8 @@ Tests for ErrorModule and NonStaffErrorModule
import unittest
from xmodule.tests import get_test_system
from xmodule.error_module import ErrorDescriptor, ErrorModule, NonStaffErrorDescriptor
-from xmodule.modulestore import Location
from xmodule.modulestore.xml import CourseLocationGenerator
+from xmodule.modulestore.locations import SlashSeparatedCourseKey, Location
from xmodule.x_module import XModuleDescriptor, XModule
from mock import MagicMock, Mock, patch
from xblock.runtime import Runtime, IdReader
@@ -17,9 +17,8 @@ from xblock.test.tools import unabc
class SetupTestErrorModules():
def setUp(self):
self.system = get_test_system()
- self.org = "org"
- self.course = "course"
- self.location = Location(['i4x', self.org, self.course, None, None])
+ self.course_id = SlashSeparatedCourseKey('org', 'course', 'run')
+ self.location = self.course_id.make_usage_key('foo', 'bar')
self.valid_xml = u"<problem>ABC \N{SNOWMAN}</problem>"
self.error_msg = "Error"
@@ -35,7 +34,7 @@ class TestErrorModule(unittest.TestCase, SetupTestErrorModules):
descriptor = ErrorDescriptor.from_xml(
self.valid_xml,
self.system,
- CourseLocationGenerator(self.org, self.course),
+ CourseLocationGenerator(self.course_id),
self.error_msg
)
self.assertIsInstance(descriptor, ErrorDescriptor)
@@ -70,7 +69,7 @@ class TestNonStaffErrorModule(unittest.TestCase, SetupTestErrorModules):
descriptor = NonStaffErrorDescriptor.from_xml(
self.valid_xml,
self.system,
- CourseLocationGenerator(self.org, self.course)
+ CourseLocationGenerator(self.course_id)
)
self.assertIsInstance(descriptor, NonStaffErrorDescriptor)
@@ -78,7 +77,7 @@ class TestNonStaffErrorModule(unittest.TestCase, SetupTestErrorModules):
descriptor = NonStaffErrorDescriptor.from_xml(
self.valid_xml,
self.system,
- CourseLocationGenerator(self.org, self.course)
+ CourseLocationGenerator(self.course_id)
)
descriptor.xmodule_runtime = self.system
context_repr = self.system.render(descriptor, 'student_view').content
@@ -130,7 +129,7 @@ class TestErrorModuleConstruction(unittest.TestCase):
self.descriptor = BrokenDescriptor(
TestRuntime(Mock(spec=IdReader), field_data),
field_data,
- ScopeIds(None, None, None, 'i4x://org/course/broken/name')
+ ScopeIds(None, None, None, Location('org', 'course', 'run', 'broken', 'name', None))
)
self.descriptor.xmodule_runtime = TestRuntime(Mock(spec=IdReader), field_data)
self.descriptor.xmodule_runtime.error_descriptor_class = ErrorDescriptor
diff --git a/common/lib/xmodule/xmodule/tests/test_export.py b/common/lib/xmodule/xmodule/tests/test_export.py
index 9bfa861a04d..d1b2edb4c20 100644
--- a/common/lib/xmodule/xmodule/tests/test_export.py
+++ b/common/lib/xmodule/xmodule/tests/test_export.py
@@ -36,7 +36,7 @@ def strip_filenames(descriptor):
"""
Recursively strips 'filename' from all children's definitions.
"""
- print("strip filename from {desc}".format(desc=descriptor.location.url()))
+ print("strip filename from {desc}".format(desc=descriptor.location.to_deprecated_string()))
if descriptor._field_data.has(descriptor, 'filename'):
descriptor._field_data.delete(descriptor, 'filename')
@@ -173,11 +173,11 @@ class TestEdxJsonEncoder(unittest.TestCase):
self.null_utc_tz = NullTZ()
def test_encode_location(self):
- loc = Location('i4x', 'org', 'course', 'category', 'name')
- self.assertEqual(loc.url(), self.encoder.default(loc))
+ loc = Location('org', 'course', 'run', 'category', 'name', None)
+ self.assertEqual(loc.to_deprecated_string(), self.encoder.default(loc))
- loc = Location('i4x', 'org', 'course', 'category', 'name', 'version')
- self.assertEqual(loc.url(), self.encoder.default(loc))
+ loc = Location('org', 'course', 'run', 'category', 'name', 'version')
+ self.assertEqual(loc.to_deprecated_string(), self.encoder.default(loc))
def test_encode_naive_datetime(self):
self.assertEqual(
diff --git a/common/lib/xmodule/xmodule/tests/test_import.py b/common/lib/xmodule/xmodule/tests/test_import.py
index 0a185ded87e..7be88f18823 100644
--- a/common/lib/xmodule/xmodule/tests/test_import.py
+++ b/common/lib/xmodule/xmodule/tests/test_import.py
@@ -12,12 +12,13 @@ from django.utils.timezone import UTC
from xmodule.xml_module import is_pointer_tag
from xmodule.modulestore import Location, only_xmodules
-from xmodule.modulestore.xml import ImportSystem, XMLModuleStore, LocationReader
+from xmodule.modulestore.xml import ImportSystem, XMLModuleStore
from xmodule.modulestore.inheritance import compute_inherited_metadata
from xmodule.x_module import XModuleMixin
from xmodule.fields import Date
from xmodule.tests import DATA_DIR
from xmodule.modulestore.inheritance import InheritanceMixin
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xblock.core import XBlock
from xblock.fields import Scope, String, Integer
@@ -34,7 +35,7 @@ class DummySystem(ImportSystem):
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", course_dirs=[], load_error_modules=load_error_modules)
- course_id = "/".join([ORG, COURSE, 'test_run'])
+ course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
parent_tracker = Mock()
@@ -48,7 +49,6 @@ class DummySystem(ImportSystem):
load_error_modules=load_error_modules,
mixins=(InheritanceMixin, XModuleMixin),
field_data=KvsFieldData(DictKeyValueStore()),
- id_reader=LocationReader(),
)
def render_template(self, _template, _context):
@@ -343,7 +343,7 @@ class ImportTestCase(BaseCourseTestCase):
def check_for_key(key, node, value):
"recursive check for presence of key"
- print("Checking {0}".format(node.location.url()))
+ print("Checking {0}".format(node.location.to_deprecated_string()))
self.assertEqual(getattr(node, key), value)
for c in node.get_children():
check_for_key(key, c, value)
@@ -383,12 +383,10 @@ class ImportTestCase(BaseCourseTestCase):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'two_toys'])
- toy_id = "edX/toy/2012_Fall"
- two_toy_id = "edX/toy/TT_2012_Fall"
-
- location = Location(["i4x", "edX", "toy", "video", "Welcome"])
- toy_video = modulestore.get_instance(toy_id, location)
- two_toy_video = modulestore.get_instance(two_toy_id, location)
+ location = Location("edX", "toy", "2012_Fall", "video", "Welcome", None)
+ toy_video = modulestore.get_item(location)
+ location_two = Location("edX", "toy", "TT_2012_Fall", "video", "Welcome", None)
+ two_toy_video = modulestore.get_item(location_two)
self.assertEqual(toy_video.youtube_id_1_0, "p2Q6BrNhdh8")
self.assertEqual(two_toy_video.youtube_id_1_0, "p2Q6BrNhdh9")
@@ -401,10 +399,9 @@ class ImportTestCase(BaseCourseTestCase):
courses = modulestore.get_courses()
self.assertEquals(len(courses), 1)
course = courses[0]
- course_id = course.id
print("course errors:")
- for (msg, err) in modulestore.get_item_errors(course.location):
+ for (msg, err) in modulestore.get_course_errors(course.id):
print(msg)
print(err)
@@ -416,13 +413,12 @@ class ImportTestCase(BaseCourseTestCase):
print("Ch2 location: ", ch2.location)
- also_ch2 = modulestore.get_instance(course_id, ch2.location)
+ also_ch2 = modulestore.get_item(ch2.location)
self.assertEquals(ch2, also_ch2)
print("making sure html loaded")
- cloc = course.location
- loc = Location(cloc.tag, cloc.org, cloc.course, 'html', 'secret:toylab')
- html = modulestore.get_instance(course_id, loc)
+ loc = course.id.make_usage_key('html', 'secret:toylab')
+ html = modulestore.get_item(loc)
self.assertEquals(html.display_name, "Toy lab")
def test_unicode(self):
@@ -442,12 +438,16 @@ class ImportTestCase(BaseCourseTestCase):
# Expect to find an error/exception about characters in "®esources"
expect = "Invalid characters"
- errors = [(msg.encode("utf-8"), err.encode("utf-8"))
- for msg, err in
- modulestore.get_item_errors(course.location)]
-
- self.assertTrue(any(expect in msg or expect in err
- for msg, err in errors))
+ errors = [
+ (msg.encode("utf-8"), err.encode("utf-8"))
+ for msg, err
+ in modulestore.get_course_errors(course.id)
+ ]
+
+ self.assertTrue(any(
+ expect in msg or expect in err
+ for msg, err in errors
+ ))
chapters = course.get_children()
self.assertEqual(len(chapters), 4)
@@ -458,7 +458,7 @@ class ImportTestCase(BaseCourseTestCase):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
- toy_id = "edX/toy/2012_Fall"
+ toy_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
course = modulestore.get_course(toy_id)
chapters = course.get_children()
@@ -484,20 +484,12 @@ class ImportTestCase(BaseCourseTestCase):
self.assertEqual(len(sections), 1)
- location = course.location
-
- conditional_location = Location(
- location.tag, location.org, location.course,
- 'conditional', 'condone'
- )
- module = modulestore.get_instance(course.id, conditional_location)
+ conditional_location = course.id.make_usage_key('conditional', 'condone')
+ module = modulestore.get_item(conditional_location)
self.assertEqual(len(module.children), 1)
- poll_location = Location(
- location.tag, location.org, location.course,
- 'poll_question', 'first_poll'
- )
- module = modulestore.get_instance(course.id, poll_location)
+ poll_location = course.id.make_usage_key('poll_question', 'first_poll')
+ module = modulestore.get_item(poll_location)
self.assertEqual(len(module.get_children()), 0)
self.assertEqual(module.voted, False)
self.assertEqual(module.poll_answer, '')
@@ -527,9 +519,9 @@ class ImportTestCase(BaseCourseTestCase):
'''
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['graphic_slider_tool'])
- sa_id = "edX/gst_test/2012_Fall"
- location = Location(["i4x", "edX", "gst_test", "graphical_slider_tool", "sample_gst"])
- gst_sample = modulestore.get_instance(sa_id, location)
+ sa_id = SlashSeparatedCourseKey("edX", "gst_test", "2012_Fall")
+ location = sa_id.make_usage_key("graphical_slider_tool", "sample_gst")
+ gst_sample = modulestore.get_item(location)
render_string_from_sample_gst_xml = """
<slider var="a" style="width:400px;float:left;"/>\
<plot style="margin-top:15px;margin-bottom:15px;"/>""".strip()
@@ -545,12 +537,8 @@ class ImportTestCase(BaseCourseTestCase):
self.assertEqual(len(sections), 1)
- location = course.location
- location = Location(
- location.tag, location.org, location.course,
- 'word_cloud', 'cloud1'
- )
- module = modulestore.get_instance(course.id, location)
+ location = course.id.make_usage_key('word_cloud', 'cloud1')
+ module = modulestore.get_item(location)
self.assertEqual(len(module.get_children()), 0)
self.assertEqual(module.num_inputs, 5)
self.assertEqual(module.num_top_words, 250)
@@ -561,7 +549,7 @@ class ImportTestCase(BaseCourseTestCase):
"""
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
- toy_id = "edX/toy/2012_Fall"
+ toy_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
course = modulestore.get_course(toy_id)
diff --git a/common/lib/xmodule/xmodule/tests/test_import_static.py b/common/lib/xmodule/xmodule/tests/test_import_static.py
index f19d41aaf44..caea3ce6bd7 100644
--- a/common/lib/xmodule/xmodule/tests/test_import_static.py
+++ b/common/lib/xmodule/xmodule/tests/test_import_static.py
@@ -3,8 +3,8 @@ Tests that check that we ignore the appropriate files when importing courses.
"""
import unittest
from mock import Mock
-from xmodule.modulestore import Location
from xmodule.modulestore.xml_importer import import_static_content
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.tests import DATA_DIR
@@ -12,10 +12,10 @@ class IgnoredFilesTestCase(unittest.TestCase):
"Tests for ignored files"
def test_ignore_tilde_static_files(self):
course_dir = DATA_DIR / "tilde"
- loc = Location("edX", "tilde", "Fall_2012")
+ course_id = SlashSeparatedCourseKey("edX", "tilde", "Fall_2012")
content_store = Mock()
content_store.generate_thumbnail.return_value = ("content", "location")
- import_static_content(Mock(), Mock(), course_dir, content_store, loc)
+ import_static_content(course_dir, content_store, course_id)
saved_static_content = [call[0][0] for call in content_store.save.call_args_list]
name_val = {sc.name: sc.data for sc in saved_static_content}
self.assertIn("example.txt", name_val)
diff --git a/common/lib/xmodule/xmodule/tests/test_lti_unit.py b/common/lib/xmodule/xmodule/tests/test_lti_unit.py
index 86cdabb3e79..1ab5fa5b680 100644
--- a/common/lib/xmodule/xmodule/tests/test_lti_unit.py
+++ b/common/lib/xmodule/xmodule/tests/test_lti_unit.py
@@ -262,26 +262,22 @@ class LTIModuleTest(LogicTest):
self.assertEqual(real_resource_link_id, expected_resource_link_id)
def test_lis_result_sourcedid(self):
- with patch('xmodule.lti_module.LTIModule.location', new_callable=PropertyMock) as mock_location:
- self.xmodule.location.html_id = lambda: 'i4x-2-3-lti-31de800015cf4afb973356dbe81496df'
- expected_sourcedId = u':'.join(urllib.quote(i) for i in (
- self.system.course_id,
- urllib.quote(self.unquoted_resource_link_id),
- self.user_id
- ))
- real_lis_result_sourcedid = self.xmodule.get_lis_result_sourcedid()
- self.assertEqual(real_lis_result_sourcedid, expected_sourcedId)
-
+ expected_sourcedId = u':'.join(urllib.quote(i) for i in (
+ self.system.course_id.to_deprecated_string(),
+ self.xmodule.get_resource_link_id(),
+ self.user_id
+ ))
+ real_lis_result_sourcedid = self.xmodule.get_lis_result_sourcedid()
+ self.assertEqual(real_lis_result_sourcedid, expected_sourcedId)
- @patch('xmodule.course_module.CourseDescriptor.id_to_location')
- def test_client_key_secret(self, test):
+ def test_client_key_secret(self):
"""
LTI module gets client key and secret provided.
"""
#this adds lti passports to system
mocked_course = Mock(lti_passports = ['lti_id:test_client:test_secret'])
modulestore = Mock()
- modulestore.get_item.return_value = mocked_course
+ modulestore.get_course.return_value = mocked_course
runtime = Mock(modulestore=modulestore)
self.xmodule.descriptor.runtime = runtime
self.xmodule.lti_id = "lti_id"
@@ -289,8 +285,7 @@ class LTIModuleTest(LogicTest):
expected = ('test_client', 'test_secret')
self.assertEqual(expected, (key, secret))
- @patch('xmodule.course_module.CourseDescriptor.id_to_location')
- def test_client_key_secret_not_provided(self, test):
+ def test_client_key_secret_not_provided(self):
"""
LTI module attempts to get client key and secret provided in cms.
@@ -300,7 +295,7 @@ class LTIModuleTest(LogicTest):
#this adds lti passports to system
mocked_course = Mock(lti_passports = ['test_id:test_client:test_secret'])
modulestore = Mock()
- modulestore.get_item.return_value = mocked_course
+ modulestore.get_course.return_value = mocked_course
runtime = Mock(modulestore=modulestore)
self.xmodule.descriptor.runtime = runtime
#set another lti_id
@@ -309,8 +304,7 @@ class LTIModuleTest(LogicTest):
expected = ('','')
self.assertEqual(expected, key_secret)
- @patch('xmodule.course_module.CourseDescriptor.id_to_location')
- def test_bad_client_key_secret(self, test):
+ def test_bad_client_key_secret(self):
"""
LTI module attempts to get client key and secret provided in cms.
@@ -319,16 +313,16 @@ class LTIModuleTest(LogicTest):
#this adds lti passports to system
mocked_course = Mock(lti_passports = ['test_id_test_client_test_secret'])
modulestore = Mock()
- modulestore.get_item.return_value = mocked_course
+ modulestore.get_course.return_value = mocked_course
runtime = Mock(modulestore=modulestore)
self.xmodule.descriptor.runtime = runtime
self.xmodule.lti_id = 'lti_id'
with self.assertRaises(LTIError):
self.xmodule.get_client_key_secret()
- @patch('xmodule.lti_module.signature.verify_hmac_sha1', return_value=True)
- @patch('xmodule.lti_module.LTIModule.get_client_key_secret', return_value=('test_client_key', u'test_client_secret'))
- def test_successful_verify_oauth_body_sign(self, get_key_secret, mocked_verify):
+ @patch('xmodule.lti_module.signature.verify_hmac_sha1', Mock(return_value=True))
+ @patch('xmodule.lti_module.LTIModule.get_client_key_secret', Mock(return_value=('test_client_key', u'test_client_secret')))
+ def test_successful_verify_oauth_body_sign(self):
"""
Test if OAuth signing was successful.
"""
@@ -337,9 +331,9 @@ class LTIModuleTest(LogicTest):
except LTIError:
self.fail("verify_oauth_body_sign() raised LTIError unexpectedly!")
- @patch('xmodule.lti_module.signature.verify_hmac_sha1', return_value=False)
- @patch('xmodule.lti_module.LTIModule.get_client_key_secret', return_value=('test_client_key', u'test_client_secret'))
- def test_failed_verify_oauth_body_sign(self, get_key_secret, mocked_verify):
+ @patch('xmodule.lti_module.signature.verify_hmac_sha1', Mock(return_value=False))
+ @patch('xmodule.lti_module.LTIModule.get_client_key_secret', Mock(return_value=('test_client_key', u'test_client_secret')))
+ def test_failed_verify_oauth_body_sign(self):
"""
Oauth signing verify fail.
"""
@@ -411,4 +405,4 @@ class LTIModuleTest(LogicTest):
"""
Tests that LTI parameter context_id is equal to course_id.
"""
- self.assertEqual(self.system.course_id, self.xmodule.context_id)
+ self.assertEqual(self.system.course_id.to_deprecated_string(), self.xmodule.context_id)
diff --git a/common/lib/xmodule/xmodule/tests/test_peer_grading.py b/common/lib/xmodule/xmodule/tests/test_peer_grading.py
index 8445527aea4..8f2e679b4a4 100644
--- a/common/lib/xmodule/xmodule/tests/test_peer_grading.py
+++ b/common/lib/xmodule/xmodule/tests/test_peer_grading.py
@@ -7,7 +7,7 @@ from webob.multidict import MultiDict
from xblock.field_data import DictFieldData
from xblock.fields import ScopeIds
-from xmodule.modulestore import Location
+from xmodule.modulestore.locations import Location, SlashSeparatedCourseKey
from xmodule.tests import get_test_system, get_test_descriptor_system
from xmodule.tests.test_util_open_ended import DummyModulestore
from xmodule.open_ended_grading_classes.peer_grading_service import MockPeerGradingService
@@ -16,20 +16,17 @@ from xmodule.modulestore.exceptions import ItemNotFoundError, NoPathToItem
log = logging.getLogger(__name__)
-ORG = "edX"
-COURSE = "open_ended"
-
class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
"""
Test peer grading xmodule at the unit level. More detailed tests are difficult, as the module relies on an
external grading service.
"""
- problem_location = Location(["i4x", "edX", "open_ended", "peergrading",
- "PeerGradingSample"])
- coe_location = Location(["i4x", "edX", "open_ended", "combinedopenended", "SampleQuestion"])
+ course_id = SlashSeparatedCourseKey('edX', 'open_ended', '2012_Fall')
+ problem_location = course_id.make_usage_key("peergrading", "PeerGradingSample")
+ coe_location = course_id.make_usage_key("combinedopenended", "SampleQuestion")
calibrated_dict = {'location': "blah"}
- coe_dict = {'location': coe_location.url()}
+ coe_dict = {'location': coe_location.to_deprecated_string()}
save_dict = MultiDict({
'location': "blah",
'submission_id': 1,
@@ -42,7 +39,7 @@ class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
save_dict.extend(('rubric_scores[]', val) for val in (0, 1))
def get_module_system(self, descriptor):
- test_system = get_test_system()
+ test_system = get_test_system(self.course_id)
test_system.open_ended_grading_interface = None
return test_system
@@ -51,9 +48,9 @@ class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
Create a peer grading module from a test system
@return:
"""
- self.setup_modulestore(COURSE)
- self.peer_grading = self.get_module_from_location(self.problem_location, COURSE)
- self.coe = self.get_module_from_location(self.coe_location, COURSE)
+ self.setup_modulestore(self.course_id.course)
+ self.peer_grading = self.get_module_from_location(self.problem_location)
+ self.coe = self.get_module_from_location(self.coe_location)
def test_module_closed(self):
"""
@@ -75,7 +72,7 @@ class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
Try getting data from the external grading service
@return:
"""
- success, _data = self.peer_grading.query_data_for_location(self.problem_location.url())
+ success, _data = self.peer_grading.query_data_for_location(self.problem_location.to_deprecated_string())
self.assertTrue(success)
def test_get_score_none(self):
@@ -149,8 +146,11 @@ class PeerGradingModuleTest(unittest.TestCase, DummyModulestore):
Mainly for diff coverage
@return:
"""
+ # pylint: disable=protected-access
with self.assertRaises(ItemNotFoundError):
- self.peer_grading._find_corresponding_module_for_location(Location('i4x', 'a', 'b', 'c', 'd'))
+ self.peer_grading._find_corresponding_module_for_location(
+ Location('org', 'course', 'run', 'category', 'name', 'revision')
+ )
def test_get_instance_state(self):
"""
@@ -235,7 +235,13 @@ class MockPeerGradingServiceProblemList(MockPeerGradingService):
def get_problem_list(self, course_id, grader_id):
return {'success': True,
'problem_list': [
- {"num_graded": 3, "num_pending": 681, "num_required": 3, "location": "i4x://edX/open_ended/combinedopenended/SampleQuestion", "problem_name": "Peer-Graded Essay"},
+ {
+ "num_graded": 3,
+ "num_pending": 681,
+ "num_required": 3,
+ "location": course_id.make_usage_key('combinedopenended', 'SampleQuestion'),
+ "problem_name": "Peer-Graded Essay"
+ },
]}
@@ -244,12 +250,12 @@ class PeerGradingModuleScoredTest(unittest.TestCase, DummyModulestore):
Test peer grading xmodule at the unit level. More detailed tests are difficult, as the module relies on an
external grading service.
"""
- problem_location = Location(
- ["i4x", "edX", "open_ended", "peergrading", "PeerGradingScored"]
- )
+
+ course_id = SlashSeparatedCourseKey('edX', 'open_ended', '2012_Fall')
+ problem_location = course_id.make_usage_key("peergrading", "PeerGradingScored")
def get_module_system(self, descriptor):
- test_system = get_test_system()
+ test_system = get_test_system(self.course_id)
test_system.open_ended_grading_interface = None
return test_system
@@ -258,10 +264,10 @@ class PeerGradingModuleScoredTest(unittest.TestCase, DummyModulestore):
Create a peer grading module from a test system
@return:
"""
- self.setup_modulestore(COURSE)
+ self.setup_modulestore(self.course_id.course)
def test_metadata_load(self):
- peer_grading = self.get_module_from_location(self.problem_location, COURSE)
+ peer_grading = self.get_module_from_location(self.problem_location)
self.assertFalse(peer_grading.closed())
def test_problem_list(self):
@@ -270,7 +276,7 @@ class PeerGradingModuleScoredTest(unittest.TestCase, DummyModulestore):
"""
# Initialize peer grading module.
- peer_grading = self.get_module_from_location(self.problem_location, COURSE)
+ peer_grading = self.get_module_from_location(self.problem_location)
# Ensure that it cannot find any peer grading.
html = peer_grading.peer_grading()
@@ -286,13 +292,12 @@ class PeerGradingModuleLinkedTest(unittest.TestCase, DummyModulestore):
"""
Test peer grading that is linked to an open ended module.
"""
- problem_location = Location(["i4x", "edX", "open_ended", "peergrading",
- "PeerGradingLinked"])
- coe_location = Location(["i4x", "edX", "open_ended", "combinedopenended",
- "SampleQuestion"])
+ course_id = SlashSeparatedCourseKey('edX', 'open_ended', '2012_Fall')
+ problem_location = course_id.make_usage_key("peergrading", "PeerGradingLinked")
+ coe_location = course_id.make_usage_key("combinedopenended", "SampleQuestion")
def get_module_system(self, descriptor):
- test_system = get_test_system()
+ test_system = get_test_system(self.course_id)
test_system.open_ended_grading_interface = None
return test_system
@@ -300,7 +305,7 @@ class PeerGradingModuleLinkedTest(unittest.TestCase, DummyModulestore):
"""
Create a peer grading module from a test system.
"""
- self.setup_modulestore(COURSE)
+ self.setup_modulestore(self.course_id.course)
@property
def field_data(self):
@@ -312,7 +317,7 @@ class PeerGradingModuleLinkedTest(unittest.TestCase, DummyModulestore):
'data': '<peergrading/>',
'location': self.problem_location,
'use_for_single_location': True,
- 'link_to_location': self.coe_location.url(),
+ 'link_to_location': self.coe_location.to_deprecated_string(),
'graded': True,
})
@@ -424,7 +429,7 @@ class PeerGradingModuleLinkedTest(unittest.TestCase, DummyModulestore):
peer_grading = self._create_peer_grading_with_linked_problem(self.coe_location)
# If we specify a location, it will render the problem for that location.
- data = peer_grading.handle_ajax('problem', {'location': self.coe_location})
+ data = peer_grading.handle_ajax('problem', {'location': self.coe_location.to_deprecated_string()})
self.assertTrue(json.loads(data)['success'])
# If we don't specify a location, it should use the linked location.
diff --git a/common/lib/xmodule/xmodule/tests/test_self_assessment.py b/common/lib/xmodule/xmodule/tests/test_self_assessment.py
index da89ce43359..e6d77f9a46d 100644
--- a/common/lib/xmodule/xmodule/tests/test_self_assessment.py
+++ b/common/lib/xmodule/xmodule/tests/test_self_assessment.py
@@ -4,6 +4,7 @@ import unittest
from mock import Mock, MagicMock
from webob.multidict import MultiDict
from pytz import UTC
+from xblock.fields import ScopeIds
from xmodule.open_ended_grading_classes.self_assessment_module import SelfAssessmentModule
from xmodule.modulestore import Location
from lxml import etree
@@ -29,8 +30,7 @@ class SelfAssessmentTest(unittest.TestCase):
'hintprompt': 'Consider this...',
}
- location = Location(["i4x", "edX", "sa_test", "selfassessment",
- "SampleQuestion"])
+ location = Location("edX", "sa_test", "run", "selfassessment", "SampleQuestion", None)
descriptor = Mock()
@@ -56,7 +56,10 @@ class SelfAssessmentTest(unittest.TestCase):
}
system = get_test_system()
- system.xmodule_instance = Mock(scope_ids=Mock(usage_id='dummy-usage-id'))
+
+ usage_key = system.course_id.make_usage_key('combinedopenended', 'test_loc')
+ scope_ids = ScopeIds(1, 'combinedopenended', usage_key, usage_key)
+ system.xmodule_instance = Mock(scope_ids=scope_ids)
self.module = SelfAssessmentModule(
system,
self.location,
diff --git a/common/lib/xmodule/xmodule/tests/test_tabs.py b/common/lib/xmodule/xmodule/tests/test_tabs.py
index 311cced0774..ec88ad2efba 100644
--- a/common/lib/xmodule/xmodule/tests/test_tabs.py
+++ b/common/lib/xmodule/xmodule/tests/test_tabs.py
@@ -2,6 +2,7 @@
from mock import MagicMock
import xmodule.tabs as tabs
import unittest
+from xmodule.modulestore.locations import SlashSeparatedCourseKey
class TabTestCase(unittest.TestCase):
@@ -9,7 +10,7 @@ class TabTestCase(unittest.TestCase):
def setUp(self):
self.course = MagicMock()
- self.course.id = 'edX/toy/2012_Fall'
+ self.course.id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
self.fake_dict_tab = {'fake_key': 'fake_value'}
self.settings = MagicMock()
self.settings.FEATURES = {}
@@ -137,7 +138,7 @@ class ProgressTestCase(TabTestCase):
return self.check_tab(
tab_class=tabs.ProgressTab,
dict_tab={'type': tabs.ProgressTab.type, 'name': 'same'},
- expected_link=self.reverse('progress', args=[self.course.id]),
+ expected_link=self.reverse('progress', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.ProgressTab.type,
invalid_dict_tab=None,
)
@@ -161,7 +162,7 @@ class WikiTestCase(TabTestCase):
return self.check_tab(
tab_class=tabs.WikiTab,
dict_tab={'type': tabs.WikiTab.type, 'name': 'same'},
- expected_link=self.reverse('course_wiki', args=[self.course.id]),
+ expected_link=self.reverse('course_wiki', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.WikiTab.type,
invalid_dict_tab=self.fake_dict_tab,
)
@@ -220,7 +221,7 @@ class StaticTabTestCase(TabTestCase):
tab = self.check_tab(
tab_class=tabs.StaticTab,
dict_tab={'type': tabs.StaticTab.type, 'name': 'same', 'url_slug': url_slug},
- expected_link=self.reverse('static_tab', args=[self.course.id, url_slug]),
+ expected_link=self.reverse('static_tab', args=[self.course.id.to_deprecated_string(), url_slug]),
expected_tab_id='static_tab_schmug',
invalid_dict_tab=self.fake_dict_tab,
)
@@ -257,7 +258,10 @@ class TextbooksTestCase(TabTestCase):
# verify all textbook type tabs
if isinstance(tab, tabs.SingleTextbookTab):
book_type, book_index = tab.tab_id.split("/", 1)
- expected_link = self.reverse(type_to_reverse_name[book_type], args=[self.course.id, book_index])
+ expected_link = self.reverse(
+ type_to_reverse_name[book_type],
+ args=[self.course.id.to_deprecated_string(), book_index]
+ )
self.assertEqual(tab.link_func(self.course, self.reverse), expected_link)
self.assertTrue(tab.name.startswith('Book{0}'.format(book_index)))
num_textbooks_found = num_textbooks_found + 1
@@ -279,7 +283,7 @@ class GradingTestCase(TabTestCase):
tab_class=tab_class,
dict_tab={'type': tab_class.type, 'name': name},
expected_name=name,
- expected_link=self.reverse(link_value, args=[self.course.id]),
+ expected_link=self.reverse(link_value, args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tab_class.type,
invalid_dict_tab=None,
)
@@ -314,7 +318,7 @@ class NotesTestCase(TabTestCase):
return self.check_tab(
tab_class=tabs.NotesTab,
dict_tab={'type': tabs.NotesTab.type, 'name': 'same'},
- expected_link=self.reverse('notes', args=[self.course.id]),
+ expected_link=self.reverse('notes', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.NotesTab.type,
invalid_dict_tab=self.fake_dict_tab,
)
@@ -341,7 +345,7 @@ class SyllabusTestCase(TabTestCase):
tab_class=tabs.SyllabusTab,
dict_tab={'type': tabs.SyllabusTab.type, 'name': name},
expected_name=name,
- expected_link=self.reverse('syllabus', args=[self.course.id]),
+ expected_link=self.reverse('syllabus', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.SyllabusTab.type,
invalid_dict_tab=None,
)
@@ -365,7 +369,7 @@ class InstructorTestCase(TabTestCase):
tab_class=tabs.InstructorTab,
dict_tab={'type': tabs.InstructorTab.type, 'name': name},
expected_name=name,
- expected_link=self.reverse('instructor_dashboard', args=[self.course.id]),
+ expected_link=self.reverse('instructor_dashboard', args=[self.course.id.to_deprecated_string()]),
expected_tab_id=tabs.InstructorTab.type,
invalid_dict_tab=None,
)
@@ -603,7 +607,7 @@ class DiscussionLinkTestCase(TabTestCase):
"""Custom reverse function"""
def reverse_discussion_link(viewname, args):
"""reverse lookup for discussion link"""
- if viewname == "django_comment_client.forum.views.forum_form_discussion" and args == [course.id]:
+ if viewname == "django_comment_client.forum.views.forum_form_discussion" and args == [course.id.to_deprecated_string()]:
return "default_discussion_link"
return reverse_discussion_link
diff --git a/common/lib/xmodule/xmodule/tests/test_util_open_ended.py b/common/lib/xmodule/xmodule/tests/test_util_open_ended.py
index 3f6b07fa0f9..0316b7caccd 100644
--- a/common/lib/xmodule/xmodule/tests/test_util_open_ended.py
+++ b/common/lib/xmodule/xmodule/tests/test_util_open_ended.py
@@ -92,11 +92,8 @@ class DummyModulestore(object):
courses = self.modulestore.get_courses()
return courses[0]
- def get_module_from_location(self, location, course):
- course = self.get_course(course)
- if not isinstance(location, Location):
- location = Location(location)
- descriptor = self.modulestore.get_instance(course.id, location, depth=None)
+ def get_module_from_location(self, usage_key):
+ descriptor = self.modulestore.get_item(usage_key, depth=None)
descriptor.xmodule_runtime = self.get_module_system(descriptor)
return descriptor
diff --git a/common/lib/xmodule/xmodule/tests/test_video.py b/common/lib/xmodule/xmodule/tests/test_video.py
index ed5afa2eddf..314869c0284 100644
--- a/common/lib/xmodule/xmodule/tests/test_video.py
+++ b/common/lib/xmodule/xmodule/tests/test_video.py
@@ -125,7 +125,7 @@ class VideoDescriptorTest(unittest.TestCase):
def setUp(self):
system = get_test_descriptor_system()
- location = Location('i4x://org/course/video/name')
+ location = Location('org', 'course', 'run', 'video', 'name', None)
self.descriptor = system.construct_xblock_from_class(
VideoDescriptor,
scope_ids=ScopeIds(None, None, location, location),
@@ -138,7 +138,7 @@ class VideoDescriptorTest(unittest.TestCase):
back out to XML.
"""
system = DummySystem(load_error_modules=True)
- location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
+ location = Location("edX", 'course', 'run', "video", 'SampleProblem1', None)
field_data = DictFieldData({'location': location})
descriptor = VideoDescriptor(system, field_data, Mock())
descriptor.youtube_id_0_75 = 'izygArpw-Qo'
@@ -154,7 +154,7 @@ class VideoDescriptorTest(unittest.TestCase):
in the output string.
"""
system = DummySystem(load_error_modules=True)
- location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
+ location = Location("edX", 'course', 'run', "video", "SampleProblem1", None)
field_data = DictFieldData({'location': location})
descriptor = VideoDescriptor(system, field_data, Mock())
descriptor.youtube_id_0_75 = 'izygArpw-Qo'
@@ -194,8 +194,7 @@ class VideoDescriptorImportTestCase(unittest.TestCase):
<transcript language="ge" src="german_translation.srt" />
</video>
'''
- location = Location(["i4x", "edX", "video", "default",
- "SampleProblem1"])
+ location = Location("edX", 'course', 'run', "video", "SampleProblem1", None)
field_data = DictFieldData({
'data': sample_xml,
'location': location
@@ -498,6 +497,9 @@ class VideoExportTestCase(unittest.TestCase):
Make sure that VideoDescriptor can export itself to XML
correctly.
"""
+ def setUp(self):
+ self.location = Location("edX", 'course', 'run', "video", "SampleProblem1", None)
+
def assertXmlEqual(self, expected, xml):
for attr in ['tag', 'attrib', 'text', 'tail']:
self.assertEqual(getattr(expected, attr), getattr(xml, attr))
@@ -507,8 +509,7 @@ class VideoExportTestCase(unittest.TestCase):
def test_export_to_xml(self):
"""Test that we write the correct XML on export."""
module_system = DummySystem(load_error_modules=True)
- location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
- desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, location, location))
+ desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, self.location, self.location))
desc.youtube_id_0_75 = 'izygArpw-Qo'
desc.youtube_id_1_0 = 'p2Q6BrNhdh8'
@@ -540,8 +541,7 @@ class VideoExportTestCase(unittest.TestCase):
def test_export_to_xml_empty_end_time(self):
"""Test that we write the correct XML on export."""
module_system = DummySystem(load_error_modules=True)
- location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
- desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, location, location))
+ desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, self.location, self.location))
desc.youtube_id_0_75 = 'izygArpw-Qo'
desc.youtube_id_1_0 = 'p2Q6BrNhdh8'
@@ -569,8 +569,7 @@ class VideoExportTestCase(unittest.TestCase):
def test_export_to_xml_empty_parameters(self):
"""Test XML export with defaults."""
module_system = DummySystem(load_error_modules=True)
- location = Location(["i4x", "edX", "video", "default", "SampleProblem1"])
- desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, location, location))
+ desc = VideoDescriptor(module_system, DictFieldData({}), ScopeIds(None, None, self.location, self.location))
xml = desc.definition_to_xml(None)
expected = '<video url_name="SampleProblem1"/>\n'
diff --git a/common/lib/xmodule/xmodule/tests/test_xblock_wrappers.py b/common/lib/xmodule/xmodule/tests/test_xblock_wrappers.py
index d54db605197..8183122eae8 100644
--- a/common/lib/xmodule/xmodule/tests/test_xblock_wrappers.py
+++ b/common/lib/xmodule/xmodule/tests/test_xblock_wrappers.py
@@ -190,7 +190,7 @@ class LeafDescriptorFactory(Factory):
@lazy_attribute
def location(self):
- return Location('i4x://org/course/category/{}'.format(self.url_name))
+ return Location('org', 'course', 'run', 'category', self.url_name, None)
@lazy_attribute
def block_type(self):
diff --git a/common/lib/xmodule/xmodule/tests/xml/__init__.py b/common/lib/xmodule/xmodule/tests/xml/__init__.py
index 416dfa8de21..1d3ea3f904e 100644
--- a/common/lib/xmodule/xmodule/tests/xml/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/xml/__init__.py
@@ -7,8 +7,8 @@ from unittest import TestCase
from xmodule.x_module import XMLParsingSystem, policy_key
from xmodule.mako_module import MakoDescriptorSystem
-from xmodule.modulestore.xml import create_block_from_xml, LocationReader, CourseLocationGenerator
-from xmodule.modulestore import Location
+from xmodule.modulestore.xml import create_block_from_xml, CourseLocationGenerator
+from xmodule.modulestore.locations import SlashSeparatedCourseKey, Location
from xblock.runtime import KvsFieldData, DictKeyValueStore
@@ -18,8 +18,7 @@ class InMemorySystem(XMLParsingSystem, MakoDescriptorSystem): # pylint: disable
The simplest possible XMLParsingSystem
"""
def __init__(self, xml_import_data):
- self.org = xml_import_data.org
- self.course = xml_import_data.course
+ self.course_id = SlashSeparatedCourseKey.from_deprecated_string(xml_import_data.course_id)
self.default_class = xml_import_data.default_class
self._descriptors = {}
@@ -37,7 +36,6 @@ class InMemorySystem(XMLParsingSystem, MakoDescriptorSystem): # pylint: disable
select=xml_import_data.xblock_select,
render_template=lambda template, context: pprint.pformat((template, context)),
field_data=KvsFieldData(DictKeyValueStore()),
- id_reader=LocationReader(),
)
def process_xml(self, xml): # pylint: disable=method-hidden
@@ -45,14 +43,14 @@ class InMemorySystem(XMLParsingSystem, MakoDescriptorSystem): # pylint: disable
descriptor = create_block_from_xml(
xml,
self,
- CourseLocationGenerator(self.org, self.course),
+ CourseLocationGenerator(self.course_id),
)
- self._descriptors[descriptor.location.url()] = descriptor
+ self._descriptors[descriptor.location.to_deprecated_string()] = descriptor
return descriptor
def load_item(self, location): # pylint: disable=method-hidden
"""Return the descriptor loaded for `location`"""
- return self._descriptors[Location(location).url()]
+ return self._descriptors[location.to_deprecated_string()]
class XModuleXmlImportTest(TestCase):
diff --git a/common/lib/xmodule/xmodule/tests/xml/factories.py b/common/lib/xmodule/xmodule/tests/xml/factories.py
index 24d81b46997..478c0d1f59e 100644
--- a/common/lib/xmodule/xmodule/tests/xml/factories.py
+++ b/common/lib/xmodule/xmodule/tests/xml/factories.py
@@ -17,15 +17,14 @@ class XmlImportData(object):
Class to capture all of the data needed to actually run an XML import,
so that the Factories have something to generate
"""
- def __init__(self, xml_node, xml=None, org=None, course=None,
+ def __init__(self, xml_node, xml=None, course_id=None,
default_class=None, policy=None,
filesystem=None, parent=None,
xblock_mixins=(), xblock_select=None):
self._xml_node = xml_node
self._xml_string = xml
- self.org = org
- self.course = course
+ self.course_id = course_id
self.default_class = default_class
self.filesystem = filesystem
self.xblock_mixins = xblock_mixins
@@ -47,8 +46,8 @@ class XmlImportData(object):
def __repr__(self):
return u"XmlImportData{!r}".format((
- self._xml_node, self._xml_string, self.org,
- self.course, self.default_class, self.policy,
+ self._xml_node, self._xml_string, self.course_id,
+ self.default_class, self.policy,
self.filesystem, self.parent, self.xblock_mixins,
self.xblock_select,
))
@@ -74,6 +73,7 @@ class XmlImportFactory(Factory):
policy = {}
inline_xml = True
tag = 'unknown'
+ course_id = 'edX/xml_test_course/101'
@classmethod
def _adjust_kwargs(cls, **kwargs):
@@ -136,8 +136,6 @@ class XmlImportFactory(Factory):
class CourseFactory(XmlImportFactory):
"""Factory for <course> nodes"""
tag = 'course'
- org = 'edX'
- course = 'xml_test_course'
name = '101'
static_asset_path = 'xml_test_course'
diff --git a/common/lib/xmodule/xmodule/vertical_module.py b/common/lib/xmodule/xmodule/vertical_module.py
index 0053cb5ca17..10456e4ce4f 100644
--- a/common/lib/xmodule/xmodule/vertical_module.py
+++ b/common/lib/xmodule/xmodule/vertical_module.py
@@ -25,7 +25,7 @@ class VerticalModule(VerticalFields, XModule):
fragment.add_frag_resources(rendered_child)
contents.append({
- 'id': child.id,
+ 'id': child.location.to_deprecated_string(),
'content': rendered_child.content
})
diff --git a/common/lib/xmodule/xmodule/video_module/transcripts_utils.py b/common/lib/xmodule/xmodule/video_module/transcripts_utils.py
index 57e77bca93b..2ec9ce4ec5c 100644
--- a/common/lib/xmodule/xmodule/video_module/transcripts_utils.py
+++ b/common/lib/xmodule/xmodule/video_module/transcripts_utils.py
@@ -289,9 +289,7 @@ def copy_or_rename_transcript(new_name, old_name, item, delete_old=False, user=N
If `delete_old` is True, removes `old_name` files from storage.
"""
filename = 'subs_{0}.srt.sjson'.format(old_name)
- content_location = StaticContent.compute_location(
- item.location.org, item.location.course, filename
- )
+ content_location = StaticContent.compute_location(item.location.course_key, filename)
transcripts = contentstore().find(content_location).data
save_subs_to_store(json.loads(transcripts), new_name, item)
item.sub = new_name
@@ -532,7 +530,7 @@ class Transcript(object):
"""
Return asset location. `location` is module location.
"""
- return StaticContent.compute_location(location.org, location.course, filename)
+ return StaticContent.compute_location(location.course_key, filename)
@staticmethod
def delete_asset(location, filename):
@@ -545,4 +543,5 @@ class Transcript(object):
log.info("Transcript asset %s was removed from store.", filename)
except NotFoundError:
pass
+ return StaticContent.compute_location(location.course_key, filename)
diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py
index 49d4789260a..a57d0edea69 100644
--- a/common/lib/xmodule/xmodule/x_module.py
+++ b/common/lib/xmodule/xmodule/x_module.py
@@ -18,14 +18,12 @@ from webob.multidict import MultiDict
from xblock.core import XBlock
from xblock.fields import Scope, Integer, Float, List, XBlockMixin, String, Dict
from xblock.fragment import Fragment
-from xblock.plugin import default_select
from xblock.runtime import Runtime
from xmodule.fields import RelativeTime
from xmodule.errortracker import exc_info_to_str
-from xmodule.modulestore import Location
-from xmodule.modulestore.exceptions import ItemNotFoundError, InsufficientSpecificationError, InvalidLocationError
-from xmodule.modulestore.locator import BlockUsageLocator
+from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.modulestore.keys import OpaqueKeyReader, UsageKey
from xmodule.exceptions import UndefinedContext
from dogapi import dog_stats_api
@@ -156,11 +154,7 @@ class XModuleMixin(XBlockMixin):
@property
def course_id(self):
- return self.runtime.course_id
-
- @property
- def id(self):
- return self.location.url()
+ return self.location.course_key
@property
def category(self):
@@ -168,16 +162,11 @@ class XModuleMixin(XBlockMixin):
@property
def location(self):
- try:
- return Location(self.scope_ids.usage_id)
- except InvalidLocationError:
- if isinstance(self.scope_ids.usage_id, BlockUsageLocator):
- return self.scope_ids.usage_id
- else:
- return BlockUsageLocator(self.scope_ids.usage_id)
+ return self.scope_ids.usage_id
@location.setter
def location(self, value):
+ assert isinstance(value, UsageKey)
self.scope_ids = self.scope_ids._replace(
def_id=value,
usage_id=value,
@@ -185,12 +174,7 @@ class XModuleMixin(XBlockMixin):
@property
def url_name(self):
- if isinstance(self.location, Location):
- return self.location.name
- elif isinstance(self.location, BlockUsageLocator):
- return self.location.block_id
- else:
- raise InsufficientSpecificationError()
+ return self.location.name
@property
def display_name_with_default(self):
@@ -203,6 +187,17 @@ class XModuleMixin(XBlockMixin):
name = self.url_name.replace('_', ' ')
return name
+ @property
+ def xblock_kvs(self):
+ """
+ Retrieves the internal KeyValueStore for this XModule.
+
+ Should only be used by the persistence layer. Use with caution.
+ """
+ # if caller wants kvs, caller's assuming it's up to date; so, decache it
+ self.save()
+ return self._field_data._kvs # pylint: disable=protected-access
+
def get_explicitly_set_fields_by_scope(self, scope=Scope.content):
"""
Get a dictionary of the fields for the given scope which are set explicitly on this xblock. (Including
@@ -214,15 +209,6 @@ class XModuleMixin(XBlockMixin):
result[field.name] = field.read_json(self)
return result
- @property
- def xblock_kvs(self):
- """
- Use w/ caution. Really intended for use by the persistence layer.
- """
- # if caller wants kvs, caller's assuming it's up to date; so, decache it
- self.save()
- return self._field_data._kvs # pylint: disable=protected-access
-
def get_content_titles(self):
"""
Returns list of content titles for all of self's children.
@@ -684,7 +670,6 @@ class XModuleDescriptor(XModuleMixin, HTMLSnippet, ResourceTemplates, XBlock):
Interpret the parsed XML in `node`, creating an XModuleDescriptor.
"""
xml = etree.tostring(node)
- # TODO: change from_xml to not take org and course, it can use self.system.
block = cls.from_xml(xml, runtime, id_generator)
return block
@@ -1023,7 +1008,7 @@ class DescriptorSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime): # p
local_resource_url: an implementation of :meth:`xblock.runtime.Runtime.local_resource_url`
"""
- super(DescriptorSystem, self).__init__(**kwargs)
+ super(DescriptorSystem, self).__init__(id_reader=OpaqueKeyReader(), **kwargs)
# This is used by XModules to write out separate files during xml export
self.export_fs = None
@@ -1217,7 +1202,7 @@ class ModuleSystem(MetricsMixin,ConfigurableFragmentWrapper, Runtime): # pylint
# Usage_store is unused, and field_data is often supplanted with an
# explicit field_data during construct_xblock.
- super(ModuleSystem, self).__init__(id_reader=None, field_data=field_data, **kwargs)
+ super(ModuleSystem, self).__init__(id_reader=OpaqueKeyReader(), field_data=field_data, **kwargs)
self.STATIC_URL = static_url
self.xqueue = xqueue
--
GitLab