Skip to content
Snippets Groups Projects
Commit ecb05e6b authored by Calen Pennington's avatar Calen Pennington
Browse files

Merge pull request #2905 from edx/opaque-keys

(WIP) Make course ids and Locations/Locators opaque to LMS/Studio
parents c96bf07b 05ca40b4
No related branches found
No related tags found
No related merge requests found
Showing
with 143 additions and 137 deletions
import ConfigParser
from django.conf import settings
import logging
......
# pylint: disable=C0111
# pylint: disable=W0621
import time
import os
from lettuce import world, step
from nose.tools import assert_true, assert_in # pylint: disable=no-name-in-module
from django.conf import settings
from student.roles import CourseRole, CourseStaffRole, CourseInstructorRole
from student.roles import CourseStaffRole, CourseInstructorRole, GlobalStaff
from student.models import get_user
from selenium.webdriver.common.keys import Keys
......@@ -162,7 +161,7 @@ def add_course_author(user, course):
"""
global_admin = AdminFactory()
for role in (CourseStaffRole, CourseInstructorRole):
auth.add_users(global_admin, role(course.location), user)
auth.add_users(global_admin, role(course.id), user)
def create_a_course():
......@@ -380,18 +379,17 @@ def create_other_user(_step, name, has_extra_perms, role_name):
user = create_studio_user(uname=name, password="test", email=email)
if has_extra_perms:
if role_name == "is_staff":
user.is_staff = True
user.save()
GlobalStaff().add_users(user)
else:
if role_name == "admin":
# admins get staff privileges, as well
roles = (CourseStaffRole, CourseInstructorRole)
else:
roles = (CourseStaffRole,)
location = world.scenario_dict["COURSE"].location
course_key = world.scenario_dict["COURSE"].id
global_admin = AdminFactory()
for role in roles:
auth.add_users(global_admin, role(location), user)
auth.add_users(global_admin, role(course_key), user)
@step('I log out')
......
......@@ -5,6 +5,8 @@
from lettuce import world, step
from component_settings_editor_helpers import enter_xml_in_advanced_problem
from nose.tools import assert_true, assert_equal
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_usage_url
@step('I go to the export page$')
......@@ -49,4 +51,9 @@ def get_an_error_dialog(step):
def i_click_on_error_dialog(step):
world.click_link_by_text('Correct failed component')
assert_true(world.css_html("span.inline-error").startswith("Problem i4x://MITx/999/problem"))
assert_equal(1, world.browser.url.count("unit/MITx.999.Robot_Super_Course/branch/draft/block/vertical"))
course_key = SlashSeparatedCourseKey("MITx", "999", "Robot_Super_Course")
# we don't know the actual ID of the vertical. So just check that we did go to a
# vertical page in the course (there should only be one).
vertical_usage_key = course_key.make_usage_key("vertical", "")
vertical_url = reverse_usage_url('unit_handler', vertical_usage_key)
assert_equal(1, world.browser.url.count(vertical_url))
......@@ -4,8 +4,9 @@
from lettuce import world, step
from common import *
from terrain.steps import reload_the_page
from selenium.common.exceptions import (
InvalidElementStateException, WebDriverException)
from selenium.common.exceptions import InvalidElementStateException
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_course_url
from nose.tools import assert_in, assert_not_in, assert_equal, assert_not_equal # pylint: disable=E0611
......@@ -68,11 +69,12 @@ def change_assignment_name(step, old_name, new_name):
@step(u'I go back to the main course page')
def main_course_page(step):
course_name = world.scenario_dict['COURSE'].display_name.replace(' ', '_')
main_page_link = '/course/{org}.{number}.{name}/branch/draft/block/{name}'.format(
org=world.scenario_dict['COURSE'].org,
number=world.scenario_dict['COURSE'].number,
name=course_name
course_key = SlashSeparatedCourseKey(
world.scenario_dict['COURSE'].org,
world.scenario_dict['COURSE'].number,
course_name
)
main_page_link = reverse_course_url('course_handler', course_key)
world.visit(main_page_link)
assert_in('Course Outline', world.css_text('h1.page-header'))
......
......@@ -14,11 +14,11 @@ Feature: CMS.Sign in
Scenario: Login with a valid redirect
Given I have opened a new course in Studio
And I am not logged in
And I visit the url "/course/MITx.999.Robot_Super_Course/branch/draft/block/Robot_Super_Course"
And I should see that the path is "/signin?next=/course/MITx.999.Robot_Super_Course/branch/draft/block/Robot_Super_Course"
And I visit the url "/course/slashes:MITx+999+Robot_Super_Course"
And I should see that the path is "/signin?next=/course/slashes%3AMITx%2B999%2BRobot_Super_Course"
When I fill in and submit the signin form
And I wait for "2" seconds
Then I should see that the path is "/course/MITx.999.Robot_Super_Course/branch/draft/block/Robot_Super_Course"
Then I should see that the path is "/course/slashes:MITx+999+Robot_Super_Course"
Scenario: Login with an invalid redirect
Given I have opened a new course in Studio
......@@ -26,4 +26,4 @@ Feature: CMS.Sign in
And I visit the url "/signin?next=http://www.google.com/"
When I fill in and submit the signin form
And I wait for "2" seconds
Then I should see that the path is "/course"
Then I should see that the path is "/course/"
......@@ -166,8 +166,7 @@ def remove_transcripts_from_store(_step, subs_id):
"""Remove from store, if transcripts content exists."""
filename = 'subs_{0}.srt.sjson'.format(subs_id.strip())
content_location = StaticContent.compute_location(
world.scenario_dict['COURSE'].org,
world.scenario_dict['COURSE'].number,
world.scenario_dict['COURSE'].id,
filename
)
try:
......
......@@ -154,7 +154,7 @@ def user_foo_is_enrolled_in_the_course(step, name):
world.create_user(name, 'test')
user = User.objects.get(username=name)
course_id = world.scenario_dict['COURSE'].location.course_id
course_id = world.scenario_dict['COURSE'].id
CourseEnrollment.enroll(user, course_id)
......
......@@ -140,10 +140,10 @@ def xml_only_video(step):
# Wait for the new unit to be created and to load the page
world.wait(1)
location = world.scenario_dict['COURSE'].location
store = get_modulestore(location)
course = world.scenario_dict['COURSE']
store = get_modulestore(course.location)
parent_location = store.get_items(Location(category='vertical', revision='draft'))[0].location
parent_location = store.get_items(course.id, category='vertical', revision='draft')[0].location
youtube_id = 'ABCDEFG'
world.scenario_dict['YOUTUBE_ID'] = youtube_id
......
......@@ -14,7 +14,6 @@ from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from xmodule.contentstore.django import contentstore
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_exporter import export_to_xml
......@@ -64,13 +63,10 @@ def cmd_log(cmd, cwd):
return output
def export_to_git(course_loc, repo, user='', rdir=None):
def export_to_git(course_id, repo, user='', rdir=None):
"""Export a course to git."""
# pylint: disable=R0915
if course_loc.startswith('i4x://'):
course_loc = course_loc[6:]
if not GIT_REPO_EXPORT_DIR:
raise GitExportError(GitExportError.NO_EXPORT_DIR)
......@@ -129,15 +125,10 @@ def export_to_git(course_loc, repo, user='', rdir=None):
raise GitExportError(GitExportError.CANNOT_PULL)
# export course as xml before commiting and pushing
try:
location = CourseDescriptor.id_to_location(course_loc)
except ValueError:
raise GitExportError(GitExportError.BAD_COURSE)
root_dir = os.path.dirname(rdirp)
course_dir = os.path.splitext(os.path.basename(rdirp))[0]
try:
export_to_xml(modulestore('direct'), contentstore(), location,
export_to_xml(modulestore('direct'), contentstore(), course_id,
root_dir, course_dir, modulestore())
except (EnvironmentError, AttributeError):
log.exception('Failed export to xml')
......
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_importer import check_module_metadata_editability
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore import Location
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
class Command(BaseCommand):
......@@ -10,14 +11,16 @@ class Command(BaseCommand):
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("check_course requires one argument: <location>")
raise CommandError("check_course requires one argument: <course_id>")
loc_str = args[0]
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
loc = CourseDescriptor.id_to_location(loc_str)
store = modulestore()
course = store.get_item(loc, depth=3)
course = store.get_course(course_key, depth=3)
err_cnt = 0
......@@ -33,7 +36,7 @@ class Command(BaseCommand):
def _check_xml_attributes_field(module):
err_cnt = 0
if hasattr(module, 'xml_attributes') and isinstance(module.xml_attributes, basestring):
print 'module = {0} has xml_attributes as a string. It should be a dict'.format(module.location.url())
print 'module = {0} has xml_attributes as a string. It should be a dict'.format(module.location)
err_cnt = err_cnt + 1
for child in module.get_children():
err_cnt = err_cnt + _check_xml_attributes_field(child)
......@@ -45,7 +48,7 @@ class Command(BaseCommand):
def _get_discussion_items(module):
discussion_items = []
if module.location.category == 'discussion':
discussion_items = discussion_items + [module.location.url()]
discussion_items = discussion_items + [module.location]
for child in module.get_children():
discussion_items = discussion_items + _get_discussion_items(child)
......@@ -55,17 +58,8 @@ class Command(BaseCommand):
discussion_items = _get_discussion_items(course)
# now query all discussion items via get_items() and compare with the tree-traversal
queried_discussion_items = store.get_items(
Location(
'i4x',
course.location.org,
course.location.course,
'discussion',
None,
None
)
)
queried_discussion_items = store.get_items(course_key=course_key, category='discussion',)
for item in queried_discussion_items:
if item.location.url() not in discussion_items:
print 'Found dangling discussion module = {0}'.format(item.location.url())
if item.location not in discussion_items:
print 'Found dangling discussion module = {0}'.format(item.location)
......@@ -5,9 +5,10 @@ from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.store_utilities import clone_course
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.course_module import CourseDescriptor
from student.roles import CourseInstructorRole, CourseStaffRole
from xmodule.modulestore import Location
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
#
......@@ -17,34 +18,36 @@ class Command(BaseCommand):
"""Clone a MongoDB-backed course to another location"""
help = 'Clone a MongoDB backed course to another location'
def course_key_from_arg(self, arg):
"""
Convert the command line arg into a course key
"""
try:
return CourseKey.from_string(arg)
except InvalidKeyError:
return SlashSeparatedCourseKey.from_deprecated_string(arg)
def handle(self, *args, **options):
"Execute the command"
if len(args) != 2:
raise CommandError("clone requires 2 arguments: <source-course_id> <dest-course_id>")
source_course_id = args[0]
dest_course_id = args[1]
source_course_id = self.course_key_from_arg(args[0])
dest_course_id = self.course_key_from_arg(args[1])
mstore = modulestore('direct')
cstore = contentstore()
course_id_dict = Location.parse_course_id(dest_course_id)
mstore.ignore_write_events_on_courses.append('{org}/{course}'.format(**course_id_dict))
mstore.ignore_write_events_on_courses.add(dest_course_id)
print("Cloning course {0} to {1}".format(source_course_id, dest_course_id))
source_location = CourseDescriptor.id_to_location(source_course_id)
dest_location = CourseDescriptor.id_to_location(dest_course_id)
if clone_course(mstore, cstore, source_location, dest_location):
# be sure to recompute metadata inheritance after all those updates
mstore.refresh_cached_metadata_inheritance_tree(dest_location)
if clone_course(mstore, cstore, source_course_id, dest_course_id):
print("copying User permissions...")
# purposely avoids auth.add_user b/c it doesn't have a caller to authorize
CourseInstructorRole(dest_location).add_users(
*CourseInstructorRole(source_location).users_with_role()
CourseInstructorRole(dest_course_id).add_users(
*CourseInstructorRole(source_course_id).users_with_role()
)
CourseStaffRole(dest_location).add_users(
*CourseStaffRole(source_location).users_with_role()
CourseStaffRole(dest_course_id).add_users(
*CourseStaffRole(source_course_id).users_with_role()
)
......@@ -4,6 +4,9 @@
from django.core.management.base import BaseCommand, CommandError
from .prompt import query_yes_no
from contentstore.utils import delete_course_and_groups
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
class Command(BaseCommand):
......@@ -11,9 +14,12 @@ class Command(BaseCommand):
def handle(self, *args, **options):
if len(args) != 1 and len(args) != 2:
raise CommandError("delete_course requires one or more arguments: <location> |commit|")
raise CommandError("delete_course requires one or more arguments: <course_id> |commit|")
course_id = args[0]
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
commit = False
if len(args) == 2:
......@@ -22,6 +28,6 @@ class Command(BaseCommand):
if commit:
print('Actually going to delete the course from DB....')
if query_yes_no("Deleting course {0}. Confirm?".format(course_id), default="no"):
if query_yes_no("Deleting course {0}. Confirm?".format(course_key), default="no"):
if query_yes_no("Are you sure. This action cannot be undone!", default="no"):
delete_course_and_groups(course_id, commit)
delete_course_and_groups(course_key, commit)
......@@ -13,6 +13,9 @@ from .prompt import query_yes_no
from courseware.courses import get_course_by_id
from contentstore.views import tabs
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.modulestore.keys import CourseKey
def print_course(course):
......@@ -64,7 +67,12 @@ command again, adding --insert or --delete to edit the list.
if not options['course']:
raise CommandError(Command.course_option.help)
course = get_course_by_id(options['course'])
try:
course_key = CourseKey.from_string(options['course'])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(options['course'])
course = get_course_by_id(course_key)
print 'Warning: this command directly edits the list of course tabs in mongo.'
print 'Tabs before any changes:'
......
from django.core.management.base import BaseCommand, CommandError
from xmodule.course_module import CourseDescriptor
from xmodule.contentstore.utils import empty_asset_trashcan
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from .prompt import query_yes_no
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
class Command(BaseCommand):
......@@ -10,16 +12,17 @@ class Command(BaseCommand):
def handle(self, *args, **options):
if len(args) != 1 and len(args) != 0:
raise CommandError("empty_asset_trashcan requires one or no arguments: |<location>|")
locs = []
raise CommandError("empty_asset_trashcan requires one or no arguments: |<course_id>|")
if len(args) == 1:
locs.append(CourseDescriptor.id_to_location(args[0]))
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
course_ids = [course_key]
else:
courses = modulestore('direct').get_courses()
for course in courses:
locs.append(course.location)
course_ids = [course.id for course in modulestore('direct').get_courses()]
if query_yes_no("Emptying trashcan. Confirm?", default="no"):
empty_asset_trashcan(locs)
empty_asset_trashcan(course_ids)
......@@ -6,8 +6,10 @@ import os
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from xmodule.contentstore.django import contentstore
from xmodule.course_module import CourseDescriptor
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
class Command(BaseCommand):
......@@ -19,16 +21,18 @@ class Command(BaseCommand):
def handle(self, *args, **options):
"Execute the command"
if len(args) != 2:
raise CommandError("export requires two arguments: <course location> <output path>")
raise CommandError("export requires two arguments: <course id> <output path>")
course_id = args[0]
output_path = args[1]
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
print("Exporting course id = {0} to {1}".format(course_id, output_path))
output_path = args[1]
location = CourseDescriptor.id_to_location(course_id)
print("Exporting course id = {0} to {1}".format(course_key, output_path))
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore('direct'), contentstore(), location, root_dir, course_dir, modulestore())
export_to_xml(modulestore('direct'), contentstore(), course_key, root_dir, course_dir, modulestore())
......@@ -5,7 +5,6 @@ from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.course_module import CourseDescriptor
class Command(BaseCommand):
......@@ -35,9 +34,8 @@ class Command(BaseCommand):
if 1:
try:
location = CourseDescriptor.id_to_location(course_id)
course_dir = course_id.replace('/', '...')
export_to_xml(ms, cs, location, root_dir, course_dir, modulestore())
export_to_xml(ms, cs, course_id, root_dir, course_dir, modulestore())
except Exception as err:
print("="*30 + "> Oops, failed to export %s" % course_id)
print("Error:")
......
......@@ -20,6 +20,10 @@ from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import ugettext as _
import contentstore.git_export_utils as git_export_utils
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from opaque_keys import InvalidKeyError
from contentstore.git_export_utils import GitExportError
from xmodule.modulestore.keys import CourseKey
log = logging.getLogger(__name__)
......@@ -52,9 +56,17 @@ class Command(BaseCommand):
'course_loc and git_url')
# Rethrow GitExportError as CommandError for SystemExit
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
try:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
except InvalidKeyError:
raise CommandError(GitExportError.BAD_COURSE)
try:
git_export_utils.export_to_git(
args[0],
course_key,
args[1],
options.get('user', ''),
options.get('rdir', None)
......
......@@ -47,11 +47,12 @@ class Command(BaseCommand):
_, course_items = import_from_xml(
mstore, data_dir, course_dirs, load_error_modules=False,
static_content_store=contentstore(), verbose=True,
do_import_static=do_import_static
do_import_static=do_import_static,
create_new_course=True,
)
for module in course_items:
course_id = module.location.course_id
for course in course_items:
course_id = course.id
if not are_permissions_roles_seeded(course_id):
self.stdout.write('Seeding forum roles for course {0}\n'.format(course_id))
seed_permissions_roles(course_id)
"""
Script for traversing all courses and add/modify mapping with 'lower_id' and 'lower_course_id'
"""
from django.core.management.base import BaseCommand
from xmodule.modulestore.django import modulestore, loc_mapper
#
# To run from command line: ./manage.py cms --settings dev map_courses_location_lower
#
class Command(BaseCommand):
"""
Create or modify map entry for each course in 'loc_mapper' with 'lower_id' and 'lower_course_id'
"""
help = "Create or modify map entry for each course in 'loc_mapper' with 'lower_id' and 'lower_course_id'"
def handle(self, *args, **options):
# get all courses
courses = modulestore('direct').get_courses()
for course in courses:
# create/modify map_entry in 'loc_mapper' with 'lower_id' and 'lower_course_id'
loc_mapper().create_map_entry(course.location)
......@@ -4,11 +4,12 @@ to the new split-Mongo modulestore.
"""
from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
from xmodule.modulestore import Location
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.split_migrator import SplitMigrator
from xmodule.modulestore import InvalidLocationError
from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
def user_from_str(identifier):
......@@ -30,24 +31,23 @@ class Command(BaseCommand):
"Migrate a course from old-Mongo to split-Mongo"
help = "Migrate a course from old-Mongo to split-Mongo"
args = "location email <locator>"
args = "course_key email <new org> <new offering>"
def parse_args(self, *args):
"""
Return a three-tuple of (location, user, locator_string).
If the user didn't specify a locator string, the third return value
will be None.
Return a 4-tuple of (course_key, user, org, offering).
If the user didn't specify an org & offering, those will be None.
"""
if len(args) < 2:
raise CommandError(
"migrate_to_split requires at least two arguments: "
"a location and a user identifier (email or ID)"
"a course_key and a user identifier (email or ID)"
)
try:
location = Location(args[0])
except InvalidLocationError:
raise CommandError("Invalid location string {}".format(args[0]))
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
try:
user = user_from_str(args[1])
......@@ -55,14 +55,15 @@ class Command(BaseCommand):
raise CommandError("No user found identified by {}".format(args[1]))
try:
package_id = args[2]
org = args[2]
offering = args[3]
except IndexError:
package_id = None
org = offering = None
return location, user, package_id
return course_key, user, org, offering
def handle(self, *args, **options):
location, user, package_id = self.parse_args(*args)
course_key, user, org, offering = self.parse_args(*args)
migrator = SplitMigrator(
draft_modulestore=modulestore('default'),
......@@ -71,4 +72,4 @@ class Command(BaseCommand):
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(location, user, package_id)
migrator.migrate_mongo_course(course_key, user, org, offering)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment