diff --git a/cms/djangoapps/contentstore/management/commands/export_olx.py b/cms/djangoapps/contentstore/management/commands/export_olx.py new file mode 100644 index 0000000000000000000000000000000000000000..67a14fcad2419d8b3c006e8815e007dc16f81f5b --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/export_olx.py @@ -0,0 +1,111 @@ +""" +A Django command that exports a course to a tar.gz file. + +If <filename> is '-', it pipes the file to stdout. + +This is used by Analytics research exports to provide researchers +with course content. + +At present, it differs from Studio exports in several ways: + +* It does not include static content. +* The top-level directory in the resulting tarball is a "safe" + (i.e. ascii) version of the course_key, rather than the word "course". +* It only supports the export of courses. It does not export libraries. + +""" + +import os +import re +import shutil +import tarfile +from tempfile import mktemp, mkdtemp +from textwrap import dedent + +from path import Path as path + +from django.core.management.base import BaseCommand, CommandError + +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.xml_exporter import export_course_to_xml +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey + + +class Command(BaseCommand): + """ + Export a course to XML. The output is compressed as a tar.gz file. + + """ + help = dedent(__doc__).strip() + + def add_arguments(self, parser): + parser.add_argument('course_id') + parser.add_argument('--output', default=None) + + def handle(self, *args, **options): + + course_id = options['course_id'] + try: + course_key = CourseKey.from_string(course_id) + except InvalidKeyError: + raise CommandError("Unparsable course_id") + except IndexError: + raise CommandError("Insufficient arguments") + + filename = options['output'] + pipe_results = False + if filename is None: + filename = mktemp() + pipe_results = True + + export_course_to_tarfile(course_key, filename) + + results = self._get_results(filename) if pipe_results else None + + self.stdout.write(results, ending="") + + def _get_results(self, filename): + """Load results from file""" + with open(filename) as f: + results = f.read() + os.remove(filename) + return results + + +def export_course_to_tarfile(course_key, filename): + """Exports a course into a tar.gz file""" + tmp_dir = mkdtemp() + try: + course_dir = export_course_to_directory(course_key, tmp_dir) + compress_directory(course_dir, filename) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +def export_course_to_directory(course_key, root_dir): + """Export course into a directory""" + store = modulestore() + course = store.get_course(course_key) + if course is None: + raise CommandError("Invalid course_id") + + # The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>. + # We represent the first four with \w. + # TODO: Once we support courses with unicode characters, we will need to revisit this. + replacement_char = u'-' + course_dir = replacement_char.join([course.id.org, course.id.course, course.id.run]) + course_dir = re.sub(r'[^\w\.\-]', replacement_char, course_dir) + + export_course_to_xml(store, None, course.id, root_dir, course_dir) + + export_dir = path(root_dir) / course_dir + return export_dir + + +def compress_directory(directory, filename): + """Compress a directory into a tar.gz file""" + mode = 'w:gz' + name = path(directory).name + with tarfile.open(filename, mode) as tar_file: + tar_file.add(directory, arcname=name) diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py b/cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py new file mode 100644 index 0000000000000000000000000000000000000000..c2be823d484686283aaa1437e5192f5644d4b6b7 --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py @@ -0,0 +1,94 @@ +""" +Tests for exporting OLX content. +""" + +import ddt +from path import Path as path +import shutil +from StringIO import StringIO +import tarfile +from tempfile import mkdtemp +import unittest + +from django.core.management import CommandError, call_command + +from xmodule.modulestore.tests.factories import CourseFactory +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.django import modulestore + + +class TestArgParsingCourseExportOlx(unittest.TestCase): + """ + Tests for parsing arguments for the `export_olx` management command + """ + def test_no_args(self): + """ + Test export command with no arguments + """ + errstring = "Error: too few arguments" + with self.assertRaisesRegexp(CommandError, errstring): + call_command('export_olx') + + +@ddt.ddt +class TestCourseExportOlx(ModuleStoreTestCase): + """ + Test exporting OLX content from a course or library. + """ + + def test_invalid_course_key(self): + """ + Test export command with an invalid course key. + """ + errstring = "Unparsable course_id" + with self.assertRaisesRegexp(CommandError, errstring): + call_command('export_olx', 'InvalidCourseID') + + def test_course_key_not_found(self): + """ + Test export command with a valid course key that doesn't exist. + """ + errstring = "Invalid course_id" + with self.assertRaisesRegexp(CommandError, errstring): + call_command('export_olx', 'x/y/z') + + def create_dummy_course(self, store_type): + """Create small course.""" + course = CourseFactory.create(default_store=store_type) + self.assertTrue( + modulestore().has_course(course.id), + "Could not find course in {}".format(store_type) + ) + return course.id + + def check_export_file(self, tar_file, course_key): + """Check content of export file.""" + names = tar_file.getnames() + dirname = "{0.org}-{0.course}-{0.run}".format(course_key) + self.assertIn(dirname, names) + # Check if some of the files are present, without being exhaustive. + self.assertIn("{}/about".format(dirname), names) + self.assertIn("{}/about/overview.html".format(dirname), names) + self.assertIn("{}/assets/assets.xml".format(dirname), names) + self.assertIn("{}/policies".format(dirname), names) + + @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) + def test_export_course(self, store_type): + test_course_key = self.create_dummy_course(store_type) + tmp_dir = path(mkdtemp()) + self.addCleanup(shutil.rmtree, tmp_dir) + filename = tmp_dir / 'test.tar.gz' + call_command('export_olx', '--output', filename, unicode(test_course_key)) + with tarfile.open(filename) as tar_file: + self.check_export_file(tar_file, test_course_key) + + @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) + def test_export_course_stdout(self, store_type): + test_course_key = self.create_dummy_course(store_type) + out = StringIO() + call_command('export_olx', unicode(test_course_key), stdout=out) + out.seek(0) + output = out.read() + with tarfile.open(fileobj=StringIO(output)) as tar_file: + self.check_export_file(tar_file, test_course_key) diff --git a/lms/djangoapps/courseware/management/commands/export_course.py b/lms/djangoapps/courseware/management/commands/export_course.py index 1b6bb0b594cacf43c259565ebd5afd2734950679..27a6c4497e42cf350cb76d9ec3ee1cd6c3744837 100644 --- a/lms/djangoapps/courseware/management/commands/export_course.py +++ b/lms/djangoapps/courseware/management/commands/export_course.py @@ -3,6 +3,14 @@ A Django command that exports a course to a tar.gz file. If <filename> is '-', it pipes the file to stdout +NOTE: This used to be used by Analytics research exports to provide +researchers with course content. It is now DEPRECATED, and +functionality has moved to export_olx.py in +cms/djangoapps/contentstore/management/commands. + +Note: when removing this file, also remove references to it +from test_dump_course. + """ import os