From 11fc1184025d41fe8b06fabce66bc7347592af7e Mon Sep 17 00:00:00 2001
From: Tim McCormack <tmccormack@edx.org>
Date: Thu, 5 Mar 2020 21:57:12 +0000
Subject: [PATCH] Remove django-celery from transitive deps; remove unused
tasks (#23318)
The video thumbnail and transcript tasks were the only things using chord_task from
edx-celeryutils, which in turn was blocking django-celery removal. But, they're no longer used.
Co-authored-by: Diana Huang <diana.k.huang@gmail.com>
---
.../commands/migrate_transcripts.py | 155 --------
.../tests/test_migrate_transcripts.py | 332 -----------------
.../commands/tests/test_video_thumbnails.py | 137 -------
.../management/commands/video_thumbnails.py | 86 -----
cms/djangoapps/contentstore/tasks.py | 342 ------------------
requirements/edx/base.txt | 12 +-
requirements/edx/development.txt | 14 +-
requirements/edx/testing.txt | 12 +-
8 files changed, 19 insertions(+), 1071 deletions(-)
delete mode 100644 cms/djangoapps/contentstore/management/commands/migrate_transcripts.py
delete mode 100644 cms/djangoapps/contentstore/management/commands/tests/test_migrate_transcripts.py
delete mode 100644 cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py
delete mode 100644 cms/djangoapps/contentstore/management/commands/video_thumbnails.py
diff --git a/cms/djangoapps/contentstore/management/commands/migrate_transcripts.py b/cms/djangoapps/contentstore/management/commands/migrate_transcripts.py
deleted file mode 100644
index ccc7c172664..00000000000
--- a/cms/djangoapps/contentstore/management/commands/migrate_transcripts.py
+++ /dev/null
@@ -1,155 +0,0 @@
-"""
-Command to migrate transcripts to django storage.
-"""
-
-
-import logging
-
-from django.core.management import BaseCommand, CommandError
-from opaque_keys import InvalidKeyError
-from opaque_keys.edx.keys import CourseKey
-from opaque_keys.edx.locator import CourseLocator
-from six.moves import map
-
-from cms.djangoapps.contentstore.tasks import (
- DEFAULT_ALL_COURSES,
- DEFAULT_COMMIT,
- DEFAULT_FORCE_UPDATE,
- enqueue_async_migrate_transcripts_tasks
-)
-from openedx.core.djangoapps.video_config.models import MigrationEnqueuedCourse, TranscriptMigrationSetting
-from openedx.core.lib.command_utils import get_mutually_exclusive_required_option, parse_course_keys
-from xmodule.modulestore.django import modulestore
-
-log = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
- """
- Example usage:
- $ ./manage.py cms migrate_transcripts --all-courses --force-update --commit
- $ ./manage.py cms migrate_transcripts --course-id 'Course1' --course-id 'Course2' --commit
- $ ./manage.py cms migrate_transcripts --from-settings
- """
- help = 'Migrates transcripts to S3 for one or more courses.'
-
- def add_arguments(self, parser):
- """
- Add arguments to the command parser.
- """
- parser.add_argument(
- '--course-id', '--course_id',
- dest='course_ids',
- action='append',
- help=u'Migrates transcripts for the list of courses.'
- )
- parser.add_argument(
- '--all-courses', '--all', '--all_courses',
- dest='all_courses',
- action='store_true',
- default=DEFAULT_ALL_COURSES,
- help=u'Migrates transcripts to the configured django storage for all courses.'
- )
- parser.add_argument(
- '--from-settings', '--from_settings',
- dest='from_settings',
- help='Migrate Transcripts with settings set via django admin',
- action='store_true',
- default=False,
- )
- parser.add_argument(
- '--force-update', '--force_update',
- dest='force_update',
- action='store_true',
- default=DEFAULT_FORCE_UPDATE,
- help=u'Force migrate transcripts for the requested courses, overwrite if already present.'
- )
- parser.add_argument(
- '--commit',
- dest='commit',
- action='store_true',
- default=DEFAULT_COMMIT,
- help=u'Commits the discovered video transcripts to django storage. '
- u'Without this flag, the command will return the transcripts discovered for migration.'
- )
-
- def _parse_course_key(self, raw_value):
- """ Parses course key from string """
- try:
- result = CourseKey.from_string(raw_value)
- except InvalidKeyError:
- raise CommandError(u"Invalid course_key: '%s'." % raw_value)
-
- if not isinstance(result, CourseLocator):
- raise CommandError(u"Argument {0} is not a course key".format(raw_value))
-
- return result
-
- def _get_migration_options(self, options):
- """
- Returns the command arguments configured via django admin.
- """
- force_update = options['force_update']
- commit = options['commit']
- courses_mode = get_mutually_exclusive_required_option(options, 'course_ids', 'all_courses', 'from_settings')
- if courses_mode == 'all_courses':
- course_keys = [course.id for course in modulestore().get_course_summaries()]
- elif courses_mode == 'course_ids':
- course_keys = list(map(self._parse_course_key, options['course_ids']))
- else:
- migration_settings = self._latest_settings()
- if migration_settings.all_courses:
- all_courses = [course.id for course in modulestore().get_course_summaries()]
- # Following is to avoid re-rerunning migrations for the already enqueued courses.
- # Although the migrations job is idempotent, but we need to track if the transcript migration
- # job was initiated for specific course(s) in order to elevate load from the workers and for
- # the job to be able identify the past enqueued courses.
- migrated_courses = MigrationEnqueuedCourse.objects.all().values_list('course_id', flat=True)
- non_migrated_courses = [
- course_key
- for course_key in all_courses
- if course_key not in migrated_courses
- ]
- # Course batch to be migrated.
- course_keys = non_migrated_courses[:migration_settings.batch_size]
-
- log.info(
- (u'[Transcript Migration] Courses(total): %s, '
- u'Courses(migrated): %s, Courses(non-migrated): %s, '
- u'Courses(migration-in-process): %s'),
- len(all_courses),
- len(migrated_courses),
- len(non_migrated_courses),
- len(course_keys),
- )
- else:
- course_keys = parse_course_keys(migration_settings.course_ids.split())
-
- force_update = migration_settings.force_update
- commit = migration_settings.commit
-
- return course_keys, force_update, commit
-
- def _latest_settings(self):
- """
- Return the latest version of the TranscriptMigrationSetting
- """
- return TranscriptMigrationSetting.current()
-
- def handle(self, *args, **options):
- """
- Invokes the migrate transcripts enqueue function.
- """
- migration_settings = self._latest_settings()
- course_keys, force_update, commit = self._get_migration_options(options)
- command_run = migration_settings.increment_run() if commit else -1
- enqueue_async_migrate_transcripts_tasks(
- course_keys=course_keys, commit=commit, command_run=command_run, force_update=force_update
- )
-
- if commit and options.get('from_settings') and migration_settings.all_courses:
- for course_key in course_keys:
- enqueued_course, created = MigrationEnqueuedCourse.objects.get_or_create(course_id=course_key)
- if created:
- enqueued_course.command_run = command_run
- enqueued_course.save()
diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_migrate_transcripts.py b/cms/djangoapps/contentstore/management/commands/tests/test_migrate_transcripts.py
deleted file mode 100644
index 143605d06ed..00000000000
--- a/cms/djangoapps/contentstore/management/commands/tests/test_migrate_transcripts.py
+++ /dev/null
@@ -1,332 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Tests for course transcript migration management command.
-"""
-
-
-import itertools
-import logging
-from datetime import datetime
-
-import ddt
-import pytz
-import six
-from django.core.management import CommandError, call_command
-from django.test import TestCase
-from edxval import api as api
-from mock import patch
-from testfixtures import LogCapture
-
-from openedx.core.djangoapps.video_config.models import MigrationEnqueuedCourse, TranscriptMigrationSetting
-from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
-from xmodule.video_module import VideoBlock
-from xmodule.video_module.transcripts_utils import save_to_store
-
-LOGGER_NAME = "cms.djangoapps.contentstore.tasks"
-
-SRT_FILEDATA = '''
-0
-00:00:00,270 --> 00:00:02,720
-sprechen sie deutsch?
-
-1
-00:00:02,720 --> 00:00:05,430
-Ja, ich spreche Deutsch
-
-2
-00:00:6,500 --> 00:00:08,600
-å¯ä»¥ç”¨â€œæˆ‘ä¸å¤ªæ‡‚艺术 但我知é“我喜欢什么â€åšæ¯”å–»
-'''
-
-CRO_SRT_FILEDATA = '''
-0
-00:00:00,270 --> 00:00:02,720
-Dobar dan!
-
-1
-00:00:02,720 --> 00:00:05,430
-Kako ste danas?
-
-2
-00:00:6,500 --> 00:00:08,600
-å¯ä»¥ç”¨â€œæˆ‘ä¸å¤ªæ‡‚艺术 但我知é“我喜欢什么â€åšæ¯”å–»
-'''
-
-
-VIDEO_DICT_STAR = dict(
- client_video_id='TWINKLE TWINKLE',
- duration=42.0,
- edx_video_id='test_edx_video_id',
- status='upload',
-)
-
-
-class TestArgParsing(TestCase):
- """
- Tests for parsing arguments for the `migrate_transcripts` management command
- """
- def test_no_args(self):
- errstring = "Must specify exactly one of --course_ids, --all_courses, --from_settings"
- with self.assertRaisesRegex(CommandError, errstring):
- call_command('migrate_transcripts')
-
- def test_invalid_course(self):
- errstring = "Invalid course_key: 'invalid-course'."
- with self.assertRaisesRegex(CommandError, errstring):
- call_command('migrate_transcripts', '--course-id', 'invalid-course')
-
-
-@ddt.ddt
-class TestMigrateTranscripts(ModuleStoreTestCase):
- """
- Tests migrating video transcripts in courses from contentstore to django storage
- """
- def setUp(self):
- """ Common setup. """
- super(TestMigrateTranscripts, self).setUp()
- self.store = modulestore()
- self.course = CourseFactory.create()
- self.course_2 = CourseFactory.create()
-
- video = {
- 'edx_video_id': 'test_edx_video_id',
- 'client_video_id': 'test1.mp4',
- 'duration': 42.0,
- 'status': 'upload',
- 'courses': [six.text_type(self.course.id)],
- 'encoded_videos': [],
- 'created': datetime.now(pytz.utc)
- }
- api.create_video(video)
-
- video_sample_xml = '''
- <video display_name="Test Video"
- edx_video_id="test_edx_video_id"
- youtube="1.0:p2Q6BrNhdh8,0.75:izygArpw-Qo,1.25:1EeWXzPdhSA,1.5:rABDYkeK0x8"
- show_captions="false"
- download_track="false"
- start_time="1.0"
- download_video="false"
- end_time="60.0">
- <source src="http://www.example.com/source.mp4"/>
- <track src="http://www.example.com/track"/>
- <handout src="http://www.example.com/handout"/>
- <transcript language="ge" src="subs_grmtran1.srt" />
- <transcript language="hr" src="subs_croatian1.srt" />
- </video>
- '''
-
- video_sample_xml_2 = '''
- <video display_name="Test Video 2"
- edx_video_id="test_edx_video_id_2"
- youtube="1.0:p2Q6BrNhdh8,0.75:izygArpw-Qo,1.25:1EeWXzPdhSA,1.5:rABDYkeK0x8"
- show_captions="false"
- download_track="false"
- start_time="1.0"
- download_video="false"
- end_time="60.0">
- <source src="http://www.example.com/source.mp4"/>
- <track src="http://www.example.com/track"/>
- <handout src="http://www.example.com/handout"/>
- <transcript language="ge" src="not_found.srt" />
- </video>
- '''
- self.video_descriptor = ItemFactory.create(
- parent_location=self.course.location, category='video',
- **VideoBlock.parse_video_xml(video_sample_xml)
- )
- self.video_descriptor_2 = ItemFactory.create(
- parent_location=self.course_2.location, category='video',
- **VideoBlock.parse_video_xml(video_sample_xml_2)
- )
-
- save_to_store(SRT_FILEDATA, 'subs_grmtran1.srt', 'text/srt', self.video_descriptor.location)
- save_to_store(CRO_SRT_FILEDATA, 'subs_croatian1.srt', 'text/srt', self.video_descriptor.location)
-
- def test_migrated_transcripts_count_with_commit(self):
- """
- Test migrating transcripts with commit
- """
- # check that transcript does not exist
- languages = api.get_available_transcript_languages(self.video_descriptor.edx_video_id)
- self.assertEqual(len(languages), 0)
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- # now call migrate_transcripts command and check the transcript availability
- call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--commit')
-
- languages = api.get_available_transcript_languages(self.video_descriptor.edx_video_id)
- self.assertEqual(len(languages), 2)
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- def test_migrated_transcripts_without_commit(self):
- """
- Test migrating transcripts as a dry-run
- """
- # check that transcripts do not exist
- languages = api.get_available_transcript_languages(self.video_descriptor.edx_video_id)
- self.assertEqual(len(languages), 0)
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- # now call migrate_transcripts command and check the transcript availability
- call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id))
-
- # check that transcripts still do not exist
- languages = api.get_available_transcript_languages(self.video_descriptor.edx_video_id)
- self.assertEqual(len(languages), 0)
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- def test_migrate_transcripts_availability(self):
- """
- Test migrating transcripts
- """
- translations = self.video_descriptor.available_translations(self.video_descriptor.get_transcripts_info())
- six.assertCountEqual(self, translations, ['hr', 'ge'])
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- # now call migrate_transcripts command and check the transcript availability
- call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--commit')
-
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- def test_migrate_transcripts_idempotency(self):
- """
- Test migrating transcripts multiple times
- """
- translations = self.video_descriptor.available_translations(self.video_descriptor.get_transcripts_info())
- six.assertCountEqual(self, translations, ['hr', 'ge'])
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- # now call migrate_transcripts command and check the transcript availability
- call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--commit')
-
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- # now call migrate_transcripts command again and check the transcript availability
- call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--commit')
-
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- # now call migrate_transcripts command with --force-update and check the transcript availability
- call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--force-update', '--commit')
-
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
- self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
-
- def test_migrate_transcripts_logging(self):
- """
- Test migrate transcripts logging and output
- """
- course_id = six.text_type(self.course.id)
- expected_log = (
- (
- 'cms.djangoapps.contentstore.tasks', 'INFO',
- (u'[Transcript Migration] [run=-1] [video-transcripts-migration-process-started-for-course] '
- u'[course={}]'.format(course_id))
- ),
- (
- 'cms.djangoapps.contentstore.tasks', 'INFO',
- (u'[Transcript Migration] [run=-1] [video-transcript-will-be-migrated] '
- u'[revision=rev-opt-published-only] [video={}] [edx_video_id=test_edx_video_id] '
- u'[language_code=hr]'.format(self.video_descriptor.location))
- ),
- (
- 'cms.djangoapps.contentstore.tasks', 'INFO',
- (u'[Transcript Migration] [run=-1] [video-transcript-will-be-migrated] '
- u'[revision=rev-opt-published-only] [video={}] [edx_video_id=test_edx_video_id] '
- u'[language_code=ge]'.format(self.video_descriptor.location))
- ),
- (
- 'cms.djangoapps.contentstore.tasks', 'INFO',
- (u'[Transcript Migration] [run=-1] [transcripts-migration-tasks-submitted] '
- u'[transcripts_count=2] [course={}] '
- u'[revision=rev-opt-published-only] [video={}]'.format(course_id, self.video_descriptor.location))
- )
- )
-
- with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
- call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id))
- logger.check(
- *expected_log
- )
-
- def test_migrate_transcripts_exception_logging(self):
- """
- Test migrate transcripts exception logging
- """
- course_id = six.text_type(self.course_2.id)
- expected_log = (
- (
- 'cms.djangoapps.contentstore.tasks', 'INFO',
- (u'[Transcript Migration] [run=1] [video-transcripts-migration-process-started-for-course] '
- u'[course={}]'.format(course_id))
- ),
- (
- 'cms.djangoapps.contentstore.tasks', 'INFO',
- (u'[Transcript Migration] [run=1] [transcripts-migration-process-started-for-video-transcript] '
- u'[revision=rev-opt-published-only] [video={}] [edx_video_id=test_edx_video_id_2] '
- u'[language_code=ge]'.format(self.video_descriptor_2.location))
- ),
- (
- 'cms.djangoapps.contentstore.tasks', 'ERROR',
- (u'[Transcript Migration] [run=1] [video-transcript-migration-failed-with-known-exc] '
- u'[revision=rev-opt-published-only] [video={}] [edx_video_id=test_edx_video_id_2] '
- u'[language_code=ge]'.format(self.video_descriptor_2.location))
- ),
- (
- 'cms.djangoapps.contentstore.tasks', 'INFO',
- (u'[Transcript Migration] [run=1] [transcripts-migration-tasks-submitted] '
- u'[transcripts_count=1] [course={}] '
- u'[revision=rev-opt-published-only] [video={}]'.format(course_id, self.video_descriptor_2.location))
- )
- )
-
- with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
- call_command('migrate_transcripts', '--course-id', six.text_type(self.course_2.id), '--commit')
- logger.check(
- *expected_log
- )
-
- @ddt.data(*itertools.product([1, 2], [True, False], [True, False]))
- @ddt.unpack
- @patch('contentstore.management.commands.migrate_transcripts.log')
- def test_migrate_transcripts_batch_size(self, batch_size, commit, all_courses, mock_logger):
- """
- Test that migrations across course batches, is working as expected.
- """
- migration_settings = TranscriptMigrationSetting.objects.create(
- batch_size=batch_size, commit=commit, all_courses=all_courses
- )
-
- # Assert the number of job runs and migration enqueued courses.
- self.assertEqual(migration_settings.command_run, 0)
- self.assertEqual(MigrationEnqueuedCourse.objects.count(), 0)
-
- call_command('migrate_transcripts', '--from-settings')
-
- migration_settings = TranscriptMigrationSetting.current()
- # Command run is only incremented if commit=True.
- expected_command_run = 1 if commit else 0
- self.assertEqual(migration_settings.command_run, expected_command_run)
-
- if all_courses:
- mock_logger.info.assert_called_with(
- (u'[Transcript Migration] Courses(total): %s, Courses(migrated): %s, '
- u'Courses(non-migrated): %s, Courses(migration-in-process): %s'),
- 2, 0, 2, batch_size
- )
-
- # enqueued courses are only persisted if commit=True and job is running for all courses.
- enqueued_courses = batch_size if commit and all_courses else 0
- self.assertEqual(MigrationEnqueuedCourse.objects.count(), enqueued_courses)
diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py b/cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py
deleted file mode 100644
index 063640af1c2..00000000000
--- a/cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Tests for course video thumbnails management command.
-"""
-
-
-import logging
-
-from django.core.management import CommandError, call_command
-from django.test import TestCase
-from mock import patch
-from six import text_type
-from testfixtures import LogCapture
-
-from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting
-from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.modulestore.tests.factories import CourseFactory
-
-LOGGER_NAME = "contentstore.management.commands.video_thumbnails"
-
-
-def setup_video_thumbnails_config(batch_size=10, commit=False, all_course_videos=False, course_ids=''):
- VideoThumbnailSetting.objects.create(
- batch_size=batch_size,
- commit=commit,
- course_ids=course_ids,
- all_course_videos=all_course_videos,
- videos_per_task=2
- )
-
-
-class TestArgParsing(TestCase):
- """
- Tests for parsing arguments for the `video_thumbnails` management command
- """
- def test_invalid_course(self):
- errstring = "Invalid key specified: <class 'opaque_keys.edx.locator.CourseLocator'>: invalid-course"
- setup_video_thumbnails_config(course_ids='invalid-course')
- with self.assertRaisesRegex(CommandError, errstring):
- call_command('video_thumbnails')
-
-
-class TestVideoThumbnails(ModuleStoreTestCase):
- """
- Tests adding thumbnails to course videos from YouTube
- """
- def setUp(self):
- """ Common setup """
- super(TestVideoThumbnails, self).setUp()
- self.course = CourseFactory.create()
- self.course_2 = CourseFactory.create()
-
- @patch('edxval.api.get_course_video_ids_with_youtube_profile')
- @patch('contentstore.management.commands.video_thumbnails.enqueue_update_thumbnail_tasks')
- def test_video_thumbnails_without_commit(self, mock_enqueue_thumbnails, mock_course_videos):
- """
- Test that when command is run without commit, correct information is logged.
- """
- course_videos = [
- (self.course.id, 'super-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP80'),
- (self.course_2.id, 'medium-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP81')
- ]
- mock_course_videos.return_value = course_videos
-
- setup_video_thumbnails_config(all_course_videos=True)
-
- with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
- call_command('video_thumbnails')
- # Verify that list of course video ids is logged.
- logger.check(
- (
- LOGGER_NAME, 'INFO',
- '[Video Thumbnails] Videos(updated): 0, Videos(update-in-process): 2'
- ),
- (
- LOGGER_NAME, 'INFO',
- u'[video thumbnails] selected course videos: {course_videos} '.format(
- course_videos=text_type(course_videos)
- )
- )
- )
-
- # Verify that `enqueue_update_thumbnail_tasks` is not called.
- self.assertFalse(mock_enqueue_thumbnails.called)
-
- @patch('edxval.api.get_course_video_ids_with_youtube_profile')
- @patch('contentstore.management.commands.video_thumbnails.enqueue_update_thumbnail_tasks')
- def test_video_thumbnails_with_commit(self, mock_enqueue_thumbnails, mock_course_videos):
- """
- Test that when command is run with with commit, it works as expected.
- """
- course_videos = [
- (self.course.id, 'super-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP80'),
- (self.course_2.id, 'medium-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP81')
- ]
- mock_course_videos.return_value = course_videos
- setup_video_thumbnails_config(commit=True, all_course_videos=True)
- with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
- call_command('video_thumbnails')
- # Verify that command information correctly logged.
- logger.check((
- LOGGER_NAME, 'INFO',
- '[Video Thumbnails] Videos(updated): 0, Videos(update-in-process): 2'
- ))
- # Verify that `enqueue_update_thumbnail_tasks` is called.
- self.assertTrue(mock_enqueue_thumbnails.called)
-
- @patch('edxval.api.get_course_video_ids_with_youtube_profile')
- @patch('contentstore.video_utils.download_youtube_video_thumbnail') # Mock(side_effect=Exception())
- def test_video_thumbnails_scraping_failed(self, mock_scrape_thumbnails, mock_course_videos):
- """
- Test that when scraping fails, it is handled correclty.
- """
- course_videos = [
- (self.course.id, 'super-soaker', 'OscRe3pSP80'),
- (self.course_2.id, 'medium-soaker', 'OscRe3pSP81')
- ]
- mock_scrape_thumbnails.side_effect = Exception('error')
- mock_course_videos.return_value = course_videos
- setup_video_thumbnails_config(commit=True, all_course_videos=True)
-
- tasks_logger = "cms.djangoapps.contentstore.tasks"
- with LogCapture(tasks_logger, level=logging.INFO) as logger:
- call_command('video_thumbnails')
- # Verify that tasks information is correctly logged.
- logger.check(
- (
- tasks_logger, 'ERROR',
- (u"[video thumbnails] [run=1] [video-thumbnails-scraping-failed-with-unknown-exc] "
- u"[edx_video_id=super-soaker] [youtube_id=OscRe3pSP80] [course={}]".format(self.course.id))
- ),
- (
- tasks_logger, 'ERROR',
- (u"[video thumbnails] [run=1] [video-thumbnails-scraping-failed-with-unknown-exc] "
- u"[edx_video_id=medium-soaker] [youtube_id=OscRe3pSP81] [course={}]".format(self.course_2.id))
- )
- )
diff --git a/cms/djangoapps/contentstore/management/commands/video_thumbnails.py b/cms/djangoapps/contentstore/management/commands/video_thumbnails.py
deleted file mode 100644
index f34b6c1766e..00000000000
--- a/cms/djangoapps/contentstore/management/commands/video_thumbnails.py
+++ /dev/null
@@ -1,86 +0,0 @@
-"""
-Command to scrape thumbnails and add them to the course-videos.
-"""
-
-
-import logging
-
-import edxval.api as edxval_api
-from django.core.management import BaseCommand
-from django.core.management.base import CommandError
-from opaque_keys import InvalidKeyError
-from opaque_keys.edx.keys import CourseKey
-from six import text_type
-
-from cms.djangoapps.contentstore.tasks import enqueue_update_thumbnail_tasks
-from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting
-
-log = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
- """
- Example usage:
- $ ./manage.py cms video_thumbnails
- """
- help = 'Adds thumbnails from YouTube to videos'
-
- def _get_command_options(self):
- """
- Returns the command arguments configured via django admin.
- """
- command_settings = self._latest_settings()
- commit = command_settings.commit
- if command_settings.all_course_videos:
- course_videos = edxval_api.get_course_video_ids_with_youtube_profile(
- offset=command_settings.offset, limit=command_settings.batch_size
- )
- log.info(
- u'[Video Thumbnails] Videos(updated): %s, Videos(update-in-process): %s',
- command_settings.offset, len(course_videos),
- )
- else:
- validated_course_ids = self._validate_course_ids(command_settings.course_ids.split())
- course_videos = edxval_api.get_course_video_ids_with_youtube_profile(validated_course_ids)
-
- return course_videos, commit
-
- def _validate_course_ids(self, course_ids):
- """
- Validate a list of course key strings.
- """
- try:
- for course_id in course_ids:
- CourseKey.from_string(course_id)
- return course_ids
- except InvalidKeyError as error:
- raise CommandError(u'Invalid key specified: {}'.format(text_type(error)))
-
- def _latest_settings(self):
- """
- Return the latest version of the VideoThumbnailSetting
- """
- return VideoThumbnailSetting.current()
-
- def handle(self, *args, **options):
- """
- Invokes the video thumbnail enqueue function.
- """
- video_thumbnail_settings = self._latest_settings()
- videos_per_task = video_thumbnail_settings.videos_per_task
-
- course_videos, commit = self._get_command_options()
-
- if commit:
- command_run = video_thumbnail_settings.increment_run()
- enqueue_update_thumbnail_tasks(
- course_videos=course_videos,
- videos_per_task=videos_per_task,
- run=command_run
- )
- if video_thumbnail_settings.all_course_videos:
- video_thumbnail_settings.update_offset()
- else:
- log.info(u'[video thumbnails] selected course videos: {course_videos} '.format(
- course_videos=text_type(course_videos)
- ))
diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py
index 51bb4be8d56..b1112dbbaae 100644
--- a/cms/djangoapps/contentstore/tasks.py
+++ b/cms/djangoapps/contentstore/tasks.py
@@ -15,7 +15,6 @@ from tempfile import NamedTemporaryFile, mkdtemp
from celery import group
from celery.task import task
from celery.utils.log import get_task_logger
-from celery_utils.chordable_django_backend import chord, chord_task
from celery_utils.persist_on_failure import LoggedPersistOnFailureTask
from django.conf import settings
from django.contrib.auth import get_user_model
@@ -76,347 +75,6 @@ User = get_user_model()
LOGGER = get_task_logger(__name__)
FILE_READ_CHUNK = 1024 # bytes
FULL_COURSE_REINDEX_THRESHOLD = 1
-DEFAULT_ALL_COURSES = False
-DEFAULT_FORCE_UPDATE = False
-DEFAULT_COMMIT = False
-MIGRATION_LOGS_PREFIX = 'Transcript Migration'
-
-RETRY_DELAY_SECONDS = 30
-COURSE_LEVEL_TIMEOUT_SECONDS = 1200
-VIDEO_LEVEL_TIMEOUT_SECONDS = 300
-
-
-def enqueue_update_thumbnail_tasks(course_videos, videos_per_task, run):
- """
- Enqueue tasks to update video thumbnails from youtube.
-
- Arguments:
- course_videos: A list of tuples, each containing course ID, video ID and youtube ID.
- videos_per_task: Number of course videos that can be processed by a single celery task.
- run: This tracks the YT thumbnail scraping job runs.
- """
- tasks = []
- batch_size = len(course_videos)
- # Further slice the course-videos batch into chunks on the
- # basis of number of course-videos per task.
- start = 0
- end = videos_per_task
- chunks_count = int(ceil(batch_size / float(videos_per_task)))
- for __ in range(0, chunks_count): # pylint: disable=C7620
- course_videos_chunk = course_videos[start:end]
- tasks.append(task_scrape_youtube_thumbnail.s(
- course_videos_chunk, run
- ))
- start = end
- end += videos_per_task
-
- # Kick off a chord of scraping tasks
- callback = task_scrape_youtube_thumbnail_callback.s(
- run=run,
- batch_size=batch_size,
- videos_per_task=videos_per_task,
- )
- chord(tasks)(callback)
-
-
-@chord_task(bind=True, routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE)
-def task_scrape_youtube_thumbnail_callback(self, results, run, # pylint: disable=unused-argument
- batch_size, videos_per_task):
- """
- Callback for collating the results of yt thumbnails scraping tasks chord.
- """
- yt_thumbnails_scraping_tasks_count = len(list(results()))
- LOGGER.info(
- (u"[video thumbnails] [run=%s] [video-thumbnails-scraping-complete-for-a-batch] [tasks_count=%s] "
- u"[batch_size=%s] [videos_per_task=%s]"),
- run, yt_thumbnails_scraping_tasks_count, batch_size, videos_per_task
- )
-
-
-@chord_task(
- bind=True,
- base=LoggedPersistOnFailureTask,
- default_retry_delay=RETRY_DELAY_SECONDS,
- max_retries=1,
- time_limit=COURSE_LEVEL_TIMEOUT_SECONDS,
- routing_key=settings.SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE
-)
-def task_scrape_youtube_thumbnail(self, course_videos, run): # pylint: disable=unused-argument
- """
- Task to scrape youtube thumbnails and update them in edxval for the given course-videos.
-
- Arguments:
- course_videos: A list of tuples, each containing course ID, video ID and youtube ID.
- run: This tracks the YT thumbnail scraping job runs.
- """
- for course_id, edx_video_id, youtube_id in course_videos:
- try:
- scrape_youtube_thumbnail(course_id, edx_video_id, youtube_id)
- except Exception: # pylint: disable=broad-except
- LOGGER.exception(
- (u"[video thumbnails] [run=%s] [video-thumbnails-scraping-failed-with-unknown-exc] "
- u"[edx_video_id=%s] [youtube_id=%s] [course=%s]"),
- run,
- edx_video_id,
- youtube_id,
- course_id
- )
- continue
-
-
-@chord_task(bind=True, routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE)
-def task_status_callback(self, results, revision, # pylint: disable=unused-argument
- course_id, command_run, video_location):
- """
- Callback for collating the results of chord.
- """
- transcript_tasks_count = len(list(results()))
-
- LOGGER.info(
- (u"[%s] [run=%s] [video-transcripts-migration-complete-for-a-video] [tasks_count=%s] [course_id=%s] "
- u"[revision=%s] [video=%s]"),
- MIGRATION_LOGS_PREFIX, command_run, transcript_tasks_count, course_id, revision, video_location
- )
-
-
-def enqueue_async_migrate_transcripts_tasks(course_keys,
- command_run,
- force_update=DEFAULT_FORCE_UPDATE,
- commit=DEFAULT_COMMIT):
- """
- Fires new Celery tasks for all the input courses or for all courses.
-
- Arguments:
- course_keys: Command line course ids as list of CourseKey objects
- command_run: A positive number indicating the run counts for transcripts migrations
- force_update: Overwrite file in S3. Default is False
- commit: Update S3 or dry-run the command to see which transcripts will be affected. Default is False
- """
- kwargs = {
- 'force_update': force_update,
- 'commit': commit,
- 'command_run': command_run
- }
- group([
- async_migrate_transcript.s(text_type(course_key), **kwargs)
- for course_key in course_keys
- ])()
-
-
-def get_course_videos(course_key):
- """
- Returns all videos in a course as list.
-
- Arguments:
- course_key: CourseKey object
- """
- all_videos = {}
- store = modulestore()
-
- # include published videos of the course.
- all_videos[ModuleStoreEnum.RevisionOption.published_only] = store.get_items(
- course_key,
- qualifiers={'category': 'video'},
- revision=ModuleStoreEnum.RevisionOption.published_only,
- include_orphans=False
- )
-
- # include draft videos of the course.
- all_videos[ModuleStoreEnum.RevisionOption.draft_only] = store.get_items(
- course_key,
- qualifiers={'category': 'video'},
- revision=ModuleStoreEnum.RevisionOption.draft_only,
- include_orphans=False
- )
-
- return all_videos
-
-
-@chord_task(
- bind=True,
- base=LoggedPersistOnFailureTask,
- default_retry_delay=RETRY_DELAY_SECONDS,
- max_retries=1,
- time_limit=COURSE_LEVEL_TIMEOUT_SECONDS,
- routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE
-)
-def async_migrate_transcript(self, course_key, **kwargs): # pylint: disable=unused-argument
- """
- Migrates the transcripts of all videos in a course as a new celery task.
- """
- force_update = kwargs['force_update']
- command_run = kwargs['command_run']
- course_videos = get_course_videos(CourseKey.from_string(course_key))
-
- LOGGER.info(
- u"[%s] [run=%s] [video-transcripts-migration-process-started-for-course] [course=%s]",
- MIGRATION_LOGS_PREFIX, command_run, course_key
- )
-
- for revision, videos in course_videos.items():
- for video in videos:
- # Gather transcripts from a video block.
- all_transcripts = {}
- if video.transcripts is not None:
- all_transcripts.update(video.transcripts)
-
- english_transcript = video.sub
- if english_transcript:
- all_transcripts.update({'en': video.sub})
-
- sub_tasks = []
- video_location = text_type(video.location)
- for lang in all_transcripts:
- sub_tasks.append(async_migrate_transcript_subtask.s(
- video_location, revision, lang, force_update, **kwargs
- ))
-
- if sub_tasks:
- callback = task_status_callback.s(
- revision=revision,
- course_id=course_key,
- command_run=command_run,
- video_location=video_location
- )
- chord(sub_tasks)(callback)
-
- LOGGER.info(
- (u"[%s] [run=%s] [transcripts-migration-tasks-submitted] "
- u"[transcripts_count=%s] [course=%s] [revision=%s] [video=%s]"),
- MIGRATION_LOGS_PREFIX, command_run, len(sub_tasks), course_key, revision, video_location
- )
- else:
- LOGGER.info(
- u"[%s] [run=%s] [no-video-transcripts] [course=%s] [revision=%s] [video=%s]",
- MIGRATION_LOGS_PREFIX, command_run, course_key, revision, video_location
- )
-
-
-def save_transcript_to_storage(command_run, edx_video_id, language_code, transcript_content, file_format, force_update):
- """
- Pushes a given transcript's data to django storage.
-
- Arguments:
- command_run: A positive integer indicating the current run
- edx_video_id: video ID
- language_code: language code
- transcript_content: content of the transcript
- file_format: format of the transcript file
- force_update: tells whether it needs to perform force update in
- case of an existing transcript for the given video.
- """
- transcript_present = is_transcript_available(video_id=edx_video_id, language_code=language_code)
- if transcript_present and force_update:
- create_or_update_video_transcript(
- edx_video_id,
- language_code,
- dict({'file_format': file_format}),
- ContentFile(transcript_content)
- )
- elif not transcript_present:
- create_video_transcript(
- edx_video_id,
- language_code,
- file_format,
- ContentFile(transcript_content)
- )
- else:
- LOGGER.info(
- u"[%s] [run=%s] [do-not-override-existing-transcript] [edx_video_id=%s] [language_code=%s]",
- MIGRATION_LOGS_PREFIX, command_run, edx_video_id, language_code
- )
-
-
-@chord_task(
- bind=True,
- base=LoggedPersistOnFailureTask,
- default_retry_delay=RETRY_DELAY_SECONDS,
- max_retries=2,
- time_limit=VIDEO_LEVEL_TIMEOUT_SECONDS,
- routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE
-)
-def async_migrate_transcript_subtask(self, *args, **kwargs): # pylint: disable=unused-argument
- """
- Migrates a transcript of a given video in a course as a new celery task.
- """
- success, failure = 'Success', 'Failure'
- video_location, revision, language_code, force_update = args
- command_run = kwargs['command_run']
- store = modulestore()
- video = store.get_item(usage_key=BlockUsageLocator.from_string(video_location), revision=revision)
- edx_video_id = clean_video_id(video.edx_video_id)
-
- if not kwargs['commit']:
- LOGGER.info(
- (u'[%s] [run=%s] [video-transcript-will-be-migrated] '
- u'[revision=%s] [video=%s] [edx_video_id=%s] [language_code=%s]'),
- MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
- )
- return success
-
- LOGGER.info(
- (u'[%s] [run=%s] [transcripts-migration-process-started-for-video-transcript] [revision=%s] '
- u'[video=%s] [edx_video_id=%s] [language_code=%s]'),
- MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
- )
-
- try:
- transcripts_info = video.get_transcripts_info()
- transcript_content, _, _ = get_transcript_from_contentstore(
- video=video,
- language=language_code,
- output_format=Transcript.SJSON,
- transcripts_info=transcripts_info,
- )
-
- is_video_valid = edx_video_id and is_video_available(edx_video_id)
- if not is_video_valid:
- edx_video_id = create_external_video('external-video')
- video.edx_video_id = edx_video_id
-
- # determine branch published/draft
- branch_setting = (
- ModuleStoreEnum.Branch.published_only
- if revision == ModuleStoreEnum.RevisionOption.published_only else
- ModuleStoreEnum.Branch.draft_preferred
- )
- with store.branch_setting(branch_setting):
- store.update_item(video, ModuleStoreEnum.UserID.mgmt_command)
-
- LOGGER.info(
- u'[%s] [run=%s] [generated-edx-video-id] [revision=%s] [video=%s] [edx_video_id=%s] [language_code=%s]',
- MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
- )
-
- save_transcript_to_storage(
- command_run=command_run,
- edx_video_id=edx_video_id,
- language_code=language_code,
- transcript_content=transcript_content,
- file_format=Transcript.SJSON,
- force_update=force_update,
- )
- except (NotFoundError, TranscriptsGenerationException, ValCannotCreateError):
- LOGGER.exception(
- (u'[%s] [run=%s] [video-transcript-migration-failed-with-known-exc] [revision=%s] [video=%s] '
- u'[edx_video_id=%s] [language_code=%s]'),
- MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
- )
- return failure
- except Exception:
- LOGGER.exception(
- (u'[%s] [run=%s] [video-transcript-migration-failed-with-unknown-exc] [revision=%s] '
- u'[video=%s] [edx_video_id=%s] [language_code=%s]'),
- MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
- )
- raise
-
- LOGGER.info(
- (u'[%s] [run=%s] [video-transcript-migration-succeeded-for-a-video] [revision=%s] '
- u'[video=%s] [edx_video_id=%s] [language_code=%s]'),
- MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
- )
- return success
def clone_instance(instance, field_values):
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index ffbd5e331b2..6f00cc2853c 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -55,7 +55,7 @@ defusedxml==0.5.0 # via -r requirements/edx/base.in, djangorestframework
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/github.in, django-statici18n
django-babel-underscore==0.5.2 # via -r requirements/edx/base.in
django-babel==0.6.2 # via django-babel-underscore
-git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/github.in, edx-celeryutils, super-csv
+git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/github.in
django-classy-tags==1.0.0 # via django-sekizai
django-config-models==2.0.0 # via -r requirements/edx/base.in, edx-enterprise
django-cors-headers==2.5.3 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.in
@@ -97,13 +97,13 @@ edx-analytics-data-api-client==0.15.3 # via -r requirements/edx/base.in
edx-api-doc-tools==1.0.2 # via -r requirements/edx/base.in
edx-bulk-grades==0.6.6 # via -r requirements/edx/base.in, staff-graded-xblock
edx-ccx-keys==1.0.0 # via -r requirements/edx/base.in
-edx-celeryutils==0.3.2 # via -r requirements/edx/base.in, super-csv
+edx-celeryutils==0.4.0 # via -r requirements/edx/base.in, super-csv
edx-completion==3.1.1 # via -r requirements/edx/base.in
edx-django-release-util==0.3.6 # via -r requirements/edx/base.in
edx-django-sites-extensions==2.4.3 # via -r requirements/edx/base.in
edx-django-utils==3.0 # via -r requirements/edx/base.in, django-config-models, edx-drf-extensions, edx-enterprise, edx-rest-api-client
edx-drf-extensions==3.0.0 # via -r requirements/edx/base.in, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
-edx-enterprise==2.5.0 # via -r requirements/edx/base.in
+edx-enterprise==2.5.1 # via -r requirements/edx/base.in
edx-i18n-tools==0.5.0 # via ora2
edx-milestones==0.2.6 # via -r requirements/edx/base.in
edx-opaque-keys[django]==2.0.1 # via -r requirements/edx/paver.txt, edx-bulk-grades, edx-ccx-keys, edx-completion, edx-drf-extensions, edx-enterprise, edx-milestones, edx-organizations, edx-proctoring, edx-user-state-client, edx-when, xmodule
@@ -168,7 +168,7 @@ numpy==1.18.1 # via calc, chem, scipy
git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 # via -r requirements/edx/github.in
oauthlib==2.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.in, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core
git+https://github.com/edx/edx-ora2.git@2.6.17#egg=ora2==2.6.17 # via -r requirements/edx/github.in
-packaging==20.1 # via drf-yasg
+packaging==20.3 # via drf-yasg
path.py==12.4.0 # via edx-enterprise, edx-i18n-tools, ora2, xmodule
path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.txt, path.py
pathtools==0.1.2 # via -r requirements/edx/paver.txt, watchdog
@@ -218,7 +218,7 @@ sailthru-client==2.2.3 # via -r requirements/edx/base.in, edx-ace
scipy==1.4.1 # via calc, chem
semantic-version==2.8.4 # via edx-drf-extensions
shapely==1.7.0 # via -r requirements/edx/base.in
-shortuuid==0.5.0 # via -r requirements/edx/base.in
+shortuuid==0.5.1 # via -r requirements/edx/base.in
simplejson==3.17.0 # via -r requirements/edx/base.in, sailthru-client, super-csv, xblock-utils
six==1.14.0 # via -r requirements/edx/../edx-sandbox/shared.txt, -r requirements/edx/base.in, -r requirements/edx/paver.txt, analytics-python, bleach, calc, cryptography, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, fs, fs-s3fs, help-tokens, html5lib, isodate, libsass, mock, nltk, packaging, paver, pycontracts, pyjwkest, python-dateutil, python-memcached, python-swiftclient, social-auth-app-django, social-auth-core, stevedore, xblock
slumber==0.7.1 # via edx-bulk-grades, edx-enterprise, edx-rest-api-client
@@ -229,7 +229,7 @@ soupsieve==2.0 # via beautifulsoup4
sqlparse==0.3.1 # via -r requirements/edx/base.in
staff-graded-xblock==0.7 # via -r requirements/edx/base.in
stevedore==1.32.0 # via -r requirements/edx/base.in, -r requirements/edx/paver.txt, code-annotations, edx-ace, edx-enterprise, edx-opaque-keys
-super-csv==0.9.6 # via -r requirements/edx/base.in, edx-bulk-grades
+super-csv==0.9.7 # via -r requirements/edx/base.in, edx-bulk-grades
sympy==1.5.1 # via symmath
testfixtures==6.14.0 # via edx-enterprise
text-unidecode==1.3 # via python-slugify
diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt
index e88ac08519c..d1201271744 100644
--- a/requirements/edx/development.txt
+++ b/requirements/edx/development.txt
@@ -66,7 +66,7 @@ distlib==0.3.0 # via -r requirements/edx/testing.txt, virtualenv
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/testing.txt, django-statici18n
django-babel-underscore==0.5.2 # via -r requirements/edx/testing.txt
django-babel==0.6.2 # via -r requirements/edx/testing.txt, django-babel-underscore
-git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/testing.txt, edx-celeryutils, super-csv
+git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/testing.txt
django-classy-tags==1.0.0 # via -r requirements/edx/testing.txt, django-sekizai
django-config-models==2.0.0 # via -r requirements/edx/testing.txt, edx-enterprise
django-cors-headers==2.5.3 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
@@ -109,13 +109,13 @@ edx-analytics-data-api-client==0.15.3 # via -r requirements/edx/testing.txt
edx-api-doc-tools==1.0.2 # via -r requirements/edx/testing.txt
edx-bulk-grades==0.6.6 # via -r requirements/edx/testing.txt, staff-graded-xblock
edx-ccx-keys==1.0.0 # via -r requirements/edx/testing.txt
-edx-celeryutils==0.3.2 # via -r requirements/edx/testing.txt, super-csv
+edx-celeryutils==0.4.0 # via -r requirements/edx/testing.txt, super-csv
edx-completion==3.1.1 # via -r requirements/edx/testing.txt
edx-django-release-util==0.3.6 # via -r requirements/edx/testing.txt
edx-django-sites-extensions==2.4.3 # via -r requirements/edx/testing.txt
edx-django-utils==3.0 # via -r requirements/edx/testing.txt, django-config-models, edx-drf-extensions, edx-enterprise, edx-rest-api-client
edx-drf-extensions==3.0.0 # via -r requirements/edx/testing.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
-edx-enterprise==2.5.0 # via -r requirements/edx/testing.txt
+edx-enterprise==2.5.1 # via -r requirements/edx/testing.txt
edx-i18n-tools==0.5.0 # via -r requirements/edx/testing.txt, ora2
edx-lint==1.4.1 # via -r requirements/edx/testing.txt
edx-milestones==0.2.6 # via -r requirements/edx/testing.txt
@@ -202,7 +202,7 @@ numpy==1.18.1 # via -r requirements/edx/testing.txt, calc, chem, pan
git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 # via -r requirements/edx/testing.txt
oauthlib==2.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core
git+https://github.com/edx/edx-ora2.git@2.6.17#egg=ora2==2.6.17 # via -r requirements/edx/testing.txt
-packaging==20.1 # via -r requirements/edx/testing.txt, drf-yasg, pytest, sphinx, tox
+packaging==20.3 # via -r requirements/edx/testing.txt, drf-yasg, pytest, sphinx, tox
pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
path.py==12.4.0 # via -r requirements/edx/testing.txt, edx-enterprise, edx-i18n-tools, ora2, xmodule
path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, path.py
@@ -277,7 +277,7 @@ scipy==1.4.1 # via -r requirements/edx/testing.txt, calc, chem
selenium==3.141.0 # via -r requirements/edx/testing.txt, bok-choy
semantic-version==2.8.4 # via -r requirements/edx/testing.txt, edx-drf-extensions
shapely==1.7.0 # via -r requirements/edx/testing.txt
-shortuuid==0.5.0 # via -r requirements/edx/testing.txt
+shortuuid==0.5.1 # via -r requirements/edx/testing.txt
simplejson==3.17.0 # via -r requirements/edx/testing.txt, sailthru-client, super-csv, xblock-utils
singledispatch==3.4.0.3 # via -r requirements/edx/testing.txt
six==1.14.0 # via -r requirements/edx/pip-tools.txt, -r requirements/edx/testing.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, edx-sphinx-theme, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, jsonschema, libsass, mando, mock, nltk, packaging, pathlib2, paver, pip-tools, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, sphinxcontrib-httpdomain, stevedore, tox, transifex-client, virtualenv, xblock
@@ -287,7 +287,7 @@ social-auth-core==3.2.0 # via -r requirements/edx/testing.txt, social-auth-app
git+https://github.com/jazzband/sorl-thumbnail.git@13bedfb7d2970809eda597e3ef79318a6fa80ac2#egg=sorl-thumbnail # via -r requirements/edx/testing.txt
sortedcontainers==2.1.0 # via -r requirements/edx/testing.txt, pdfminer.six
soupsieve==2.0 # via -r requirements/edx/testing.txt, beautifulsoup4
-sphinx==2.4.3 # via edx-sphinx-theme, sphinxcontrib-httpdomain
+sphinx==2.4.4 # via edx-sphinx-theme, sphinxcontrib-httpdomain
sphinxcontrib-applehelp==1.0.2 # via sphinx
sphinxcontrib-devhelp==1.0.2 # via sphinx
sphinxcontrib-htmlhelp==1.0.3 # via sphinx
@@ -299,7 +299,7 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx
sqlparse==0.3.1 # via -r requirements/edx/testing.txt, django-debug-toolbar
staff-graded-xblock==0.7 # via -r requirements/edx/testing.txt
stevedore==1.32.0 # via -r requirements/edx/testing.txt, code-annotations, edx-ace, edx-enterprise, edx-opaque-keys
-super-csv==0.9.6 # via -r requirements/edx/testing.txt, edx-bulk-grades
+super-csv==0.9.7 # via -r requirements/edx/testing.txt, edx-bulk-grades
sympy==1.5.1 # via -r requirements/edx/testing.txt, symmath
testfixtures==6.14.0 # via -r requirements/edx/testing.txt, edx-enterprise
text-unidecode==1.3 # via -r requirements/edx/testing.txt, faker, python-slugify
diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt
index fa45677e494..60b4ff34998 100644
--- a/requirements/edx/testing.txt
+++ b/requirements/edx/testing.txt
@@ -65,7 +65,7 @@ distlib==0.3.0 # via virtualenv
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/base.txt, django-statici18n
django-babel-underscore==0.5.2 # via -r requirements/edx/base.txt
django-babel==0.6.2 # via -r requirements/edx/base.txt, django-babel-underscore
-git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/base.txt, edx-celeryutils, super-csv
+git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/base.txt
django-classy-tags==1.0.0 # via -r requirements/edx/base.txt, django-sekizai
django-config-models==2.0.0 # via -r requirements/edx/base.txt, edx-enterprise
django-cors-headers==2.5.3 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt
@@ -105,13 +105,13 @@ edx-analytics-data-api-client==0.15.3 # via -r requirements/edx/base.txt
edx-api-doc-tools==1.0.2 # via -r requirements/edx/base.txt
edx-bulk-grades==0.6.6 # via -r requirements/edx/base.txt, staff-graded-xblock
edx-ccx-keys==1.0.0 # via -r requirements/edx/base.txt
-edx-celeryutils==0.3.2 # via -r requirements/edx/base.txt, super-csv
+edx-celeryutils==0.4.0 # via -r requirements/edx/base.txt, super-csv
edx-completion==3.1.1 # via -r requirements/edx/base.txt
edx-django-release-util==0.3.6 # via -r requirements/edx/base.txt
edx-django-sites-extensions==2.4.3 # via -r requirements/edx/base.txt
edx-django-utils==3.0 # via -r requirements/edx/base.txt, django-config-models, edx-drf-extensions, edx-enterprise, edx-rest-api-client
edx-drf-extensions==3.0.0 # via -r requirements/edx/base.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
-edx-enterprise==2.5.0 # via -r requirements/edx/base.txt
+edx-enterprise==2.5.1 # via -r requirements/edx/base.txt
edx-i18n-tools==0.5.0 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, ora2
edx-lint==1.4.1 # via -r requirements/edx/testing.in
edx-milestones==0.2.6 # via -r requirements/edx/base.txt
@@ -193,7 +193,7 @@ numpy==1.18.1 # via -r requirements/edx/base.txt, -r requirements/ed
git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 # via -r requirements/edx/base.txt
oauthlib==2.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core
git+https://github.com/edx/edx-ora2.git@2.6.17#egg=ora2==2.6.17 # via -r requirements/edx/base.txt
-packaging==20.1 # via -r requirements/edx/base.txt, drf-yasg, pytest, tox
+packaging==20.3 # via -r requirements/edx/base.txt, drf-yasg, pytest, tox
pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/coverage.txt
path.py==12.4.0 # via -r requirements/edx/base.txt, edx-enterprise, edx-i18n-tools, ora2, xmodule
path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, path.py
@@ -265,7 +265,7 @@ scipy==1.4.1 # via -r requirements/edx/base.txt, calc, chem
selenium==3.141.0 # via -r requirements/edx/testing.in, bok-choy
semantic-version==2.8.4 # via -r requirements/edx/base.txt, edx-drf-extensions
shapely==1.7.0 # via -r requirements/edx/base.txt
-shortuuid==0.5.0 # via -r requirements/edx/base.txt
+shortuuid==0.5.1 # via -r requirements/edx/base.txt
simplejson==3.17.0 # via -r requirements/edx/base.txt, sailthru-client, super-csv, xblock-utils
singledispatch==3.4.0.3 # via -r requirements/edx/testing.in
six==1.14.0 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, libsass, mando, mock, nltk, packaging, pathlib2, paver, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, stevedore, tox, transifex-client, virtualenv, xblock
@@ -277,7 +277,7 @@ soupsieve==2.0 # via -r requirements/edx/base.txt, beautifulsoup4
sqlparse==0.3.1 # via -r requirements/edx/base.txt
staff-graded-xblock==0.7 # via -r requirements/edx/base.txt
stevedore==1.32.0 # via -r requirements/edx/base.txt, code-annotations, edx-ace, edx-enterprise, edx-opaque-keys
-super-csv==0.9.6 # via -r requirements/edx/base.txt, edx-bulk-grades
+super-csv==0.9.7 # via -r requirements/edx/base.txt, edx-bulk-grades
sympy==1.5.1 # via -r requirements/edx/base.txt, symmath
testfixtures==6.14.0 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, edx-enterprise
text-unidecode==1.3 # via -r requirements/edx/base.txt, faker, python-slugify
--
GitLab