[BD-29] [TNL-7264] Add Milestones, Content Gating, and Special Exams Outline Processors (#24545)

Extend the learning_sequences Course Outline API to handle milestones,
content gating, and special exams. This includes things like entrance exams
that would block other content from being accessible, and proctored exams
which would be unavailable until an exam is started.

Co-authored-by: Agrendalath <piotr@surowiec.it>

openedx/core/djangoapps/content/learning_sequences/api/outlines.py

@@ -16,15 +16,28 @@ from edx_django_utils.monitoring import function_trace
 from opaque_keys.edx.keys import CourseKey, UsageKey
 
 from ..data import (
-    CourseOutlineData, CourseSectionData, CourseLearningSequenceData,
-    UserCourseOutlineData, UserCourseOutlineDetailsData, VisibilityData,
-    CourseVisibility
+    CourseLearningSequenceData,
+    CourseOutlineData,
+    CourseSectionData,
+    CourseVisibility,
+    ExamData,
+    UserCourseOutlineData,
+    UserCourseOutlineDetailsData,
+    VisibilityData,
 )
 from ..models import (
-    CourseSection, CourseSectionSequence, CourseContext, LearningContext, LearningSequence
+    CourseSection,
+    CourseSectionSequence,
+    CourseContext,
+    CourseSequenceExam,
+    LearningContext,
+    LearningSequence
 )
 from .permissions import can_see_all_content
+from .processors.content_gating import ContentGatingOutlineProcessor
+from .processors.milestones import MilestonesOutlineProcessor
 from .processors.schedule import ScheduleOutlineProcessor
+from .processors.special_exams import SpecialExamsOutlineProcessor
 from .processors.visibility import VisibilityOutlineProcessor
 from .processors.enrollment import EnrollmentOutlineProcessor
 
@@ -67,13 +80,23 @@ def get_course_outline(course_key: CourseKey) -> CourseOutlineData:
     section_sequence_models = CourseSectionSequence.objects \
         .filter(course_context=course_context) \
         .order_by('ordering') \
-        .select_related('sequence')
+        .select_related('sequence', 'exam')
 
     # Build mapping of section.id keys to sequence lists.
     sec_ids_to_sequence_list = defaultdict(list)
 
     for sec_seq_model in section_sequence_models:
         sequence_model = sec_seq_model.sequence
+
+        try:
+            exam_data = ExamData(
+                is_practice_exam=sec_seq_model.exam.is_practice_exam,
+                is_proctored_enabled=sec_seq_model.exam.is_proctored_enabled,
+                is_time_limited=sec_seq_model.exam.is_time_limited
+            )
+        except CourseSequenceExam.DoesNotExist:
+            exam_data = ExamData()
+
         sequence_data = CourseLearningSequenceData(
             usage_key=sequence_model.usage_key,
             title=sequence_model.title,
@@ -81,7 +104,8 @@ def get_course_outline(course_key: CourseKey) -> CourseOutlineData:
             visibility=VisibilityData(
                 hide_from_toc=sec_seq_model.hide_from_toc,
                 visible_to_staff_only=sec_seq_model.visible_to_staff_only,
-            )
+            ),
+            exam=exam_data
         )
         sec_ids_to_sequence_list[sec_seq_model.section_id].append(sequence_data)
 
@@ -104,6 +128,7 @@ def get_course_outline(course_key: CourseKey) -> CourseOutlineData:
         published_at=course_context.learning_context.published_at,
         published_version=course_context.learning_context.published_version,
         days_early_for_beta=course_context.days_early_for_beta,
+        entrance_exam_id=course_context.entrance_exam_id,
         sections=sections_data,
         self_paced=course_context.self_paced,
         course_visibility=CourseVisibility(course_context.course_visibility),
@@ -163,10 +188,12 @@ def get_user_course_outline_details(course_key: CourseKey,
         course_key, user, at_time
     )
     schedule_processor = processors['schedule']
+    special_exams_processor = processors['special_exams']
 
     return UserCourseOutlineDetailsData(
         outline=user_course_outline,
-        schedule=schedule_processor.schedule_data(user_course_outline)
+        schedule=schedule_processor.schedule_data(user_course_outline),
+        special_exam_attempts=special_exams_processor.exam_data(user_course_outline)
     )
 
 
@@ -181,12 +208,13 @@ def _get_user_course_outline_and_processors(course_key: CourseKey,
     # released. These do not need to be run for staff users. This is where we
     # would add in pluggability for OutlineProcessors down the road.
     processor_classes = [
+        ('content_gating', ContentGatingOutlineProcessor),
+        ('milestones', MilestonesOutlineProcessor),
         ('schedule', ScheduleOutlineProcessor),
+        ('special_exams', SpecialExamsOutlineProcessor),
         ('visibility', VisibilityOutlineProcessor),
         ('enrollment', EnrollmentOutlineProcessor),
         # Future:
-        # ('content_gating', ContentGatingOutlineProcessor),
-        # ('milestones', MilestonesOutlineProcessor),
         # ('user_partitions', UserPartitionsOutlineProcessor),
     ]
 
@@ -225,6 +253,7 @@ def _get_user_course_outline_and_processors(course_key: CourseKey,
                 'title',
                 'published_at',
                 'published_version',
+                'entrance_exam_id',
                 'sections',
                 'self_paced',
                 'course_visibility',
@@ -279,6 +308,7 @@ def _update_course_context(course_outline: CourseOutlineData):
             'course_visibility': course_outline.course_visibility.value,
             'days_early_for_beta': course_outline.days_early_for_beta,
             'self_paced': course_outline.self_paced,
+            'entrance_exam_id': course_outline.entrance_exam_id,
         }
     )
     if created:
@@ -349,7 +379,7 @@ def _update_course_section_sequences(course_outline: CourseOutlineData, course_c
     ordering = 0
     for section_data in course_outline.sections:
         for sequence_data in section_data.sequences:
-            CourseSectionSequence.objects.update_or_create(
+            course_section_sequence, _ = CourseSectionSequence.objects.update_or_create(
                 course_context=course_context,
                 section=section_models[section_data.usage_key],
                 sequence=sequence_models[sequence_data.usage_key],
@@ -361,3 +391,17 @@ def _update_course_section_sequences(course_outline: CourseOutlineData, course_c
                 },
             )
             ordering += 1
+
+            # If a sequence is an exam, update or create an exam record
+            if bool(sequence_data.exam):
+                CourseSequenceExam.objects.update_or_create(
+                    course_section_sequence=course_section_sequence,
+                    defaults={
+                        'is_practice_exam': sequence_data.exam.is_practice_exam,
+                        'is_proctored_enabled': sequence_data.exam.is_proctored_enabled,
+                        'is_time_limited': sequence_data.exam.is_time_limited,
+                    },
+                )
+            else:
+                # Otherwise, delete any exams associated with it
+                CourseSequenceExam.objects.filter(course_section_sequence=course_section_sequence).delete()

openedx/core/djangoapps/content/learning_sequences/api/processors/content_gating.py

+import logging
+from datetime import datetime
+
+from django.contrib.auth import get_user_model
+from opaque_keys.edx.keys import CourseKey
+from student.models import EntranceExamConfiguration
+from util import milestones_helpers
+
+from .base import OutlineProcessor
+
+User = get_user_model()
+log = logging.getLogger(__name__)
+
+
+class ContentGatingOutlineProcessor(OutlineProcessor):
+    """
+    Responsible for applying all content gating outline processing.
+
+    This includes:
+    - Entrance Exams
+    - Chapter gated content
+    """
+    def __init__(self, course_key: CourseKey, user: User, at_time: datetime):
+        super().__init__(course_key, user, at_time)
+        self.required_content = None
+        self.can_skip_entrance_exam = False
+
+    def load_data(self):
+        """
+        Get the required content for the course, and whether
+        or not the user can skip the entrance exam.
+        """
+        self.required_content = milestones_helpers.get_required_content(self.course_key, self.user)
+
+        if self.user.is_authenticated:
+            self.can_skip_entrance_exam = EntranceExamConfiguration.user_can_skip_entrance_exam(
+                self.user, self.course_key
+            )
+
+    def inaccessible_sequences(self, full_course_outline):
+        """
+        Mark any section that is gated by required content as inaccessible
+        """
+        if full_course_outline.entrance_exam_id and self.can_skip_entrance_exam:
+            self.required_content = [
+                content
+                for content in self.required_content
+                if not content == full_course_outline.entrance_exam_id
+            ]
+
+        inaccessible = set()
+        for section in full_course_outline.sections:
+            if self.gated_by_required_content(section.usage_key):
+                inaccessible |= {
+                    seq.usage_key
+                    for seq in section.sequences
+                }
+
+        return inaccessible
+
+    def gated_by_required_content(self, section_usage_key):
+        """
+        Returns True if the current section associated with the usage_key should be gated by the given required_content.
+        Returns False otherwise.
+        """
+        if not self.required_content:
+            return False
+
+        # This should always be a chapter block
+        assert section_usage_key.block_type == 'chapter'
+        if str(section_usage_key) not in self.required_content:
+            return True
+
+        return False

openedx/core/djangoapps/content/learning_sequences/api/processors/milestones.py

+import logging
+
+from django.contrib.auth import get_user_model
+from opaque_keys.edx.keys import CourseKey
+from util import milestones_helpers
+
+from .base import OutlineProcessor
+
+User = get_user_model()
+log = logging.getLogger(__name__)
+
+
+class MilestonesOutlineProcessor(OutlineProcessor):
+    """
+    Responsible for applying all general course milestones outline processing.
+
+    This does not include Entrance Exams (see `ContentGatingOutlineProcessor`),
+    or Special Exams (see `SpecialExamsOutlineProcessor`)
+    """
+    def inaccessible_sequences(self, full_course_outline):
+        """
+        Returns the set of sequence usage keys for which the
+        user has pending milestones
+        """
+        inaccessible = set()
+        for section in full_course_outline.sections:
+            inaccessible |= {
+                seq.usage_key
+                for seq in section.sequences
+                if self.has_pending_milestones(seq.usage_key)
+            }
+
+        return inaccessible
+
+    def has_pending_milestones(self, usage_key):
+        return bool(milestones_helpers.get_course_content_milestones(
+            str(self.course_key),
+            str(usage_key),
+            'requires',
+            self.user.id
+        ))

openedx/core/djangoapps/content/learning_sequences/api/processors/special_exams.py

+"""
+As currently designed, this processor ignores the course specific
+`Enable Timed Exams` setting when determining whether or not it should
+remove keys and/or supplement exam data. This matches the exact behavior
+of `MilestonesAndSpecialExamsTransformer`. It is not entirely clear if
+the behavior should be modified, so it has been decided to consider any
+necessary fixes in a new ticket.
+
+Please see the PR and discussion linked below for further context
+https://github.com/edx/edx-platform/pull/24545#discussion_r501738511
+"""
+
+import logging
+
+from edx_proctoring.api import get_attempt_status_summary
+from edx_proctoring.exceptions import ProctoredExamNotFoundException
+from django.conf import settings
+from django.contrib.auth import get_user_model
+
+from ...data import SpecialExamAttemptData, UserCourseOutlineData
+from .base import OutlineProcessor
+
+User = get_user_model()
+log = logging.getLogger(__name__)
+
+
+class SpecialExamsOutlineProcessor(OutlineProcessor):
+    """
+    Responsible for applying all outline processing related to special exams.
+    """
+    def load_data(self):
+        """
+        Check if special exams are enabled
+        """
+        self.special_exams_enabled = settings.FEATURES.get('ENABLE_SPECIAL_EXAMS', False)
+
+    def exam_data(self, pruned_course_outline: UserCourseOutlineData) -> SpecialExamAttemptData:
+        """
+        Return supplementary special exam information for this outline.
+
+        Be careful to pass in a UserCourseOutlineData - i.e. an outline that has
+        already been pruned to what a user is allowed to see. That way, we can
+        use this to make sure that we're not returning data about
+        LearningSequences that the user can't see because it was hidden by a
+        different OutlineProcessor.
+        """
+        sequences = {}
+        if self.special_exams_enabled:
+            for section in pruned_course_outline.sections:
+                for sequence in section.sequences:
+                    # Don't bother checking for information
+                    # on non-exam sequences
+                    if not bool(sequence.exam):
+                        continue
+
+                    special_exam_attempt_context = None
+                    try:
+                        # Calls into edx_proctoring subsystem to get relevant special exam information.
+                        # This will return None, if (user, course_id, content_id) is not applicable.
+                        special_exam_attempt_context = get_attempt_status_summary(
+                            self.user.id,
+                            str(self.course_key),
+                            str(sequence.usage_key)
+                        )
+                    except ProctoredExamNotFoundException:
+                        log.info(
+                            'No exam found for {sequence_key} in {course_key}'.format(
+                                sequence_key=sequence.usage_key,
+                                course_key=self.course_key
+                            )
+                        )
+
+                    if special_exam_attempt_context:
+                        # Return exactly the same format as the edx_proctoring API response
+                        sequences[sequence.usage_key] = special_exam_attempt_context
+
+        return SpecialExamAttemptData(
+            sequences=sequences,
+        )

openedx/core/djangoapps/content/learning_sequences/api/tests/test_data.py

@@ -32,6 +32,7 @@ class TestCourseOutlineData(TestCase):
             title="Exciting Test Course!",
             published_at=datetime(2020, 5, 19, tzinfo=timezone.utc),
             published_version="5ebece4b69dd593d82fe2014",
+            entrance_exam_id=None,
             days_early_for_beta=None,
             sections=generate_sections(cls.course_key, [3, 2]),
             self_paced=False,

openedx/core/djangoapps/content/learning_sequences/data.py

@@ -68,6 +68,19 @@ class VisibilityData:
     visible_to_staff_only = attr.ib(type=bool)
 
 
+@attr.s(frozen=True)
+class ExamData:
+    """
+    XBlock attributes that describe exams
+    """
+    is_practice_exam = attr.ib(type=bool, default=False)
+    is_proctored_enabled = attr.ib(type=bool, default=False)
+    is_time_limited = attr.ib(type=bool, default=False)
+
+    def __bool__(self):
+        return self.is_practice_exam or self.is_proctored_enabled or self.is_time_limited
+
+
 @attr.s(frozen=True)
 class CourseLearningSequenceData:
     """
@@ -82,6 +95,7 @@ class CourseLearningSequenceData:
     title = attr.ib(type=str)
     visibility = attr.ib(type=VisibilityData)
 
+    exam = attr.ib(type=ExamData, default=ExamData())
     inaccessible_after_due = attr.ib(type=bool, default=True)
 
 
@@ -147,6 +161,9 @@ class CourseOutlineData:
 
     course_visibility = attr.ib(validator=attr.validators.in_(CourseVisibility))
 
+    # Entrance Exam ID
+    entrance_exam_id = attr.ib(type=str)
+
     def __attrs_post_init__(self):
         """Post-init hook that validates and inits the `sequences` field."""
         sequences = {}
@@ -241,6 +258,14 @@ class ScheduleData:
     sequences = attr.ib(type=Dict[UsageKey, ScheduleItemData])
 
 
+@attr.s(frozen=True)
+class SpecialExamAttemptData:
+    """
+    Overall special exam attempt data.
+    """
+    sequences = attr.ib(type=Dict[UsageKey, Dict])
+
+
 @attr.s(frozen=True)
 class UserCourseOutlineData(CourseOutlineData):
     """
@@ -288,3 +313,4 @@ class UserCourseOutlineDetailsData:
     """
     outline = attr.ib(type=UserCourseOutlineData)
     schedule = attr.ib(type=ScheduleData)
+    special_exam_attempts = attr.ib(type=SpecialExamAttemptData)

openedx/core/djangoapps/content/learning_sequences/migrations/0006_coursecontext_entrance_exam_id.py

+# Generated by Django 2.2.14 on 2020-07-20 10:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('learning_sequences', '0005_coursecontext_days_early_for_beta'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='coursecontext',
+            name='entrance_exam_id',
+            field=models.CharField(max_length=255, null=True),
+        ),
+    ]

openedx/core/djangoapps/content/learning_sequences/migrations/0007_coursesequenceexam.py

+# Generated by Django 2.2.16 on 2020-09-30 07:14
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import model_utils.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('learning_sequences', '0006_coursecontext_entrance_exam_id'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CourseSequenceExam',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+                ('is_practice_exam', models.BooleanField(default=False)),
+                ('is_proctored_enabled', models.BooleanField(default=False)),
+                ('is_time_limited', models.BooleanField(default=False)),
+                ('course_section_sequence', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='exam', to='learning_sequences.CourseSectionSequence')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

openedx/core/djangoapps/content/learning_sequences/models.py

@@ -82,6 +82,7 @@ class CourseContext(TimeStampedModel):
     )
     days_early_for_beta = models.IntegerField(null=True, blank=True)
     self_paced = models.BooleanField(default=False)
+    entrance_exam_id = models.CharField(max_length=255, null=True)
 
 
 class LearningSequence(TimeStampedModel):
@@ -196,3 +197,15 @@ class CourseSectionSequence(CourseContentVisibilityMixin, TimeStampedModel):
         unique_together = [
             ['course_context', 'ordering'],
         ]
+
+
+class CourseSequenceExam(TimeStampedModel):
+    """
+    This model stores XBlock information that affects outline level information
+    pertaining to special exams
+    """
+    course_section_sequence = models.OneToOneField(CourseSectionSequence, on_delete=models.CASCADE, related_name='exam')
+
+    is_practice_exam = models.BooleanField(default=False)
+    is_proctored_enabled = models.BooleanField(default=False)
+    is_time_limited = models.BooleanField(default=False)

openedx/core/djangoapps/content/learning_sequences/tasks.py

@@ -10,7 +10,11 @@ from xmodule.modulestore.django import modulestore
 
 from .api import replace_course_outline
 from .data import (
-    CourseOutlineData, CourseSectionData, CourseLearningSequenceData, VisibilityData,
+    CourseOutlineData,
+    CourseSectionData,
+    CourseLearningSequenceData,
+    ExamData,
+    VisibilityData,
     CourseVisibility
 )
 
@@ -39,6 +43,11 @@ def get_outline_from_modulestore(course_key):
                     usage_key=sequence.location,
                     title=sequence.display_name,
                     inaccessible_after_due=sequence.hide_after_due,
+                    exam=ExamData(
+                        is_practice_exam=sequence.is_practice_exam,
+                        is_proctored_enabled=sequence.is_proctored_enabled,
+                        is_time_limited=sequence.is_timed_exam
+                    ),
                     visibility=VisibilityData(
                         hide_from_toc=sequence.hide_from_toc,
                         visible_to_staff_only=sequence.visible_to_staff_only
@@ -71,6 +80,7 @@ def get_outline_from_modulestore(course_key):
             title=course.display_name,
             published_at=course.subtree_edited_on,
             published_version=str(course.course_version),  # .course_version is a BSON obj
+            entrance_exam_id=course.entrance_exam_id,
             days_early_for_beta=course.days_early_for_beta,
             sections=sections_data,
             self_paced=course.self_paced,

openedx/core/djangoapps/content/learning_sequences/tests/test_views.py

@@ -44,6 +44,7 @@ class CourseOutlineViewTest(CacheIsolationTestCase, APITestCase):
             title="Views Test Course!",
             published_at=datetime(2020, 5, 20, tzinfo=timezone.utc),
             published_version="5ebece4b69dd593d82fe2020",
+            entrance_exam_id=None,
             days_early_for_beta=None,
             sections=generate_sections(cls.course_key, [2, 2]),
             self_paced=False,

openedx/core/djangoapps/content/learning_sequences/views.py

@@ -6,6 +6,7 @@ from datetime import datetime, timezone
 import json
 import logging
 
+from django.conf import settings
 from django.contrib.auth import get_user_model
 from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
 from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser
@@ -62,6 +63,7 @@ class CourseOutlineView(APIView):
             """
             user_course_outline = user_course_outline_details.outline
             schedule = user_course_outline_details.schedule
+            exam_information = user_course_outline_details.special_exam_attempts
             return {
                 # Top level course information
                 "course_key": str(user_course_outline.course_key),
@@ -70,6 +72,7 @@ class CourseOutlineView(APIView):
                 "title": user_course_outline.title,
                 "published_at": user_course_outline.published_at,
                 "published_version": user_course_outline.published_version,
+                "entrance_exam_id": user_course_outline.entrance_exam_id,
                 "days_early_for_beta": user_course_outline.days_early_for_beta,
                 "self_paced": user_course_outline.self_paced,
 
@@ -89,6 +92,7 @@ class CourseOutlineView(APIView):
                         str(seq_usage_key): self._sequence_repr(
                             sequence,
                             schedule.sequences.get(seq_usage_key),
+                            exam_information.sequences.get(seq_usage_key, {}),
                             user_course_outline.accessible_sequences,
                         )
                         for seq_usage_key, sequence in user_course_outline.sequences.items()
@@ -96,7 +100,7 @@ class CourseOutlineView(APIView):
                 },
             }
 
-        def _sequence_repr(self, sequence, sequence_schedule, accessible_sequences):
+        def _sequence_repr(self, sequence, sequence_schedule, sequence_exam, accessible_sequences):
             """Representation of a Sequence."""
             if sequence_schedule is None:
                 schedule_item_dict = {'start': None, 'effective_start': None, 'due': None}
@@ -108,7 +112,7 @@ class CourseOutlineView(APIView):
                     'due': sequence_schedule.due,
                 }
 
-            return {
+            sequence_representation = {
                 "id": str(sequence.usage_key),
                 "title": sequence.title,
                 "accessible": sequence.usage_key in accessible_sequences,
@@ -116,6 +120,12 @@ class CourseOutlineView(APIView):
                 **schedule_item_dict,
             }
 
+            # Only include this data if special exams are on
+            if settings.FEATURES.get('ENABLE_SPECIAL_EXAMS', False):
+                sequence_representation["exam"] = sequence_exam
+
+            return sequence_representation
+
         def _section_repr(self, section, section_schedule):
             """Representation of a Section."""
             if section_schedule is None: