Merge pull request #18014 from edx/bmedx/retirement_state_mgmt

Add a management command and settings to populate RetirementState models 

cms/envs/aws.py

@@ -584,6 +584,7 @@ RETIREMENT_SERVICE_WORKER_USERNAME = ENV_TOKENS.get(
     'RETIREMENT_SERVICE_WORKER_USERNAME',
     RETIREMENT_SERVICE_WORKER_USERNAME
 )
+RETIREMENT_STATES = ENV_TOKENS.get('RETIREMENT_STATES', RETIREMENT_STATES)
 
 ####################### Plugin Settings ##########################
 

cms/envs/common.py

@@ -134,6 +134,7 @@ from lms.envs.common import (
     RETIRED_EMAIL_FMT,
     RETIRED_USER_SALTS,
     RETIREMENT_SERVICE_WORKER_USERNAME,
+    RETIREMENT_STATES,
 
     # Methods to derive settings
     _make_mako_template_dirs,

lms/envs/aws.py

@@ -1088,6 +1088,7 @@ RETIREMENT_SERVICE_WORKER_USERNAME = ENV_TOKENS.get(
     'RETIREMENT_SERVICE_WORKER_USERNAME',
     RETIREMENT_SERVICE_WORKER_USERNAME
 )
+RETIREMENT_STATES = ENV_TOKENS.get('RETIREMENT_STATES', RETIREMENT_STATES)
 
 ############################### Plugin Settings ###############################
 

lms/envs/common.py

@@ -3414,6 +3414,42 @@ derived('RETIRED_USERNAME_FMT', 'RETIRED_EMAIL_FMT')
 RETIRED_USER_SALTS = ['abc', '123']
 RETIREMENT_SERVICE_WORKER_USERNAME = 'RETIREMENT_SERVICE_USER'
 
+# These states are the default, but are designed to be overridden in configuration.
+RETIREMENT_STATES = [
+    'PENDING',
+
+    'LOCKING_ACCOUNT',
+    'LOCKING_COMPLETE',
+
+    'RETIRING_CREDENTIALS',
+    'CREDENTIALS_COMPLETE',
+
+    'RETIRING_ECOM',
+    'ECOM_COMPLETE',
+
+    'RETIRING_FORUMS',
+    'FORUMS_COMPLETE',
+
+    'RETIRING_EMAIL_LISTS',
+    'EMAIL_LISTS_COMPLETE',
+
+    'RETIRING_ENROLLMENTS',
+    'ENROLLMENTS_COMPLETE',
+
+    'RETIRING_NOTES',
+    'NOTES_COMPLETE',
+
+    'NOTIFYING_PARTNERS',
+    'PARTNERS_NOTIFIED',
+
+    'RETIRING_LMS',
+    'LMS_COMPLETE',
+
+    'ERRORED',
+    'ABORTED',
+    'COMPLETE',
+]
+
 ############### Settings for django-fernet-fields ##################
 FERNET_KEYS = [
     'DUMMY KEY CHANGE BEFORE GOING TO PRODUCTION',

openedx/core/djangoapps/user_api/management/commands/populate_retirement_states.py

+"""
+Take the list of states from settings.RETIREMENT_STATES and forces the
+RetirementState table to mirror it.
+
+We use a foreign keyed table for this instead of just using the settings
+directly to generate a `choices` tuple for the model because the states
+need to be configurable by open source partners and modifying the
+`choices` for a model field causes new migrations to be generated,
+with a variety of unpleasant follow-on effects for the partner when
+upgrading the model at a later date.
+"""
+from __future__ import print_function
+
+import copy
+import logging
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from openedx.core.djangoapps.user_api.models import RetirementState, UserRetirementStatus
+
+
+LOGGER = logging.getLogger(__name__)
+
+START_STATE = 'PENDING'
+END_STATES = ['ERRORED', 'ABORTED', 'COMPLETE']
+REQUIRED_STATES = copy.deepcopy(END_STATES)
+REQUIRED_STATES.insert(0, START_STATE)
+REQ_STR = ','.join(REQUIRED_STATES)
+
+
+class Command(BaseCommand):
+    """
+    Implementation of the populate command
+    """
+    help = 'Populates the RetirementState table with the states present in settings.'
+
+    def _validate_new_states(self, new_states):
+        """
+        Check settings for existence of states, required states
+        """
+        if not new_states:
+            raise CommandError('settings.RETIREMENT_STATES does not exist or is empty.')
+
+        if not set(REQUIRED_STATES).issubset(set(new_states)):
+            raise CommandError('settings.RETIREMENT_STATES ({}) does not contain all required states '
+                               '({})'.format(new_states, REQ_STR))
+
+        # Confirm that the start and end states are in the right places
+        if new_states.index(START_STATE) != 0:
+            raise CommandError('{} must be the first state'.format(START_STATE))
+
+        num_end_states = len(END_STATES)
+
+        if new_states[-num_end_states:] != END_STATES:
+            raise CommandError('The last {} states must be these (in this order): '
+                               '{}'.format(num_end_states, END_STATES))
+
+    def _check_current_users(self):
+        """
+        Check UserRetirementStatus for users currently in progress
+        """
+        if UserRetirementStatus.objects.exclude(current_state__state_name__in=REQUIRED_STATES).exists():
+            raise CommandError(
+                'Users are currently being processed. All users must be in one of these states to run this command: '
+                '{}'.format(REQ_STR)
+            )
+
+    def _delete_old_states_and_create_new(self, new_states):
+        """
+        Wipes the RetirementState table and creates new entries based on new_states
+        - Note that the state_execution_order is incremented by 10 for each entry
+          this should allow manual insert of "in between" states via the Django admin
+          if necessary, without having to manually re-sort all of the states.
+        """
+
+        # Save off old states before
+        current_states = RetirementState.objects.all().values_list('state_name', flat=True)
+
+        # Delete all existing rows, easier than messing with the ordering
+        RetirementState.objects.all().delete()
+
+        # Add new rows, with space in between to manually insert stages via Django admin if necessary
+        curr_sort_order = 1
+        for state in new_states:
+            row = {
+                'state_name': state,
+                'state_execution_order': curr_sort_order,
+                'is_dead_end_state': state in END_STATES,
+                'required': state in REQUIRED_STATES
+            }
+
+            RetirementState.objects.create(**row)
+            curr_sort_order += 10
+
+        # Generate the diff
+        set_current_states = set(current_states)
+        set_new_states = set(new_states)
+
+        states_to_create = set_new_states - set_current_states
+        states_remaining = set_current_states.intersection(set_new_states)
+        states_to_delete = set_current_states - set_new_states
+
+        return states_to_create, states_remaining, states_to_delete
+
+    def handle(self, *args, **options):
+        """
+        Execute the command.
+        """
+        new_states = settings.RETIREMENT_STATES
+        self._validate_new_states(new_states)
+        self._check_current_users()
+        created, existed, deleted = self._delete_old_states_and_create_new(new_states)
+
+        # Report
+        print("All states removed and new states added. Differences:")
+        print("   Added: {}".format(created))
+        print("   Removed: {}".format(deleted))
+        print("   Remaining: {}".format(existed))
+        print("States updated successfully. Current states:")
+
+        for state in RetirementState.objects.all():
+            print(state)

openedx/core/djangoapps/user_api/management/tests/test_populate_retirement_states.py

+"""
+Test the populate_retirement_states management command
+"""
+import copy
+import pytest
+
+from django.core.management import call_command, CommandError
+
+from openedx.core.djangoapps.user_api.models import RetirementState, UserRetirementStatus
+from openedx.core.djangoapps.user_api.management.commands.populate_retirement_states import START_STATE
+from student.tests.factories import UserFactory
+
+pytestmark = pytest.mark.django_db
+
+
+def test_successful_create(settings):
+    """
+    Run the command with default states for a successful initial population
+    """
+    call_command('populate_retirement_states')
+    curr_states = RetirementState.objects.all().values_list('state_name', flat=True)
+    assert list(curr_states) == settings.RETIREMENT_STATES
+
+
+def test_successful_update(settings):
+    """
+    Run the command with expected inputs for a successful update
+    """
+    settings.RETIREMENT_STATES = copy.deepcopy(settings.RETIREMENT_STATES)
+    settings.RETIREMENT_STATES.insert(3, 'FOO_START')
+    settings.RETIREMENT_STATES.insert(4, 'FOO_COMPLETE')
+
+    call_command('populate_retirement_states')
+    curr_states = RetirementState.objects.all().values_list('state_name', flat=True)
+    assert list(curr_states) == settings.RETIREMENT_STATES
+
+
+def test_no_states(settings):
+    """
+    Test with empty settings.RETIREMENT_STATES
+    """
+    settings.RETIREMENT_STATES = None
+    with pytest.raises(CommandError, match=r'settings.RETIREMENT_STATES does not exist or is empty.'):
+        call_command('populate_retirement_states')
+
+    settings.RETIREMENT_STATES = []
+    with pytest.raises(CommandError, match=r'settings.RETIREMENT_STATES does not exist or is empty.'):
+        call_command('populate_retirement_states')
+
+
+def test_missing_required_states_start(settings):
+    """
+    Test with missing PENDING
+    """
+    # This is used throughout this file to force pytest to actually revert our settings changes.
+    # Since we're modifying the list and not directly modifying the settings it doesn't get picked
+    # up here:
+    # https://github.com/pytest-dev/pytest-django/blob/master/pytest_django/fixtures.py#L254
+    settings.RETIREMENT_STATES = copy.deepcopy(settings.RETIREMENT_STATES)
+
+    # Remove "PENDING" state
+    del settings.RETIREMENT_STATES[0]
+
+    with pytest.raises(CommandError, match=r'does not contain all required states'):
+        call_command('populate_retirement_states')
+
+
+def test_missing_required_states_end(settings):
+    """
+    Test with missing required end states
+    """
+    # Remove last state, a required dead end state
+    settings.RETIREMENT_STATES = copy.deepcopy(settings.RETIREMENT_STATES)
+    del settings.RETIREMENT_STATES[-1]
+
+    with pytest.raises(CommandError, match=r'does not contain all required states'):
+        call_command('populate_retirement_states')
+
+
+def test_out_of_order_start_state(settings):
+    """
+    Test with PENDING somewhere other than the beginning
+    """
+    settings.RETIREMENT_STATES = copy.deepcopy(settings.RETIREMENT_STATES)
+    del settings.RETIREMENT_STATES[0]
+    settings.RETIREMENT_STATES.insert(4, 'PENDING')
+
+    with pytest.raises(CommandError, match=r'{} must be the first state'.format(START_STATE)):
+        call_command('populate_retirement_states')
+
+
+def test_out_of_order_end_states(settings):
+    """
+    Test with missing PENDING and/or end states
+    """
+    # Remove last state, a required dead end state
+    settings.RETIREMENT_STATES = copy.deepcopy(settings.RETIREMENT_STATES)
+    del settings.RETIREMENT_STATES[-1]
+    settings.RETIREMENT_STATES.insert(-2, 'COMPLETE')
+
+    with pytest.raises(CommandError, match=r'in this order'):
+        call_command('populate_retirement_states')
+
+
+def test_end_states_not_at_end(settings):
+    """
+    Test putting a state after the end states
+    """
+    settings.RETIREMENT_STATES = copy.deepcopy(settings.RETIREMENT_STATES)
+    settings.RETIREMENT_STATES.append('ANOTHER_STATE')
+    with pytest.raises(CommandError, match=r'in this order'):
+        call_command('populate_retirement_states')
+
+
+def test_users_in_bad_states():
+    """
+    Test that having users in the process of retirement cause this to fail
+    """
+    user = UserFactory()
+
+    # First populate the table
+    call_command('populate_retirement_states')
+
+    # Create a UserRetirementStatus in an active state
+    retirement = UserRetirementStatus.create_retirement(user)
+    retirement.current_state = RetirementState.objects.get(state_name='LOCKING_ACCOUNT')
+    retirement.save()
+
+    # Now try to update
+    with pytest.raises(CommandError, match=r'Users are currently being processed'):
+        call_command('populate_retirement_states')