Added custom arguments to the back-populate job

BackPopulate Program Credentials job performs data for all the programs
and for all the available learners which takes a lot of time.
Adding in custom arguments will help run this job for specific learners
and specific programs.

openedx/core/djangoapps/programs/admin.py

@@ -6,7 +6,7 @@ from __future__ import absolute_import
 from config_models.admin import ConfigurationModelAdmin
 from django.contrib import admin
 
-from openedx.core.djangoapps.programs.models import ProgramsApiConfig
+from openedx.core.djangoapps.programs.models import ProgramsApiConfig, CustomProgramsConfig
 
 
 class ProgramsApiConfigAdmin(ConfigurationModelAdmin):
@@ -14,3 +14,4 @@ class ProgramsApiConfigAdmin(ConfigurationModelAdmin):
 
 
 admin.site.register(ProgramsApiConfig, ProgramsApiConfigAdmin)
+admin.site.register(CustomProgramsConfig, ConfigurationModelAdmin)

openedx/core/djangoapps/programs/management/commands/backpopulate_program_credentials.py

@@ -3,10 +3,9 @@ from __future__ import absolute_import
 
 import logging
 from collections import namedtuple
-from functools import reduce  # pylint: disable=redefined-builtin
-
+from functools import reduce  # pylint: disable=redefined-builtin, useless-suppression
 from django.contrib.sites.models import Site
-from django.core.management import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
 from django.db.models import Q
 from opaque_keys.edx.keys import CourseKey
 
@@ -14,6 +13,8 @@ from course_modes.models import CourseMode
 from lms.djangoapps.certificates.models import CertificateStatuses, GeneratedCertificate
 from openedx.core.djangoapps.catalog.utils import get_programs
 from openedx.core.djangoapps.programs.tasks.v1.tasks import award_program_certificates
+from openedx.core.djangoapps.programs.models import CustomProgramsConfig
+
 
 # TODO: Log to console, even with debug mode disabled?
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
@@ -38,13 +39,49 @@ class Command(BaseCommand):
             default=False,
             help='Submit tasks for processing.'
         )
+        parser.add_argument(
+            '--args-from-database',
+            action='store_true',
+            default=False,
+            help='Use arguments from the Config model instead of the command line.',
+        )
+        parser.add_argument(
+            '--program-uuids',
+            nargs='+',
+            help='Award certificates only for specific programs.',
+        )
+        parser.add_argument(
+            '--usernames',
+            nargs='+',
+            help='Award certificates only to specific users.',
+        )
+
+    def get_args_from_database(self):
+        """ Returns an options dictionary from the current NotifyCredentialsConfig model. """
+        config = CustomProgramsConfig.current()
+        if not config.enabled:
+            raise CommandError('CustomProgramsConfig is disabled, but --args-from-database was requested.')
+
+        # We don't need fancy shell-style whitespace/quote handling - none of our arguments are complicated
+        argv = config.arguments.split()
+
+        parser = self.create_parser('manage.py', 'backpopulate_program_credentials')
+        return parser.parse_args(argv).__dict__   # we want a dictionary, not a non-iterable Namespace object
 
     def handle(self, *args, **options):
+        program_uuids, usernames = None, None
+        if options['args_from_database']:
+            logger.info('Loading arguments from the database for custom programs or learners.')
+
+            arguments = self.get_args_from_batabase()  # pylint: disable=no-member
+            program_uuids = arguments.get('program-uuids', None)
+            usernames = arguments.get('usernames', None)
+
         logger.info('Loading programs from the catalog.')
-        self._load_course_runs()
+        self._load_course_runs(program_uuids=program_uuids)
 
         logger.info('Looking for users who may be eligible for a program certificate.')
-        self._load_usernames()
+        self._load_usernames(users=usernames)
 
         if options.get('commit'):
             logger.info(u'Enqueuing program certification tasks for %d candidates.', len(self.usernames))
@@ -73,12 +110,15 @@ class Command(BaseCommand):
             failed
         )
 
-    def _load_course_runs(self):
+    def _load_course_runs(self, program_uuids=None):
         """Find all course runs which are part of a program."""
         programs = []
-        for site in Site.objects.all():
-            logger.info(u'Loading programs from the catalog for site %s.', site.domain)
-            programs.extend(get_programs(site))
+        if program_uuids:
+            programs.extend(get_programs(uuids=program_uuids))
+        else:
+            for site in Site.objects.all():
+                logger.info(u'Loading programs from the catalog for site %s.', site.domain)
+                programs.extend(get_programs(site))
 
         self.course_runs = self._flatten(programs)
 
@@ -95,7 +135,7 @@ class Command(BaseCommand):
 
         return course_runs
 
-    def _load_usernames(self):
+    def _load_usernames(self, users=None):
         """Identify a subset of users who may be eligible for a program certificate.
 
         This is done by finding users who have earned a qualifying certificate in
@@ -118,3 +158,6 @@ class Command(BaseCommand):
             query
         ).values('user__username').distinct()
         self.usernames = [d['user__username'] for d in username_dicts]
+        if users:
+            # keeping only those learners who are in the arguments
+            self.usernames = list(set(self.usernames) & set(users))

openedx/core/djangoapps/programs/migrations/0013_customprogramsconfig.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.26 on 2019-12-13 07:44
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('programs', '0012_auto_20170419_0018'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CustomProgramsConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')),
+                ('enabled', models.BooleanField(default=False, verbose_name='Enabled')),
+                ('arguments', models.TextField(blank=True, default='', help_text='Useful for manually running a Jenkins job. Specify like "--usernames A B --program-uuids X Y".')),
+                ('changed_by', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL, verbose_name='Changed by')),
+            ],
+            options={
+                'verbose_name': 'backpopulate_program_credentials argument',
+            },
+        ),
+    ]

openedx/core/djangoapps/programs/models.py

@@ -2,6 +2,7 @@
 
 from __future__ import absolute_import
 
+import six
 from config_models.models import ConfigurationModel
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
@@ -25,3 +26,21 @@ class ProgramsApiConfig(ConfigurationModel):
             'Path used to construct URLs to programs marketing pages (e.g., "/foo").'
         )
     )
+
+
+class CustomProgramsConfig(ConfigurationModel):  # pylint: disable=model-missing-unicode, useless-suppression
+    """
+    Manages configuration for a run of the backpopulate_program_credentials management command.
+    """
+    class Meta(object):
+        app_label = 'programs'
+        verbose_name = 'backpopulate_program_credentials argument'
+
+    arguments = models.TextField(
+        blank=True,
+        help_text='Useful for manually running a Jenkins job. Specify like "--usernames A B --program-uuids X Y".',
+        default='',
+    )
+
+    def __str__(self):
+        return six.text_type(self.arguments)

openedx/core/djangoapps/programs/tasks/v1/tasks.py

@@ -193,6 +193,8 @@ def award_program_certificates(self, username):
         for program_uuid in new_program_uuids:
             visible_date = completed_programs[program_uuid]
             try:
+                LOGGER.info(u'Visible date for user %s : program %s is %s', username, program_uuid,
+                            visible_date)
                 award_program_certificate(credentials_client, username, program_uuid, visible_date)
                 LOGGER.info(u'Awarded certificate for program %s to user %s', program_uuid, username)
             except exceptions.HttpNotFoundError: