Annotates block structures settings and waffle switches

This includes the following settings:

- BLOCK_STRUCTURES_SETTINGS : setting dictionary which stores the other different block structures related settings

- BLOCK_STRUCTURES_SETTINGS['PRUNING_ACTIVE'] : keeps only a specific number of versions of each block structure, and deletes the rest

- BLOCK_STRUCTURES_SETTINGS['COURSE_PUBLISH_TASK_DELAY'] : specifies the delay, in seconds, after a new edit of a course is published before updating the block structures cache

- BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY'] : Specifies the delay, in seconds, between retry attempts for block structure tasks

- BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES'] : specifies the max number of retries per block structure task

- BLOCK_STRUCTURES_SETTINGS['STORAGE_CLASS'] : specifies the storage which block structures would be saved to when storage backed block structures are enabled

    Example: 'storages.backends.s3boto.S3BotoStorage'

- BLOCK_STRUCTURES_SETTINGS['STORAGE_KWARGS'] : specifies additional arguments needed when utilizing storage for storing storage backed block structures

    Example: { bucket: 'test-edxapp' }

- BLOCK_STRUCTURES_SETTINGS['DIRECTORY_PREFIX'] : specifies the path to which storage backed block structures are saved

    Example: '/courses/'

This also includes the following waffle switches:

- block_structure.storage_backing_for_cache : enables storage backed block structures, used to retrieve the block structures from storage instead of regenerating the structure, when not available in cache

    it is important to note that this is important for production because it reduces response times on block structure related apis

- block_structure.raise_error_when_not_found : raises an error if the block structure requested doesnt exist in store or is outdated

- block_structure.invalidate_cache_on_publish : invalidates the block structure cache when courses are published

For an example with additional context, see the following forum post:

https://discuss.openedx.org/t/help-please-very-slow-load-time-10-seconds-for-courseware-on-sections-with-several-subsections-and-xblocks/2998/16

This also includes information about the toggles that will likely be deprecated and removed:

https://github.com/edx/edx-platform/pull/26175#issuecomment-769080286

lms/envs/common.py

@@ -2347,21 +2347,47 @@ HEARTBEAT_CELERY_ROUTING_KEY = HIGH_PRIORITY_QUEUE
 
 ################################ Block Structures ###################################
 
+# .. setting_name: BLOCK_STRUCTURES_SETTINGS
+# .. setting_default: dict of settings
+# .. setting_description: Stores all the settings used by block structures and block structure
+#   related tasks. See BLOCK_STRUCTURES_SETTINGS[XXX] documentation for details of each setting.
+#   For more information, check https://openedx.atlassian.net/browse/TNL-5041.
 BLOCK_STRUCTURES_SETTINGS = dict(
-    # Delay, in seconds, after a new edit of a course is published
-    # before updating the block structures cache.  This is needed
-    # for a better chance at getting the latest changes when there
-    # are secondary reads in sharded mongoDB clusters. See TNL-5041
-    # for more info.
+    # .. setting_name: BLOCK_STRUCTURES_SETTINGS['COURSE_PUBLISH_TASK_DELAY']
+    # .. setting_default: 30
+    # .. setting_description: Delay, in seconds, after a new edit of a course is published before
+    #   updating the block structures cache. This is needed for a better chance at getting
+    #   the latest changes when there are secondary reads in sharded mongoDB clusters.
+    #   For more information, check https://openedx.atlassian.net/browse/TNL-5041.
     COURSE_PUBLISH_TASK_DELAY=30,
 
-    # Delay, in seconds, between retry attempts if a task fails.
+    # .. setting_name: BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY']
+    # .. setting_default: 30
+    # .. setting_description: Delay, in seconds, between retry attempts if a block structure task
+    #   fails. For more information, check https://openedx.atlassian.net/browse/TNL-5041.
     TASK_DEFAULT_RETRY_DELAY=30,
 
-    # Maximum number of retries per task.
+    # .. setting_name: BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES']
+    # .. setting_default: 5
+    # .. setting_description: Maximum number of retries per block structure task.
+    #   If the maximum number of retries is exceeded, then you can attempt to either manually run
+    #   the celery task, or wait for it to be triggered again.
+    #   For more information, check https://openedx.atlassian.net/browse/TNL-5041.
     TASK_MAX_RETRIES=5,
 
-    # Backend storage options
+    # .. toggle_name: BLOCK_STRUCTURES_SETTINGS['PRUNING_ACTIVE']
+    # .. toggle_implementation: DjangoSetting
+    # .. toggle_default: False
+    # .. toggle_description: When `True`, only a specified number of versions of block structure
+    #   files are kept for each structure, and the rest are cleaned up. The number of versions that
+    #   are kept can be specified in the `BlockStructureConfiguration`, which can be edited in
+    #   Django Admin. The default number of versions that are kept is `5`.
+    # .. toggle_warnings: This toggle will likely be deprecated and removed.
+    #   The annotation will be updated with the DEPR ticket once that process has started.
+    # .. toggle_use_cases: temporary
+    # .. toggle_creation_date: 2018-03-22
+    # .. toggle_target_removal_date: 2018-06-22
+    # .. toggle_tickets: https://openedx.atlassian.net/browse/EDUCATOR-499
     PRUNING_ACTIVE=False,
 )
 

openedx/core/djangoapps/content/block_structure/config/__init__.py

@@ -10,12 +10,53 @@ from openedx.core.lib.cache_utils import request_cached
 from .models import BlockStructureConfiguration
 
 # Switches
+# .. toggle_name: block_structure.invalidate_cache_on_publish
+# .. toggle_implementation: WaffleSwitch
+# .. toggle_default: False
+# .. toggle_description: When enabled, the block structure cache is invalidated when changes to
+#   courses are published. If `block_structure.storage_backing_for_cache` is active, all block
+#   structures related to the published course are also cleared from storage.
+# .. toggle_warnings: This switch will likely be deprecated and removed.
+#   The annotation will be updated with the DEPR ticket once that process has started.
+# .. toggle_use_cases: temporary
+# .. toggle_creation_date: 2017-02-23
+# .. toggle_target_removal_date: 2017-05-23
+# .. toggle_tickets: https://github.com/edx/edx-platform/pull/14571
 INVALIDATE_CACHE_ON_PUBLISH = WaffleSwitch(
     "block_structure.invalidate_cache_on_publish", __name__
 )
+
+# .. toggle_name: block_structure.storage_backing_for_cache
+# .. toggle_implementation: WaffleSwitch
+# .. toggle_default: False
+# .. toggle_description: When enabled, block structures are stored in a more permanent storage,
+#   like a database, which provides an additional backup for cache misses, instead having them
+#   regenerated. The regenration of block structures is a time consuming process. Therefore,
+#   enabling this switch is recommended for Production.
+# .. toggle_warnings: Depends on `BLOCK_STRUCTURES_SETTINGS['STORAGE_CLASS']` and
+#   `BLOCK_STRUCTURES_SETTINGS['STORAGE_KWARGS']`.
+#   This switch will likely be deprecated and removed.
+#   The annotation will be updated with the DEPR ticket once that process has started.
+# .. toggle_use_cases: temporary
+# .. toggle_creation_date: 2017-02-23
+# .. toggle_target_removal_date: 2017-05-23
+# .. toggle_tickets: https://github.com/edx/edx-platform/pull/14512
 STORAGE_BACKING_FOR_CACHE = WaffleSwitch(
     "block_structure.storage_backing_for_cache", __name__
 )
+
+# .. toggle_name: block_structure.raise_error_when_not_found
+# .. toggle_implementation: WaffleSwitch
+# .. toggle_default: False
+# .. toggle_description: Raises an error if the requested block structure does not exist in block
+#   structure store, or if it is outdated. Block structure store refers to both cache and storage,
+#   if enabled.
+# .. toggle_warnings: This switch will likely be deprecated and removed.
+#   The annotation will be updated with the DEPR ticket once that process has started.
+# .. toggle_use_cases: temporary
+# .. toggle_creation_date: 2017-02-23
+# .. toggle_target_removal_date: 2017-05-23
+# .. toggle_tickets: https://github.com/edx/edx-platform/pull/14512
 RAISE_ERROR_WHEN_NOT_FOUND = WaffleSwitch(
     "block_structure.raise_error_when_not_found", __name__
 )

openedx/core/djangoapps/content/block_structure/models.py

@@ -38,11 +38,19 @@ def _directory_name(data_usage_key):
     Returns the directory name for the given
     data_usage_key.
     """
+    # .. setting_name: BLOCK_STRUCTURES_SETTINGS['DIRECTORY_PREFIX']
+    # .. setting_default: ''
+    # .. setting_description: Specifies the path in storage where block structures would be saved,
+    #   for storage-backed block structure cache.
+    # .. setting_warnings: Depends on `BLOCK_STRUCTURES_SETTINGS['STORAGE_CLASS']` and on
+    #   `block_structure.storage_backing_for_cache`.
+    directory_prefix = settings.BLOCK_STRUCTURES_SETTINGS.get('DIRECTORY_PREFIX', '')
+
     # replace any '/' in the usage key so they aren't interpreted
     # as folder separators.
     encoded_usage_key = six.text_type(data_usage_key).replace('/', '_')
     return '{}{}'.format(
-        settings.BLOCK_STRUCTURES_SETTINGS.get('DIRECTORY_PREFIX', ''),
+        directory_prefix,
         encoded_usage_key,
     )
 
@@ -63,10 +71,21 @@ def _bs_model_storage():
     """
     Get django Storage object for BlockStructureModel.
     """
-    return get_storage(
-        settings.BLOCK_STRUCTURES_SETTINGS.get('STORAGE_CLASS'),
-        **settings.BLOCK_STRUCTURES_SETTINGS.get('STORAGE_KWARGS', {})
-    )
+    # .. setting_name: BLOCK_STRUCTURES_SETTINGS['STORAGE_CLASS']
+    # .. setting_default: None
+    # .. setting_description: Specifies the storage used for storage-backed block structure cache.
+    # .. setting_warnings: Depends on `block_structure.storage_backing_for_cache`.
+    storage_class = settings.BLOCK_STRUCTURES_SETTINGS.get('STORAGE_CLASS')
+
+    # .. setting_name: BLOCK_STRUCTURES_SETTINGS['STORAGE_KWARGS']
+    # .. setting_default: {}
+    # .. setting_description: Specifies the keyword arguments needed to setup the storage, which
+    #   would be used for storage-backed block structure cache.
+    # .. setting_warnings: Depends on `BLOCK_STRUCTURES_SETTINGS['STORAGE_CLASS']` and on
+    #   `block_structure.storage_backing_for_cache`.
+    storage_kwargs = settings.BLOCK_STRUCTURES_SETTINGS.get('STORAGE_KWARGS', {})
+
+    return get_storage(storage_class, **storage_kwargs)
 
 
 class CustomizableFileField(models.FileField):