Add backoff module as a new requirement.

Use backoff when calling Sailthru API to retry upon optout timeouts.

lms/djangoapps/email_marketing/signals.py

@@ -5,6 +5,7 @@ import datetime
 import logging
 from random import randint
 
+import backoff
 import crum
 from celery.exceptions import TimeoutError
 from django.conf import settings
@@ -263,7 +264,23 @@ def _log_sailthru_api_call_time(time_before_call):
              delta_sailthru_api_call_time.microseconds / 1000)
 
 
+def _backoff_handler(details):
+    """
+    Simple logging handler for when timeout backoff occurs.
+    """
+    log.info('Trying again in {wait:0.1f} seconds after {tries} tries calling {target}'.format(**details))
+
+
+def _not_a_timeout(e):
+    return 'timed out' not in e.message
+
+
 @receiver(USER_RETIRE_MAILINGS)
+@backoff.on_exception(backoff.expo,
+                      Exception,
+                      max_time=600,  # Only 10 minutes of trying
+                      giveup=_not_a_timeout,  # Stop trying if exception is *not* a timeout.
+                      on_backoff=_backoff_handler)
 def force_unsubscribe_all(sender, **kwargs):  # pylint: disable=unused-argument
     """
     Synchronously(!) unsubscribes the given user from all Sailthru email lists.

requirements/edx/base.in

@@ -26,6 +26,7 @@
 analytics-python==1.1.0             # Used for Segment analytics
 attrs                               # Reduces boilerplate code involving class attributes
 Babel==1.3                          # Internationalization utilities, used for date formatting in a few places
+backoff
 bleach==1.4                         # Allowed-list-based HTML sanitizing library that escapes or strips markup and attributes; used for capa and LTI
 boto==2.39.0                        # Deprecated version of the AWS SDK; we should stop using this
 boto3==1.4.8                        # Amazon Web Services SDK for Python

requirements/edx/base.txt

@@ -51,6 +51,7 @@ argparse==1.4.0
 asn1crypto==0.24.0
 attrs==17.4.0
 babel==1.3
+backoff==1.5.0
 beautifulsoup==3.2.1      # via pynliner
 billiard==3.3.0.23        # via celery
 bleach==1.4

requirements/edx/development.txt

@@ -57,6 +57,7 @@ asn1crypto==0.24.0
 astroid==1.5.2
 attrs==17.4.0
 babel==1.3
+backoff==1.5.0
 backports.functools-lru-cache==1.5
 beautifulsoup4==4.6.0
 beautifulsoup==3.2.1

requirements/edx/testing.txt

@@ -54,6 +54,7 @@ asn1crypto==0.24.0
 astroid==1.5.2            # via edx-lint, pylint, pylint-celery, pylint-plugin-utils
 attrs==17.4.0
 babel==1.3
+backoff==1.5.0
 backports.functools-lru-cache==1.5  # via astroid, pylint
 beautifulsoup4==4.6.0
 beautifulsoup==3.2.1