Merge pull request #15354 from edx/jmbowman/PLAT-1228

PLAT-1228 Support bok-choy in Docker Devstack

cms/envs/bok_choy.py

@@ -21,7 +21,7 @@ from openedx.core.release import RELEASE_LINE
 # Unlike in prod, we use the JSON files stored in this repo.
 # This is a convenience for ensuring (a) that we can consistently find the files
 # and (b) that the files are the same in Jenkins as in local dev.
-os.environ['SERVICE_VARIANT'] = 'bok_choy'
+os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy'
 os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname()
 
 from .aws import *  # pylint: disable=wildcard-import, unused-wildcard-import

cms/envs/bok_choy_docker.auth.json

+{
+    "ANALYTICS_API_KEY": "",
+    "AWS_ACCESS_KEY_ID": "",
+    "AWS_SECRET_ACCESS_KEY": "",
+    "CELERY_BROKER_PASSWORD": "celery",
+    "CELERY_BROKER_USER": "celery",
+    "CONTENTSTORE": {
+        "DOC_STORE_CONFIG": {
+            "collection": "modulestore",
+            "db": "test",
+            "host": [
+                "edx.devstack.mongo"
+            ],
+            "port": 27017
+        },
+        "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
+        "OPTIONS": {
+            "db": "test",
+            "host": [
+                "edx.devstack.mongo"
+            ],
+            "port": 27017
+        }
+    },
+    "DATABASES": {
+        "default": {
+            "ENGINE": "django.db.backends.mysql",
+            "HOST": "edx.devstack.mysql",
+            "NAME": "edxtest",
+            "PASSWORD": "",
+            "PORT": "3306",
+            "USER": "root"
+        },
+        "student_module_history": {
+            "ENGINE": "django.db.backends.mysql",
+            "HOST": "edx.devstack.mysql",
+            "NAME": "student_module_history_test",
+            "PASSWORD": "",
+            "PORT": "3306",
+            "USER": "root"
+        }
+    },
+    "DOC_STORE_CONFIG": {
+        "collection": "modulestore",
+        "db": "test",
+        "host": [
+            "edx.devstack.mongo"
+        ],
+        "port": 27017
+    },
+    "MODULESTORE": {
+        "default": {
+            "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
+            "OPTIONS": {
+                "mappings": {},
+                "stores": [
+                    {
+                        "NAME": "draft",
+                        "DOC_STORE_CONFIG": {
+                            "collection": "modulestore",
+                            "db": "test",
+                            "host": [
+                                "edx.devstack.mongo"
+                            ],
+                            "port": 27017
+                        },
+                        "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
+                        "OPTIONS": {
+                            "collection": "modulestore",
+                            "db": "test",
+                            "default_class": "xmodule.hidden_module.HiddenDescriptor",
+                            "fs_root": "** OVERRIDDEN **",
+                            "host": [
+                                "edx.devstack.mongo"
+                            ],
+                            "port": 27017,
+                            "render_template": "edxmako.shortcuts.render_to_string"
+                        }
+                    },
+                    {
+                        "NAME": "xml",
+                        "ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
+                        "OPTIONS": {
+                            "data_dir": "** OVERRIDDEN **",
+                            "default_class": "xmodule.hidden_module.HiddenDescriptor"
+                        }
+                    }
+                ]
+            }
+        }
+    },
+    "DJFS": {
+            "type": "s3fs",
+            "bucket": "test",
+            "prefix": "test",
+            "aws_access_key_id": "test",
+            "aws_secret_access_key": "test"
+    },
+    "SECRET_KEY": "",
+    "XQUEUE_INTERFACE": {
+        "basic_auth": [
+            "edx",
+            "edx"
+        ],
+        "django_auth": {
+            "password": "password",
+            "username": "lms"
+        },
+        "url": "http://localhost:18040"
+    },
+    "ZENDESK_API_KEY": "",
+    "ZENDESK_USER": ""
+}

cms/envs/bok_choy_docker.env.json

+{
+    "ANALYTICS_SERVER_URL": "",
+    "BOOK_URL": "",
+    "BUGS_EMAIL": "bugs@example.com",
+    "BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com",
+    "CACHES": {
+        "celery": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "integration_celery",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        },
+        "default": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "sandbox_default",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        },
+        "general": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "sandbox_general",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        },
+        "mongo_metadata_inheritance": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "integration_mongo_metadata_inheritance",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        },
+        "staticfiles": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "integration_static_files",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        }
+    },
+    "CELERY_ALWAYS_EAGER": true,
+    "CELERY_BROKER_HOSTNAME": "localhost",
+    "CELERY_BROKER_TRANSPORT": "amqp",
+    "CERT_QUEUE": "certificates",
+    "CMS_BASE": "** OVERRIDDEN **",
+    "CODE_JAIL": {
+        "limits": {
+            "REALTIME": 3,
+            "VMEM": 0
+        }
+    },
+    "COMMENTS_SERVICE_KEY": "password",
+    "COMMENTS_SERVICE_URL": "http://localhost:4567",
+    "CONTACT_EMAIL": "info@example.com",
+    "DEFAULT_FEEDBACK_EMAIL": "feedback@example.com",
+    "DEFAULT_FROM_EMAIL": "registration@example.com",
+    "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend",
+    "SOCIAL_SHARING_SETTINGS": {
+        "CUSTOM_COURSE_URLS": true
+    },
+    "FEATURES": {
+        "AUTH_USE_OPENID_PROVIDER": true,
+        "CERTIFICATES_HTML_VIEW": true,
+        "ENABLE_DISCUSSION_SERVICE": true,
+        "ENABLE_GRADE_DOWNLOADS": true,
+        "ENTRANCE_EXAMS": true,
+        "MILESTONES_APP": true,
+        "PREVIEW_LMS_BASE": "preview.localhost:8003",
+        "ENABLE_CONTENT_LIBRARIES": true,
+        "ENABLE_SPECIAL_EXAMS": true,
+        "SHOW_HEADER_LANGUAGE_SELECTOR": true,
+        "ENABLE_EXTENDED_COURSE_DETAILS": true,
+        "CUSTOM_COURSES_EDX": true
+    },
+    "FEEDBACK_SUBMISSION_EMAIL": "",
+    "GITHUB_REPO_ROOT": "** OVERRIDDEN **",
+    "JWT_AUTH": {
+        "JWT_SECRET_KEY": "super-secret-key"
+    },
+    "GRADES_DOWNLOAD": {
+        "BUCKET": "edx-grades",
+        "ROOT_PATH": "/tmp/edx-s3/grades",
+        "STORAGE_TYPE": "localfs"
+    },
+    "LMS_BASE": "** OVERRIDDEN **",
+    "LMS_ROOT_URL": "** OVERRIDDEN **",
+    "LOCAL_LOGLEVEL": "INFO",
+    "LOGGING_ENV": "sandbox",
+    "LOG_DIR": "** OVERRIDDEN **",
+    "MEDIA_URL": "/media/",
+    "MKTG_URL_LINK_MAP": {},
+    "PLATFORM_NAME": "édX",
+    "SERVER_EMAIL": "devops@example.com",
+    "SESSION_COOKIE_DOMAIN": null,
+    "SITE_NAME": "localhost",
+    "STATIC_URL_BASE": "/static/",
+    "SYSLOG_SERVER": "",
+    "TECH_SUPPORT_EMAIL": "technical@example.com",
+    "TIME_ZONE": "America/New_York",
+    "WIKI_ENABLED": true,
+    "OAUTH_OIDC_ISSUER": "https://www.example.com/oauth2"
+}

cms/envs/bok_choy_docker.py

+# -*- coding: utf-8 -*-
+"""
+Settings for Bok Choy tests that are used when running Studio in Docker-based devstack.
+"""
+
+# noinspection PyUnresolvedReferences
+from .bok_choy import *  # pylint: disable=wildcard-import
+
+CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_CMS_PORT'])
+LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_LMS_PORT'])
+LMS_ROOT_URL = 'http://{}'.format(LMS_BASE)
+
+# Docker does not support the syslog socket at /dev/log. Rely on the console.
+LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = {
+    'class': 'logging.NullHandler',
+}
+
+LOGGING['loggers']['tracking']['handlers'] = ['console']

common/test/acceptance/fixtures/__init__.py

 import os
 
 
+HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
+CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', '8031')
+LMS_PORT = os.environ.get('BOK_CHOY_LMS_PORT', '8003')
+
 # Get the URL of the Studio instance under test
-STUDIO_BASE_URL = os.environ.get('studio_url', 'http://localhost:8031')
+STUDIO_BASE_URL = os.environ.get('studio_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT))
 
 # Get the URL of the LMS instance under test
-LMS_BASE_URL = os.environ.get('lms_url', 'http://localhost:8003')
+LMS_BASE_URL = os.environ.get('lms_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT))
 
 # Get the URL of the XQueue stub used in the test
 XQUEUE_STUB_URL = os.environ.get('xqueue_url', 'http://localhost:8040')

common/test/acceptance/pages/common/__init__.py

 import os
 
+HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
+CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031)
+LMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8003)
+
 # Get the URL of the instance under test
-BASE_URL = os.environ.get('test_url', 'http://localhost:8003')
+BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT))
 
 # The URL used for user auth in testing
-AUTH_BASE_URL = os.environ.get('test_url', 'http://localhost:8031')
+AUTH_BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT))

common/test/acceptance/pages/common/auto_auth.py

@@ -8,7 +8,9 @@ import urllib
 from bok_choy.page_object import XSS_INJECTION, PageObject, unguarded
 
 # The URL used for user auth in testing
-AUTH_BASE_URL = os.environ.get('test_url', 'http://localhost:8031')
+HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
+CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031)
+AUTH_BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT))
 
 
 class AutoAuthPage(PageObject):

common/test/acceptance/pages/lms/__init__.py

 import os
 
 # Get the URL of the instance under test
-BASE_URL = os.environ.get('test_url', 'http://localhost:8003')
+HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
+LMS_PORT = os.environ.get('BOK_CHOY_LMS_PORT', 8003)
+BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT))

common/test/acceptance/pages/studio/__init__.py

 import os
 
 # Get the URL of the instance under test
-BASE_URL = os.environ.get('test_url', 'http://localhost:8031')
+HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost')
+CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031)
+BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT))

lms/envs/bok_choy.py

@@ -25,7 +25,7 @@ TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"
 # Unlike in prod, we use the JSON files stored in this repo.
 # This is a convenience for ensuring (a) that we can consistently find the files
 # and (b) that the files are the same in Jenkins as in local dev.
-os.environ['SERVICE_VARIANT'] = 'bok_choy'
+os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy'
 os.environ['CONFIG_ROOT'] = CONFIG_ROOT
 
 from .aws import *  # pylint: disable=wildcard-import, unused-wildcard-import

lms/envs/bok_choy_docker.auth.json

+{
+    "ANALYTICS_API_KEY": "",
+    "AWS_ACCESS_KEY_ID": "",
+    "AWS_SECRET_ACCESS_KEY": "",
+    "CC_PROCESSOR_NAME": "CyberSource2",
+    "CC_PROCESSOR": {
+        "CyberSource2": {
+            "SECRET_KEY": "abcd123",
+            "ACCESS_KEY": "abcd123",
+            "PROFILE_ID": "edx",
+            "PURCHASE_ENDPOINT": "/shoppingcart/payment_fake"
+        }
+    },
+    "CELERY_BROKER_PASSWORD": "celery",
+    "CELERY_BROKER_USER": "celery",
+    "CONTENTSTORE": {
+        "DOC_STORE_CONFIG": {
+            "collection": "modulestore",
+            "db": "test",
+            "host": [
+                "edx.devstack.mongo"
+            ],
+            "port": 27017
+        },
+        "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
+        "OPTIONS": {
+            "db": "test",
+            "host": [
+                "edx.devstack.mongo"
+            ],
+            "port": 27017
+        }
+    },
+    "DATABASES": {
+        "default": {
+            "ENGINE": "django.db.backends.mysql",
+            "HOST": "edx.devstack.mysql",
+            "NAME": "edxtest",
+            "PASSWORD": "",
+            "PORT": "3306",
+            "USER": "root"
+        },
+        "student_module_history": {
+            "ENGINE": "django.db.backends.mysql",
+            "HOST": "edx.devstack.mysql",
+            "NAME": "student_module_history_test",
+            "PASSWORD": "",
+            "PORT": "3306",
+            "USER": "root"
+        }
+    },
+    "DOC_STORE_CONFIG": {
+        "collection": "modulestore",
+        "db": "test",
+        "host": [
+            "edx.devstack.mongo"
+        ],
+        "port": 27017
+    },
+    "TRACKING_BACKENDS": {
+        "mongo": {
+            "ENGINE": "track.backends.mongodb.MongoBackend",
+            "OPTIONS": {
+                "database": "test",
+                "collection": "events",
+                "host": [
+                    "edx.devstack.mongo"
+                ],
+                "port": 27017
+            }
+        }
+    },
+    "EVENT_TRACKING_BACKENDS": {
+        "mongo": {
+            "ENGINE": "eventtracking.backends.mongodb.MongoBackend",
+            "OPTIONS": {
+                "database": "test",
+                "collection": "events",
+                "host": [
+                    "edx.devstack.mongo"
+                ],
+                "port": 27017
+            }
+        }
+    },
+    "MODULESTORE": {
+        "default": {
+            "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
+            "OPTIONS": {
+                "mappings": {},
+                "stores": [
+                    {
+                        "NAME": "draft",
+                        "DOC_STORE_CONFIG": {
+                            "collection": "modulestore",
+                            "db": "test",
+                            "host": [
+                                "edx.devstack.mongo"
+                            ],
+                            "port": 27017
+                        },
+                        "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore",
+                        "OPTIONS": {
+                            "collection": "modulestore",
+                            "db": "test",
+                            "default_class": "xmodule.hidden_module.HiddenDescriptor",
+                            "fs_root": "** OVERRIDDEN **",
+                            "host": [
+                                "edx.devstack.mongo"
+                            ],
+                            "port": 27017,
+                            "render_template": "edxmako.shortcuts.render_to_string"
+                        }
+                    },
+                    {
+                        "NAME": "xml",
+                        "ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
+                        "OPTIONS": {
+                            "data_dir": "** OVERRIDDEN **",
+                            "default_class": "xmodule.hidden_module.HiddenDescriptor"
+                        }
+                    }
+                ]
+            }
+        }
+    },
+    "SECRET_KEY": "",
+    "DJFS": {
+            "type": "s3fs",
+            "bucket": "test",
+            "prefix": "test",
+            "aws_access_key_id": "test",
+            "aws_secret_access_key": "test"
+    },
+    "XQUEUE_INTERFACE": {
+        "basic_auth": [
+            "edx",
+            "edx"
+        ],
+        "django_auth": {
+            "password": "password",
+            "username": "lms"
+        },
+        "url": "** OVERRIDDEN **"
+    },
+    "ZENDESK_API_KEY": "",
+    "ZENDESK_USER": ""
+}

lms/envs/bok_choy_docker.env.json

+{
+    "ANALYTICS_SERVER_URL": "",
+    "ANALYTICS_DASHBOARD_URL": "",
+    "BOOK_URL": "",
+    "BUGS_EMAIL": "bugs@example.com",
+    "BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com",
+    "CACHES": {
+        "celery": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "integration_celery",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        },
+        "default": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "sandbox_default",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        },
+        "general": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "sandbox_general",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        },
+        "mongo_metadata_inheritance": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "integration_mongo_metadata_inheritance",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        },
+        "staticfiles": {
+            "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+            "KEY_FUNCTION": "util.memcache.safe_key",
+            "KEY_PREFIX": "integration_static_files",
+            "LOCATION": [
+                "edx.devstack.memcached:11211"
+            ]
+        }
+    },
+    "CELERY_BROKER_HOSTNAME": "localhost",
+    "CELERY_BROKER_TRANSPORT": "amqp",
+    "CERT_QUEUE": "certificates",
+    "CMS_BASE": "** OVERRIDDEN **",
+    "CODE_JAIL": {
+        "limits": {
+            "REALTIME": 3,
+            "VMEM": 0
+        }
+    },
+    "COMMENTS_SERVICE_KEY": "password",
+    "COMMENTS_SERVICE_URL": "http://localhost:4567",
+    "CONTACT_EMAIL": "info@example.com",
+    "DEFAULT_FEEDBACK_EMAIL": "feedback@example.com",
+    "DEFAULT_FROM_EMAIL": "registration@example.com",
+    "EMAIL_BACKEND": "django.core.mail.backends.dummy.EmailBackend",
+    "SOCIAL_SHARING_SETTINGS": {
+        "CUSTOM_COURSE_URLS": true,
+        "DASHBOARD_FACEBOOK": true,
+        "CERTIFICATE_FACEBOOK": true,
+        "CERTIFICATE_FACEBOOK_TEXT": "Testing facebook feature:",
+        "DASHBOARD_TWITTER": true,
+        "DASHBOARD_TWITTER_TEXT": "Testing feature:"
+    },
+    "FEATURES": {
+        "AUTH_USE_OPENID_PROVIDER": true,
+        "CERTIFICATES_HTML_VIEW": true,
+        "CERTIFICATES_INSTRUCTOR_GENERATION": true,
+        "ENABLE_PAYMENT_FAKE": true,
+        "ENABLE_VERIFIED_CERTIFICATES": true,
+        "ENABLE_DISCUSSION_SERVICE": true,
+        "ENABLE_GRADE_DOWNLOADS": true,
+        "ENABLE_THIRD_PARTY_AUTH": true,
+        "ENABLE_COMBINED_LOGIN_REGISTRATION": true,
+        "PREVIEW_LMS_BASE": "preview.localhost:8003",
+        "ALLOW_AUTOMATED_SIGNUPS": true,
+        "AUTOMATIC_AUTH_FOR_TESTING": true,
+        "MODE_CREATION_FOR_TESTING": true,
+        "EXPOSE_CACHE_PROGRAMS_ENDPOINT": true,
+        "AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING": true,
+        "ENABLE_COURSE_DISCOVERY": true,
+        "ENABLE_SPECIAL_EXAMS": true,
+        "SHOW_HEADER_LANGUAGE_SELECTOR": true,
+        "CUSTOM_COURSES_EDX": true
+    },
+    "FEEDBACK_SUBMISSION_EMAIL": "",
+    "GITHUB_REPO_ROOT": "** OVERRIDDEN **",
+    "JWT_AUTH": {
+        "JWT_SECRET_KEY": "super-secret-key"
+    },
+    "LMS_BASE": "** OVERRIDDEN **",
+    "LMS_ROOT_URL": "** OVERRIDDEN **",
+    "LOCAL_LOGLEVEL": "INFO",
+    "LOGGING_ENV": "sandbox",
+    "LOG_DIR": "** OVERRIDDEN **",
+    "MEDIA_URL": "/media/",
+    "MKTG_URL_LINK_MAP": {
+        "ABOUT": "about",
+        "PRIVACY": "privacy",
+        "TOS": "tos",
+        "WHAT_IS_VERIFIED_CERT": "verified-certificate",
+        "COURSES": "courses",
+        "CONTACT": "contact",
+        "NEWS": "news",
+        "HONOR": "honor",
+        "CAREERS": "careers",
+        "HELP_CENTER": "help-center",
+        "BLOG": "blog",
+        "PRESS": "press",
+        "DONATE": "donate",
+        "ROOT": "root",
+        "SITEMAP.XML": "sitemap_xml"
+    },
+    "PLATFORM_NAME": "édX",
+    "REGISTRATION_EXTENSION_FORM": "openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm",
+    "REGISTRATION_EXTRA_FIELDS": {
+        "level_of_education": "optional",
+        "gender": "optional",
+        "year_of_birth": "optional",
+        "mailing_address": "optional",
+        "goals": "optional",
+        "honor_code": "required",
+        "terms_of_service": "hidden",
+        "city": "hidden",
+        "country": "required"
+    },
+    "SERVER_EMAIL": "devops@example.com",
+    "SESSION_COOKIE_DOMAIN": null,
+    "SITE_NAME": "localhost:8003",
+    "STATIC_URL_BASE": "/static/",
+    "SUPPORT_SITE_LINK": "https://support.example.com",
+    "PASSWORD_RESET_SUPPORT_LINK": "https://support.example.com/password-reset-help.html",
+    "ACTIVATION_EMAIL_SUPPORT_LINK": "https://support.example.com/activation-email-help.html",
+    "SYSLOG_SERVER": "",
+    "TECH_SUPPORT_EMAIL": "technical@example.com",
+    "THIRD_PARTY_AUTH_BACKENDS": [
+        "social.backends.google.GoogleOAuth2",
+        "social.backends.linkedin.LinkedinOAuth2",
+        "social.backends.facebook.FacebookOAuth2",
+        "third_party_auth.dummy.DummyBackend",
+        "third_party_auth.saml.SAMLAuthBackend"
+    ],
+    "TIME_ZONE": "America/New_York",
+    "WIKI_ENABLED": true
+}

lms/envs/bok_choy_docker.py

+# -*- coding: utf-8 -*-
+"""
+Settings for Bok Choy tests that are used when running Studio in Docker-based devstack.
+"""
+
+# noinspection PyUnresolvedReferences
+from .bok_choy import *  # pylint: disable=wildcard-import
+
+CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_CMS_PORT'])
+LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_LMS_PORT'])
+LMS_ROOT_URL = 'http://{}'.format(LMS_BASE)
+
+# Docker does not support the syslog socket at /dev/log. Rely on the console.
+LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = {
+    'class': 'logging.NullHandler',
+}
+
+LOGGING['loggers']['tracking']['handlers'] = ['console']

pavelib/utils/envs.py

@@ -65,13 +65,20 @@ class Env(object):
     # Directory that videos are served from
     VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video"
 
+    # Detect if in a Docker container, and if so which one
+    SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0')
+    USING_DOCKER = SERVER_HOST != '0.0.0.0'
+    SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy'
+
     BOK_CHOY_SERVERS = {
         'lms': {
-            'port': 8003,
+            'host': SERVER_HOST,
+            'port': os.environ.get('BOK_CHOY_LMS_PORT', '8003'),
             'log': BOK_CHOY_LOG_DIR / "bok_choy_lms.log"
         },
         'cms': {
-            'port': 8031,
+            'host': SERVER_HOST,
+            'port': os.environ.get('BOK_CHOY_CMS_PORT', '8031'),
             'log': BOK_CHOY_LOG_DIR / "bok_choy_studio.log"
         }
     }
@@ -123,8 +130,10 @@ class Env(object):
     }
 
     # Mongo databases that will be dropped before/after the tests run
+    BOK_CHOY_MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost'
     BOK_CHOY_MONGO_DATABASE = "test"
-    BOK_CHOY_CACHE = memcache.Client(['0.0.0.0:11211'], debug=0)
+    BOK_CHOY_CACHE_HOST = 'edx.devstack.memcached' if USING_DOCKER else '0.0.0.0'
+    BOK_CHOY_CACHE = memcache.Client(['{}:11211'.format(BOK_CHOY_CACHE_HOST)], debug=0)
 
     # Test Ids Directory
     TEST_DIR = REPO_ROOT / ".testids"

pavelib/utils/test/bokchoy_utils.py

@@ -44,11 +44,12 @@ def start_servers(options):
         cmd = (
             "DEFAULT_STORE={default_store} "
             "coverage run --rcfile={coveragerc} -m "
-            "manage {service} --settings bok_choy runserver "
+            "manage {service} --settings {settings} runserver "
             "{address} --traceback --noreload".format(
                 default_store=options.default_store,
                 coveragerc=coveragerc,
                 service=service,
+                settings=Env.SETTINGS,
                 address=address,
             )
         )
@@ -83,7 +84,7 @@ def wait_for_server(server, port):
     attempts = 0
     server_ok = False
 
-    while attempts < 30:
+    while attempts < 120:
         try:
             connection = httplib.HTTPConnection(server, port, timeout=10)
             connection.request('GET', '/')
@@ -107,7 +108,7 @@ def wait_for_test_servers():
     """
 
     for service, info in Env.BOK_CHOY_SERVERS.iteritems():
-        ready = wait_for_server("0.0.0.0", info['port'])
+        ready = wait_for_server(info['host'], info['port'])
         if not ready:
             msg = colorize(
                 "red",
@@ -123,7 +124,7 @@ def is_mongo_running():
     """
     # The mongo command will connect to the service,
     # failing with a non-zero exit code if it cannot connect.
-    output = os.popen('mongo --eval "print(\'running\')"').read()
+    output = os.popen('mongo --host {} --eval "print(\'running\')"'.format(Env.BOK_CHOY_MONGO_HOST)).read()
     return output and "running" in output
 
 
@@ -155,7 +156,8 @@ def clear_mongo():
     Clears mongo database.
     """
     sh(
-        "mongo {} --eval 'db.dropDatabase()' > /dev/null".format(
+        "mongo --host {} {} --eval 'db.dropDatabase()' > /dev/null".format(
+            Env.BOK_CHOY_MONGO_HOST,
             Env.BOK_CHOY_MONGO_DATABASE,
         )
     )
@@ -191,6 +193,9 @@ def check_mysql():
     """
     Check that mysql is running
     """
+    if 'BOK_CHOY_HOSTNAME' in os.environ:
+        # mysql should be running in a separate Docker container
+        return
     if not is_mysql_running():
         msg = colorize('red', "MySQL is not running locally.")
         print msg

pavelib/utils/test/suites/bokchoy_suite.py

@@ -48,9 +48,10 @@ def load_bok_choy_data(options):
     print 'Loading data from json fixtures in db_fixtures directory'
     sh(
         "DEFAULT_STORE={default_store}"
-        " ./manage.py lms --settings bok_choy loaddata --traceback"
+        " ./manage.py lms --settings {settings} loaddata --traceback"
         " common/test/db_fixtures/*.json".format(
             default_store=options.default_store,
+            settings=Env.SETTINGS
         )
     )
 
@@ -76,9 +77,10 @@ def load_courses(options):
 
         sh(
             "DEFAULT_STORE={default_store}"
-            " ./manage.py cms --settings=bok_choy import {import_dir}".format(
+            " ./manage.py cms --settings={settings} import {import_dir}".format(
                 default_store=options.default_store,
-                import_dir=options.imports_dir
+                import_dir=options.imports_dir,
+                settings=Env.SETTINGS
             )
         )
     else:
@@ -261,7 +263,7 @@ class BokChoyTestSuite(TestSuite):
             # Clean up data we created in the databases
             msg = colorize('green', "Cleaning up databases...")
             print msg
-            sh("./manage.py lms --settings bok_choy flush --traceback --noinput")
+            sh("./manage.py lms --settings {settings} flush --traceback --noinput".format(settings=Env.SETTINGS))
             clear_mongo()
 
     @property

pavelib/utils/test/utils.py

@@ -10,6 +10,8 @@ from paver.easy import cmdopts, sh, task
 from pavelib.utils.envs import Env
 from pavelib.utils.timer import timed
 
+from bok_choy.browser import browser
+
 MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
 MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
 MINIMUM_FIREFOX_VERSION = 28.0
@@ -78,6 +80,33 @@ def check_firefox_version():
     """
     Check that firefox is the correct version.
     """
+    if 'BOK_CHOY_HOSTNAME' in os.environ:
+        # Firefox is running in a separate Docker container; get its version via Selenium
+        driver = browser()
+        capabilities = driver.capabilities
+        if capabilities['browserName'] == 'firefox':
+            firefox_version_regex = re.compile(r'^\d+\.\d+')
+            version_key = 'browserVersion' if 'browserVersion' in 'capabilities' else 'version'
+            try:
+                firefox_ver = float(firefox_version_regex.search(capabilities[version_key]).group(0))
+            except AttributeError:
+                firefox_ver = 0.0
+        else:
+            firefox_ver = 0.0
+        driver.close()
+        if firefox_ver < MINIMUM_FIREFOX_VERSION:
+            raise Exception(
+                'Required firefox version not found.\n'
+                'Expected: {expected_version}; Actual: {actual_version}.\n\n'
+                'Make sure that the edx.devstack.firefox container is up-to-date and running\n'
+                '\t{expected_version}'.format(
+                    actual_version=firefox_ver,
+                    expected_version=MINIMUM_FIREFOX_VERSION
+                )
+            )
+        return
+
+    # Firefox will be run as a local process
     expected_firefox_ver = "Mozilla Firefox " + str(MINIMUM_FIREFOX_VERSION)
     firefox_ver_string = subprocess.check_output("firefox --version", shell=True).strip()
     firefox_version_regex = re.compile(r"Mozilla Firefox (\d+.\d+)")

requirements/edx/base.txt

@@ -154,7 +154,7 @@ sqlparse>=0.2.0,<0.3.0
 
 # Used for testing
 before_after==0.1.3
-bok-choy==0.7.0
+bok-choy==0.7.1
 chrono==1.0.2
 ddt==0.8.0
 django-crum==0.5

scripts/reset-test-db.sh

@@ -27,6 +27,14 @@ set -e
 
 DB_CACHE_DIR="common/test/db_cache"
 
+if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then
+    MYSQL_HOST=""
+    SETTINGS="bok_choy"
+else
+    MYSQL_HOST="--host=edx.devstack.mysql"
+    SETTINGS="bok_choy_docker"
+fi
+
 declare -A databases
 declare -a database_order
 databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test")
@@ -34,7 +42,7 @@ database_order=("default" "student_module_history")
 
 # Ensure the test database exists.
 for db in "${database_order[@]}"; do
-    echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql -u root
+    echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql $MYSQL_HOST -u root
 
     # Clear out the test database
     #
@@ -43,7 +51,7 @@ for db in "${database_order[@]}"; do
     # or a jenkins worker environment) that already ran tests on another commit that had
     # different migrations that created, dropped, or altered tables.
     echo "Issuing a reset_db command to the $db bok_choy MySQL database."
-    ./manage.py lms --settings bok_choy reset_db --traceback --noinput --router $db
+    ./manage.py lms --settings $SETTINGS reset_db --traceback --noinput --router $db
 
     # If there are cached database schemas/data, load them
     if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then
@@ -61,19 +69,19 @@ if [[ -z $REBUILD_CACHE ]]; then
     for db in "${database_order[@]}"; do
         # Load the schema, then the data (including the migration history)
         echo "Loading the schema from the filesystem into the $db MySQL DB."
-        mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql
+        mysql $MYSQL_HOST -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql
         echo "Loading the fixture data from the filesystem into the $db MySQL DB."
-        ./manage.py lms --settings bok_choy loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json
+        ./manage.py lms --settings $SETTINGS loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json
 
         # Migrations are stored in the default database
         echo "Loading the migration data from the filesystem into the $db MySQL DB."
-        mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
+        mysql $MYSQL_HOST -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
 
         # Re-run migrations to ensure we are up-to-date
         echo "Running the lms migrations on the $db bok_choy DB."
-        ./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput
+        ./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput
         echo "Running the cms migrations on the $db bok_choy DB."
-        ./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput
+        ./manage.py cms --settings $SETTINGS migrate --database $db --traceback --noinput
 
     done
 
@@ -86,18 +94,18 @@ else
     for db in "${database_order[@]}"; do
         # Re-run migrations on the test database
         echo "Issuing a migrate command to the $db bok_choy MySQL database for the lms django apps."
-        ./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput
+        ./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput
         echo "Issuing a migrate command to the $db bok_choy MySQL database for the cms django apps."
-        ./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput
+        ./manage.py cms --settings $SETTINGS migrate --database $db --traceback --noinput
 
         # Dump the schema and data to the cache
         echo "Using the dumpdata command to save the $db fixture data to the filesystem."
-        ./manage.py lms --settings bok_choy dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json --exclude=api_admin.Catalog
+        ./manage.py lms --settings $SETTINGS dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json --exclude=api_admin.Catalog
         echo "Saving the schema of the $db bok_choy DB to the filesystem."
-        mysqldump -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql
+        mysqldump $MYSQL_HOST -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql
 
         # dump_data does not dump the django_migrations table so we do it separately.
         echo "Saving the django_migrations table of the $db bok_choy DB to the filesystem."
-        mysqldump -u root --no-create-info "${databases["$db"]}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
+        mysqldump $MYSQL_HOST -u root --no-create-info "${databases["$db"]}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql
     done
 fi