From b55362b925637ae553289a7405e93f0cdfd7792d Mon Sep 17 00:00:00 2001
From: Jason Bau <jbau@stanford.edu>
Date: Fri, 8 Aug 2014 00:50:15 -0700
Subject: [PATCH] make unit tests respect mongo port/host settings (with
default)
settings are read in from environment variable
---
cms/envs/test.py | 13 +++++++++--
.../lib/xmodule/xmodule/contentstore/mongo.py | 2 +-
.../xmodule/modulestore/tests/django_utils.py | 10 ++++++---
.../modulestore/tests/mongo_connection.py | 10 +++++++++
.../modulestore/tests/test_contentstore.py | 7 +++---
.../test_cross_modulestore_import_export.py | 6 +++--
.../tests/test_mixed_modulestore.py | 7 ++++--
.../xmodule/modulestore/tests/test_mongo.py | 11 +++++-----
.../tests/test_split_modulestore.py | 4 +++-
.../tests/test_split_w_old_mongo.py | 4 +++-
.../modulestore/tests/test_xml_importer.py | 6 +++--
lms/djangoapps/dashboard/git_import.py | 7 +++---
.../commands/tests/test_git_add_course.py | 6 +++--
.../dashboard/tests/test_sysadmin.py | 5 +++--
lms/envs/test.py | 22 +++++++++++++++++--
pavelib/utils/test/utils.py | 11 +++++++++-
16 files changed, 99 insertions(+), 32 deletions(-)
create mode 100644 common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py
diff --git a/cms/envs/test.py b/cms/envs/test.py
index 068bd39c438..8522bda6bbf 100644
--- a/cms/envs/test.py
+++ b/cms/envs/test.py
@@ -21,6 +21,12 @@ from uuid import uuid4
# import settings from LMS for consistent behavior with CMS
from lms.envs.test import (WIKI_ENABLED, PLATFORM_NAME, SITE_NAME)
+# mongo connection settings
+MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
+
+THIS_UUID = uuid4().hex[:5]
+
# Nose Test Runner
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
@@ -79,15 +85,18 @@ update_module_store_settings(
},
doc_store_settings={
'db': 'test_xmodule',
- 'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]),
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
+ 'collection': 'test_modulestore{0}'.format(THIS_UUID),
},
)
CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
'DOC_STORE_CONFIG': {
- 'host': 'localhost',
+ 'host': MONGO_HOST,
'db': 'test_xcontent',
+ 'port': MONGO_PORT_NUM,
'collection': 'dont_trip',
},
# allow for additional options that can be keyed on a name, e.g. 'trashcan'
diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py
index 287ceeb32aa..5a04bf91ee8 100644
--- a/common/lib/xmodule/xmodule/contentstore/mongo.py
+++ b/common/lib/xmodule/xmodule/contentstore/mongo.py
@@ -25,7 +25,7 @@ class MongoContentStore(ContentStore):
:param collection: ignores but provided for consistency w/ other doc_store_config patterns
"""
- logging.debug('Using MongoDB for static content serving at host={0} db={1}'.format(host, db))
+ logging.debug('Using MongoDB for static content serving at host={0} port={1} db={2}'.format(host, port, db))
_db = pymongo.database.Database(
pymongo.MongoClient(
host=host,
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
index 6a6d8843d3f..38ce3055927 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py
@@ -2,7 +2,6 @@
"""
Modulestore configuration for test cases.
"""
-
from uuid import uuid4
from django.test import TestCase
from django.contrib.auth.models import User
@@ -13,6 +12,7 @@ import datetime
import pytz
from xmodule.tabs import CoursewareTab, CourseInfoTab, StaticTab, DiscussionTab, ProgressTab, WikiTab
from xmodule.modulestore.tests.sample_courses import default_block_info_tree, TOY_BLOCK_INFO_TREE
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
def mixed_store_config(data_dir, mappings):
@@ -67,7 +67,8 @@ def draft_mongo_store_config(data_dir):
'NAME': 'draft',
'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore',
'DOC_STORE_CONFIG': {
- 'host': 'localhost',
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
},
@@ -93,7 +94,8 @@ def split_mongo_store_config(data_dir):
'NAME': 'draft',
'ENGINE': 'xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore',
'DOC_STORE_CONFIG': {
- 'host': 'localhost',
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid4().hex[:5]),
},
@@ -229,6 +231,8 @@ class ModuleStoreTestCase(TestCase):
if hasattr(module_store, '_drop_database'):
module_store._drop_database() # pylint: disable=protected-access
_CONTENTSTORE.clear()
+ if hasattr(module_store, 'close_connections'):
+ module_store.close_connections()
@classmethod
def setUpClass(cls):
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py
new file mode 100644
index 00000000000..1b18d485d66
--- /dev/null
+++ b/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py
@@ -0,0 +1,10 @@
+"""
+This file is intended to provide settings for the mongodb connection used for tests.
+The settings can be provided by environment variables in the shell running the tests. This reads
+in a variety of environment variables but provides sensible defaults in case those env var
+overrides don't exist
+"""
+import os
+
+MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py
index cda981e3298..9856730643a 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py
@@ -1,6 +1,7 @@
"""
Test contentstore.mongo functionality
"""
+import os
import logging
from uuid import uuid4
import unittest
@@ -17,12 +18,12 @@ from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
import ddt
from __builtin__ import delattr
-
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
log = logging.getLogger(__name__)
-HOST = 'localhost'
-PORT = 27017
+HOST = MONGO_HOST
+PORT = MONGO_PORT_NUM
DB = 'test_mongo_%s' % uuid4().hex[:5]
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
index 3c407eceef7..2dd62b92178 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
@@ -11,7 +11,6 @@ and then for each combination of modulestores, performing the sequence:
4) Compare all modules in the source and destination modulestores to make sure that they line up
"""
-
import ddt
import itertools
import random
@@ -28,9 +27,12 @@ from xmodule.contentstore.mongo import MongoContentStore
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.split_mongo.split_draft import DraftVersioningModuleStore
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
+
COMMON_DOCSTORE_CONFIG = {
- 'host': 'localhost'
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
}
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
index 598784b5dff..f1abe0426e1 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
@@ -15,6 +15,7 @@ from xmodule.exceptions import InvalidVersionError
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
+
# Mixed modulestore depends on django, so we'll manually configure some django settings
# before importing the module
# TODO remove this import and the configuration -- xmodule should not depend on django!
@@ -26,6 +27,7 @@ if not settings.configured:
settings.configure()
from xmodule.modulestore.mixed import MixedModuleStore
from xmodule.modulestore.draft_and_published import UnsupportedRevisionError
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
@ddt.ddt
@@ -34,8 +36,8 @@ class TestMixedModuleStore(unittest.TestCase):
Quasi-superclass which tests Location based apps against both split and mongo dbs (Locator and
Location-based dbs)
"""
- HOST = 'localhost'
- PORT = 27017
+ HOST = MONGO_HOST
+ PORT = MONGO_PORT_NUM
DB = 'test_mongo_%s' % uuid4().hex[:5]
COLLECTION = 'modulestore'
FS_ROOT = DATA_DIR
@@ -54,6 +56,7 @@ class TestMixedModuleStore(unittest.TestCase):
}
DOC_STORE_CONFIG = {
'host': HOST,
+ 'port': PORT,
'db': DB,
'collection': COLLECTION,
}
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
index 15ba65aa9e0..3cefa702036 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
@@ -36,12 +36,12 @@ from xmodule.exceptions import NotFoundError
from git.test.lib.asserts import assert_not_none
from xmodule.x_module import XModuleMixin
from xmodule.modulestore.mongo.base import as_draft
-
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
log = logging.getLogger(__name__)
-HOST = 'localhost'
-PORT = 27017
+HOST = MONGO_HOST
+PORT = MONGO_PORT_NUM
DB = 'test_mongo_%s' % uuid4().hex[:5]
COLLECTION = 'modulestore'
FS_ROOT = DATA_DIR # TODO (vshnayder): will need a real fs_root for testing load_item
@@ -91,12 +91,13 @@ class TestMongoModuleStore(unittest.TestCase):
# connect to the db
doc_store_config = {
'host': HOST,
+ 'port': PORT,
'db': DB,
'collection': COLLECTION,
}
# since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
# as well
- content_store = MongoContentStore(HOST, DB)
+ content_store = MongoContentStore(HOST, DB, port=PORT)
#
# Also test draft store imports
#
@@ -148,7 +149,7 @@ class TestMongoModuleStore(unittest.TestCase):
def test_mongo_modulestore_type(self):
store = DraftModuleStore(
None,
- {'host': HOST, 'db': DB, 'collection': COLLECTION},
+ {'host': HOST, 'db': DB, 'port': PORT, 'collection': COLLECTION},
FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS
)
assert_equals(store.get_modulestore_type(''), ModuleStoreEnum.Type.mongo)
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
index 9ede47a126e..abe634ed885 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
@@ -22,6 +22,7 @@ from xmodule.x_module import XModuleMixin
from xmodule.fields import Date, Timedelta
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from xmodule.modulestore.tests.test_modulestore import check_has_course_method
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
BRANCH_NAME_DRAFT = ModuleStoreEnum.BranchName.draft
@@ -36,8 +37,9 @@ class SplitModuleTest(unittest.TestCase):
'''
# Snippets of what would be in the django settings envs file
DOC_STORE_CONFIG = {
- 'host': 'localhost',
+ 'host': MONGO_HOST,
'db': 'test_xmodule',
+ 'port': MONGO_PORT_NUM,
'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]),
}
modulestore_options = {
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
index a2e2c2dcb78..da3c270641d 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py
@@ -9,6 +9,7 @@ from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore
from xmodule.modulestore.mongo import DraftMongoModuleStore
from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
class SplitWMongoCourseBoostrapper(unittest.TestCase):
@@ -27,7 +28,8 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase):
"""
# Snippet of what would be in the django settings envs file
db_config = {
- 'host': 'localhost',
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
'db': 'test_xmodule',
}
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py b/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
index cf4a2f971ee..4882c42734c 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
@@ -10,6 +10,7 @@ from opaque_keys.edx.locations import Location
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore.xml_importer import _import_module_and_update_references
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.tests import DATA_DIR
from uuid import uuid4
@@ -21,8 +22,8 @@ class ModuleStoreNoSettings(unittest.TestCase):
"""
A mixin to create a mongo modulestore that avoids settings
"""
- HOST = 'localhost'
- PORT = 27017
+ HOST = MONGO_HOST
+ PORT = MONGO_PORT_NUM
DB = 'test_mongo_%s' % uuid4().hex[:5]
COLLECTION = 'modulestore'
FS_ROOT = DATA_DIR
@@ -36,6 +37,7 @@ class ModuleStoreNoSettings(unittest.TestCase):
}
DOC_STORE_CONFIG = {
'host': HOST,
+ 'port': PORT,
'db': DB,
'collection': COLLECTION,
}
diff --git a/lms/djangoapps/dashboard/git_import.py b/lms/djangoapps/dashboard/git_import.py
index 9f0788d011d..1fcf322ec18 100644
--- a/lms/djangoapps/dashboard/git_import.py
+++ b/lms/djangoapps/dashboard/git_import.py
@@ -128,6 +128,7 @@ def add_repo(repo, rdir_in, branch=None):
# Set defaults even if it isn't defined in settings
mongo_db = {
'host': 'localhost',
+ 'port': 27017,
'user': '',
'password': '',
'db': 'xlog',
@@ -135,7 +136,7 @@ def add_repo(repo, rdir_in, branch=None):
# Allow overrides
if hasattr(settings, 'MONGODB_LOG'):
- for config_item in ['host', 'user', 'password', 'db', ]:
+ for config_item in ['host', 'user', 'password', 'db', 'port']:
mongo_db[config_item] = settings.MONGODB_LOG.get(
config_item, mongo_db[config_item])
@@ -258,13 +259,13 @@ def add_repo(repo, rdir_in, branch=None):
cwd=os.path.abspath(cdir)))
# store import-command-run output in mongo
- mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
+ mongouri = 'mongodb://{user}:{password}@{host}:{port}/{db}'.format(**mongo_db)
try:
if mongo_db['user'] and mongo_db['password']:
mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
else:
- mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
+ mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'], port=mongo_db['port'])
except mongoengine.connection.ConnectionError:
log.exception('Unable to connect to mongodb to save log, please '
'check MONGODB_LOG settings')
diff --git a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py
index a64a9c89cb1..06cf8984ffe 100644
--- a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py
+++ b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py
@@ -1,7 +1,6 @@
"""
Provide tests for git_add_course management command.
"""
-
import logging
import os
import shutil
@@ -21,9 +20,12 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
import dashboard.git_import as git_import
from dashboard.git_import import GitImportError
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
+
TEST_MONGODB_LOG = {
- 'host': 'localhost',
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
'user': '',
'password': '',
'db': 'test_xlog',
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 6f75c7a961c..091f4525531 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -1,7 +1,6 @@
"""
Provide tests for sysadmin dashboard feature in sysadmin.py
"""
-
import glob
import os
import re
@@ -30,10 +29,12 @@ from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.xml import XMLModuleStore
from opaque_keys.edx.locations import SlashSeparatedCourseKey
+from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
TEST_MONGODB_LOG = {
- 'host': 'localhost',
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
'user': '',
'password': '',
'db': 'test_xlog',
diff --git a/lms/envs/test.py b/lms/envs/test.py
index 62b50022392..62f540007cc 100644
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -18,8 +18,14 @@ from path import path
from warnings import filterwarnings, simplefilter
from uuid import uuid4
+# mongo connection settings
+MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
+
os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = 'localhost:8000-9000'
+THIS_UUID = uuid4().hex[:5]
+
# can't test start dates with this True, but on the other hand,
# can test everything else :)
FEATURES['DISABLE_START_DATES'] = True
@@ -118,16 +124,19 @@ update_module_store_settings(
'data_dir': COMMON_TEST_DATA_ROOT,
},
doc_store_settings={
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
'db': 'test_xmodule',
- 'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]),
+ 'collection': 'test_modulestore{0}'.format(THIS_UUID),
},
)
CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
'DOC_STORE_CONFIG': {
- 'host': 'localhost',
+ 'host': MONGO_HOST,
'db': 'xcontent',
+ 'port': MONGO_PORT_NUM,
}
}
@@ -338,3 +347,12 @@ VERIFY_STUDENT["SOFTWARE_SECURE"] = {
VIDEO_CDN_URL = {
'CN': 'http://api.xuetangx.com/edx/video?s3_url='
}
+
+######### dashboard git log settings #########
+MONGODB_LOG = {
+ 'host': MONGO_HOST,
+ 'port': MONGO_PORT_NUM,
+ 'user': '',
+ 'password': '',
+ 'db': 'xlog',
+}
diff --git a/pavelib/utils/test/utils.py b/pavelib/utils/test/utils.py
index 82ec70915af..233cd0c9ff8 100644
--- a/pavelib/utils/test/utils.py
+++ b/pavelib/utils/test/utils.py
@@ -3,6 +3,11 @@ Helper functions for test tasks
"""
from paver.easy import sh, task
from pavelib.utils.envs import Env
+import os
+
+MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
+
__test__ = False # do not collect
@@ -42,4 +47,8 @@ def clean_mongo():
"""
Clean mongo test databases
"""
- sh("mongo {repo_root}/scripts/delete-mongo-test-dbs.js".format(repo_root=Env.REPO_ROOT))
+ sh("mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format(
+ host=MONGO_HOST,
+ port=MONGO_PORT_NUM,
+ repo_root=Env.REPO_ROOT,
+ ))
--
GitLab