diff --git a/cms/envs/dev.py b/cms/envs/dev.py
index 216817d66c08f1c4d5894a2dbce28c9d2dd8bba4..7ba5d46121e30935fb6124d4a9ec34c3751d12ef 100644
--- a/cms/envs/dev.py
+++ b/cms/envs/dev.py
@@ -115,7 +115,10 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'edx_location_mem_cache',
},
-
+ 'course_structure_cache': {
+ 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+ 'LOCATION': 'edx_course_structure_mem_cache',
+ },
}
# Make the keyedcache startup warnings go away
diff --git a/cms/envs/test.py b/cms/envs/test.py
index b637368c9f6b352ce893c2d3ba2bc8dc2c28908e..082960a80455c54e5da4f961b94d58f17e6d2a19 100644
--- a/cms/envs/test.py
+++ b/cms/envs/test.py
@@ -166,7 +166,9 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'edx_location_mem_cache',
},
-
+ 'course_structure_cache': {
+ 'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+ },
}
# Add external_auth to Installed apps for testing
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
index ab7f922df5f0f4b532a263155f6c4302be9851ab..bcee23b7f06303d593938ee41079dee32fc4e4f9 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
@@ -2,7 +2,11 @@
Segregation of pymongo functions from the data modeling mechanisms for split modulestore.
"""
import datetime
+import cPickle as pickle
import math
+import re
+import zlib
+from mongodb_proxy import autoretry_read, MongoProxy
import pymongo
import pytz
import re
@@ -11,6 +15,8 @@ from time import time
# Import this just to export it
from pymongo.errors import DuplicateKeyError # pylint: disable=unused-import
+from django.core.cache import get_cache, InvalidCacheBackendError
+import dogstats_wrapper as dog_stats_api
from contracts import check, new_contract
from mongodb_proxy import autoretry_read, MongoProxy
@@ -203,6 +209,40 @@ def structure_to_mongo(structure, course_context=None):
return new_structure
+class CourseStructureCache(object):
+ """
+ Wrapper around django cache object to cache course structure objects.
+ The course structures are pickled and compressed when cached.
+ """
+ def __init__(self):
+ try:
+ self.cache = get_cache('course_structure_cache')
+ except InvalidCacheBackendError:
+ self.cache = get_cache('default')
+
+ def get(self, key):
+ """Pull the compressed, pickled struct data from cache and deserialize."""
+ compressed_pickled_data = self.cache.get(key)
+ if compressed_pickled_data is None:
+ return None
+ return pickle.loads(zlib.decompress(compressed_pickled_data))
+
+ def set(self, key, structure):
+ """Given a structure, will pickle, compress, and write to cache."""
+ pickled_data = pickle.dumps(structure, pickle.HIGHEST_PROTOCOL)
+ # 1 = Fastest (slightly larger results)
+ compressed_pickled_data = zlib.compress(pickled_data, 1)
+
+ # record compressed course structure sizes
+ dog_stats_api.histogram(
+ 'compressed_course_structure.size',
+ len(compressed_pickled_data),
+ tags=[key]
+ )
+ # Stuctures are immutable, so we set a timeout of "never"
+ self.cache.set(key, compressed_pickled_data, None)
+
+
class MongoConnection(object):
"""
Segregation of pymongo functions from the data modeling mechanisms for split modulestore.
@@ -256,15 +296,23 @@ class MongoConnection(object):
def get_structure(self, key, course_context=None):
"""
- Get the structure from the persistence mechanism whose id is the given key
+ Get the structure from the persistence mechanism whose id is the given key.
+
+ This method will use a cached version of the structure if it is availble.
"""
with TIMER.timer("get_structure", course_context) as tagger_get_structure:
- with TIMER.timer("get_structure.find_one", course_context) as tagger_find_one:
- doc = self.structures.find_one({'_id': key})
- tagger_find_one.measure("blocks", len(doc['blocks']))
- tagger_get_structure.measure("blocks", len(doc['blocks']))
-
- return structure_from_mongo(doc, course_context)
+ cache = CourseStructureCache()
+
+ structure = cache.get(key)
+ tagger_get_structure.tag(from_cache='true' if structure else 'false')
+ if not structure:
+ with TIMER.timer("get_structure.find_one", course_context) as tagger_find_one:
+ doc = self.structures.find_one({'_id': key})
+ tagger_find_one.measure("blocks", len(doc['blocks']))
+ structure = structure_from_mongo(doc, course_context)
+ cache.set(key, structure)
+
+ return structure
@autoretry_read()
def find_structures_by_id(self, ids, course_context=None):
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
index b41bf268d6e8244889d6f2d86e29dfbf0876df06..e83c9be5168f7e851c9d449657a099b1fa695321 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
@@ -794,7 +794,7 @@ class TestMixedModuleStore(CommonMixedModuleStoreSetup):
# find: find parent (definition.children) 2x, find draft item, get inheritance items
# send: one delete query for specific item
# Split:
- # find: active_version & structure
+ # find: active_version & structure (cached)
# send: update structure and active_versions
@ddt.data(('draft', 4, 1), ('split', 2, 2))
@ddt.unpack
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index b8060f230d202a95a893d1a7f4e40a19260e7939..a0e3dee83f02a757fa934c992fb23fca29978136 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -94,6 +94,10 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'edx_location_mem_cache',
},
+ 'course_structure_cache': {
+ 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+ 'LOCATION': 'edx_course_structure_mem_cache',
+ },
}
diff --git a/lms/envs/test.py b/lms/envs/test.py
index ffc2013c47f81d4212a71474bb3d680b5d346d21..98038795790c736a4d8b97ce656b79e0057ad688 100644
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -207,7 +207,9 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'edx_location_mem_cache',
},
-
+ 'course_structure_cache': {
+ 'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+ },
}
# Dummy secret key for dev