From bd891c21db4d79ff11b37f65acf40c90a837004b Mon Sep 17 00:00:00 2001
From: "E. Kolpakov" <eugeny.kolpakov@gmail.com>
Date: Wed, 8 Apr 2015 17:05:36 +0300
Subject: [PATCH] Added setting to enable/disable library indexing + moved
is_index_enabled into search indexer classes Added task and signal to fire
library reindex
---
.../contentstore/courseware_index.py | 20 +++++++++----------
cms/djangoapps/contentstore/signals.py | 15 ++++++++++++--
cms/djangoapps/contentstore/tasks.py | 14 ++++++++++++-
cms/envs/aws.py | 2 +-
cms/envs/bok_choy.py | 1 +
cms/envs/common.py | 3 +++
cms/envs/devstack.py | 1 +
cms/envs/test.py | 1 +
.../lib/xmodule/xmodule/modulestore/django.py | 4 +++-
9 files changed, 46 insertions(+), 15 deletions(-)
diff --git a/cms/djangoapps/contentstore/courseware_index.py b/cms/djangoapps/contentstore/courseware_index.py
index ab522f44ca7..3a28ee90fbf 100644
--- a/cms/djangoapps/contentstore/courseware_index.py
+++ b/cms/djangoapps/contentstore/courseware_index.py
@@ -10,9 +10,6 @@ from eventtracking import tracker
from xmodule.modulestore import ModuleStoreEnum
from search.search_engine_base import SearchEngine
-from opaque_keys.edx.locator import CourseLocator, LibraryLocator
-
-
# REINDEX_AGE is the default amount of time that we look back for changes
# that might have happened. If we are provided with a time at which the
# indexing is triggered, then we know it is safe to only index items
@@ -23,13 +20,6 @@ REINDEX_AGE = timedelta(0, 60) # 60 seconds
log = logging.getLogger('edx.modulestore')
-def indexing_is_enabled():
- """
- Checks to see if the indexing feature is enabled
- """
- return settings.FEATURES.get('ENABLE_COURSEWARE_INDEX', False)
-
-
class SearchIndexingError(Exception):
""" Indicates some error(s) occured during indexing """
@@ -45,12 +35,20 @@ class SearchIndexBase(object):
INDEX_NAME = None
DOCUMENT_TYPE = None
+ ENABLE_INDEXING_KEY = None
INDEX_EVENT = {
'name': None,
'category': None
}
+ @classmethod
+ def indexing_is_enabled(cls):
+ """
+ Checks to see if the indexing feature is enabled
+ """
+ return settings.FEATURES.get(cls.ENABLE_INDEXING_KEY, False)
+
@classmethod
def _fetch_top_level(self, modulestore, structure_key):
""" Fetch the item from the modulestore location """
@@ -220,6 +218,7 @@ class SearchIndexBase(object):
class CoursewareSearchIndexer(SearchIndexBase):
INDEX_NAME = "courseware_index"
DOCUMENT_TYPE = "courseware_content"
+ ENABLE_INDEXING_KEY = 'ENABLE_COURSEWARE_INDEX'
INDEX_EVENT = {
'name': 'edx.course.index.reindexed',
@@ -247,6 +246,7 @@ class CoursewareSearchIndexer(SearchIndexBase):
class LibrarySearchIndexer(SearchIndexBase):
INDEX_NAME = "library_index"
DOCUMENT_TYPE = "library_content"
+ ENABLE_INDEXING_KEY = 'ENABLE_LIBRARY_INDEX'
INDEX_EVENT = {
'name': 'edx.library.index.reindexed',
diff --git a/cms/djangoapps/contentstore/signals.py b/cms/djangoapps/contentstore/signals.py
index df21a57ccd5..27e9bda4fea 100644
--- a/cms/djangoapps/contentstore/signals.py
+++ b/cms/djangoapps/contentstore/signals.py
@@ -5,7 +5,7 @@ from pytz import UTC
from django.dispatch import receiver
from xmodule.modulestore.django import SignalHandler
-from contentstore.courseware_index import indexing_is_enabled
+from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer
@receiver(SignalHandler.course_published)
@@ -15,5 +15,16 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=
"""
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
from .tasks import update_search_index
- if indexing_is_enabled():
+ if CoursewareSearchIndexer.indexing_is_enabled():
update_search_index.delay(unicode(course_key), datetime.now(UTC).isoformat())
+
+
+@receiver(SignalHandler.library_updated)
+def listen_for_course_publish(sender, library_key, **kwargs): # pylint: disable=unused-argument
+ """
+ Receives signal and kicks off celery task to update search index
+ """
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
+ from .tasks import update_library_index
+ if LibrarySearchIndexer.indexing_is_enabled():
+ update_library_index.delay(unicode(library_key), datetime.now(UTC).isoformat())
diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py
index cebf5aee73c..9b1942df1ea 100644
--- a/cms/djangoapps/contentstore/tasks.py
+++ b/cms/djangoapps/contentstore/tasks.py
@@ -10,7 +10,7 @@ from pytz import UTC
from django.contrib.auth.models import User
-from contentstore.courseware_index import CoursewareSearchIndexer, SearchIndexingError
+from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer, SearchIndexingError
from contentstore.utils import initialize_permissions
from course_action_state.models import CourseRerunState
from opaque_keys.edx.keys import CourseKey
@@ -98,3 +98,15 @@ def update_search_index(course_id, triggered_time_isoformat):
LOGGER.error('Search indexing error for complete course %s - %s', course_id, unicode(exc))
else:
LOGGER.debug('Search indexing successful for complete course %s', course_id)
+
+@task()
+def update_library_index(library_id, triggered_time):
+ """ Updates course search index. """
+ try:
+ library_key = CourseKey.from_string(library_id)
+ LibrarySearchIndexed.indexindex_course(modulestore(), library_key, triggered_at=triggered_time)
+
+ except SearchIndexingError as exc:
+ LOGGER.error('Search indexing error for library %s - %s', library_id, unicode(exc))
+ else:
+ LOGGER.debug('Search indexing successful for library %s', library_id)
\ No newline at end of file
diff --git a/cms/envs/aws.py b/cms/envs/aws.py
index e78bae1c411..3eb5dc99b8f 100644
--- a/cms/envs/aws.py
+++ b/cms/envs/aws.py
@@ -328,7 +328,7 @@ API_DATE_FORMAT = ENV_TOKENS.get('API_DATE_FORMAT', API_DATE_FORMAT)
# Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='}
VIDEO_CDN_URL = ENV_TOKENS.get('VIDEO_CDN_URL', {})
-if FEATURES['ENABLE_COURSEWARE_INDEX']:
+if FEATURES['ENABLE_COURSEWARE_INDEX'] or FEATURES['ENABLE_LIBRARY_INDEX']:
# Use ElasticSearch for the search engine
SEARCH_ENGINE = "search.elastic.ElasticSearchEngine"
diff --git a/cms/envs/bok_choy.py b/cms/envs/bok_choy.py
index eb5e32193b9..0f0cdb85cb8 100644
--- a/cms/envs/bok_choy.py
+++ b/cms/envs/bok_choy.py
@@ -78,6 +78,7 @@ YOUTUBE['TEST_URL'] = "127.0.0.1:{0}/test_youtube/".format(YOUTUBE_PORT)
YOUTUBE['TEXT_API']['url'] = "127.0.0.1:{0}/test_transcripts_youtube/".format(YOUTUBE_PORT)
FEATURES['ENABLE_COURSEWARE_INDEX'] = True
+FEATURES['ENABLE_LIBRARY_INDEX'] = True
SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
# Path at which to store the mock index
MOCK_SEARCH_BACKING_FILE = (
diff --git a/cms/envs/common.py b/cms/envs/common.py
index f3278b0a8b9..4654aef5b7e 100644
--- a/cms/envs/common.py
+++ b/cms/envs/common.py
@@ -140,6 +140,9 @@ FEATURES = {
# Enable the courseware search functionality
'ENABLE_COURSEWARE_INDEX': False,
+ # Enable content libraries search functionality
+ 'ENABLE_LIBRARY_INDEX': False,
+
# Enable course reruns, which will always use the split modulestore
'ALLOW_COURSE_RERUNS': True,
diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py
index 533b88bcc79..3aa7f47b000 100644
--- a/cms/envs/devstack.py
+++ b/cms/envs/devstack.py
@@ -80,6 +80,7 @@ FEATURES['ENTRANCE_EXAMS'] = True
################################ SEARCH INDEX ################################
FEATURES['ENABLE_COURSEWARE_INDEX'] = True
+FEATURES['ENABLE_LIBRARY_INDEX'] = True
SEARCH_ENGINE = "search.elastic.ElasticSearchEngine"
###############################################################################
diff --git a/cms/envs/test.py b/cms/envs/test.py
index 87b03469b42..ddc48208b45 100644
--- a/cms/envs/test.py
+++ b/cms/envs/test.py
@@ -267,6 +267,7 @@ VIDEO_CDN_URL = {
# Courseware Search Index
FEATURES['ENABLE_COURSEWARE_INDEX'] = True
+FEATURES['ENABLE_LIBRARY_INDEX'] = True
SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
# Dummy secret key for dev/test
diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py
index a6bdb6ce75a..a8c73d7f3df 100644
--- a/common/lib/xmodule/xmodule/modulestore/django.py
+++ b/common/lib/xmodule/xmodule/modulestore/django.py
@@ -80,9 +80,11 @@ class SignalHandler(object):
"""
course_published = django.dispatch.Signal(providing_args=["course_key"])
+ library_updated = django.dispatch.Signal(providing_args=["library_key"])
_mapping = {
- "course_published": course_published
+ "course_published": course_published,
+ "library_updated": library_updated
}
def __init__(self, modulestore_class):
--
GitLab