Merge pull request #5819 from edx/jeskew/asset_mongo_coursewide

Implement course-wide asset paging methods.

common/lib/xmodule/xmodule/assetstore/__init__.py

@@ -5,6 +5,7 @@ Classes representing asset & asset thumbnail metadata.
 from datetime import datetime
 import pytz
 from contracts import contract, new_contract
+from bisect import bisect_left, bisect_right
 from opaque_keys.edx.keys import CourseKey, AssetKey
 
 new_contract('AssetKey', AssetKey)
@@ -33,8 +34,8 @@ class AssetMetadata(object):
     # All AssetMetadata objects should have AssetLocators with this type.
     ASSET_TYPE = 'asset'
 
-    @contract(asset_id='AssetKey', basename='basestring | None', internal_name='str | None', locked='bool | None', contenttype='basestring | None',
-              md5='str | None', curr_version='str | None', prev_version='str | None', edited_by='int | None', edited_on='datetime | None')
+    @contract(asset_id='AssetKey', basename='basestring|None', internal_name='basestring|None', locked='bool|None', contenttype='basestring|None',
+              md5='basestring|None', curr_version='basestring|None', prev_version='basestring|None', edited_by='int|None', edited_on='datetime|None')
     def __init__(self, asset_id,
                  basename=None, internal_name=None,
                  locked=None, contenttype=None, md5=None,
@@ -99,15 +100,13 @@ class AssetMetadata(object):
             'locked': self.locked,
             'contenttype': self.contenttype,
             'md5': self.md5,
-            'edit_info': {
-                'curr_version': self.curr_version,
-                'prev_version': self.prev_version,
-                'edited_by': self.edited_by,
-                'edited_on': self.edited_on
-            }
+            'curr_version': self.curr_version,
+            'prev_version': self.prev_version,
+            'edited_by': self.edited_by,
+            'edited_on': self.edited_on
         }
 
-    @contract(asset_doc='dict | None')
+    @contract(asset_doc='dict|None')
     def from_mongo(self, asset_doc):
         """
         Fill in all metadata fields from a MongoDB document.
@@ -121,11 +120,10 @@ class AssetMetadata(object):
         self.locked = asset_doc['locked']
         self.contenttype = asset_doc['contenttype']
         self.md5 = asset_doc['md5']
-        edit_info = asset_doc['edit_info']
-        self.curr_version = edit_info['curr_version']
-        self.prev_version = edit_info['prev_version']
-        self.edited_by = edit_info['edited_by']
-        self.edited_on = edit_info['edited_on']
+        self.curr_version = asset_doc['curr_version']
+        self.prev_version = asset_doc['prev_version']
+        self.edited_by = asset_doc['edited_by']
+        self.edited_on = asset_doc['edited_on']
 
 
 class AssetThumbnailMetadata(object):
@@ -136,7 +134,7 @@ class AssetThumbnailMetadata(object):
     # All AssetThumbnailMetadata objects should have AssetLocators with this type.
     ASSET_TYPE = 'thumbnail'
 
-    @contract(asset_id='AssetKey', internal_name='str | unicode | None')
+    @contract(asset_id='AssetKey', internal_name='basestring|None')
     def __init__(self, asset_id, internal_name=None, field_decorator=None):
         """
         Construct a AssetThumbnailMetadata object.
@@ -162,7 +160,7 @@ class AssetThumbnailMetadata(object):
             'internal_name': self.internal_name
         }
 
-    @contract(thumbnail_doc='dict | None')
+    @contract(thumbnail_doc='dict|None')
     def from_mongo(self, thumbnail_doc):
         """
         Fill in all metadata fields from a MongoDB document.

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -14,6 +14,8 @@ import collections
 from contextlib import contextmanager
 import functools
 import threading
+from operator import itemgetter
+from sortedcontainers import SortedListWithKey
 
 from abc import ABCMeta, abstractmethod
 from contracts import contract, new_contract
@@ -98,6 +100,13 @@ class ModuleStoreEnum(object):
         # user ID to use for tests that do not have a django user available
         test = -3
 
+    class SortOrder(object):
+        """
+        Values for sorting asset metadata.
+        """
+        ascending = 1
+        descending = 2
+
 
 class BulkOpsRecord(object):
     """
@@ -292,19 +301,23 @@ class ModuleStoreAssetInterface(object):
         if course_assets is None:
             return None, None
 
-        if get_thumbnail:
-            all_assets = course_assets['thumbnails']
-        else:
-            all_assets = course_assets['assets']
+        info = 'thumbnails' if get_thumbnail else 'assets'
+        all_assets = SortedListWithKey([], key=itemgetter('filename'))
+        # Assets should be pre-sorted, so add them efficiently without sorting.
+        # extend() will raise a ValueError if the passed-in list is not sorted.
+        all_assets.extend(course_assets.get(info, []))
 
         # See if this asset already exists by checking the external_filename.
         # Studio doesn't currently support using multiple course assets with the same filename.
         # So use the filename as the unique identifier.
-        for idx, asset in enumerate(all_assets):
-            if asset['filename'] == filename:
-                return course_assets, idx
+        idx = None
+        idx_left = all_assets.bisect_left({'filename': filename})
+        idx_right = all_assets.bisect_right({'filename': filename})
+        if idx_left != idx_right:
+            # Asset was found in the list.
+            idx = idx_left
 
-        return course_assets, None
+        return course_assets, idx
 
     @contract(asset_key='AssetKey')
     def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
@@ -358,19 +371,19 @@ class ModuleStoreAssetInterface(object):
         """
         return self._find_asset_info(asset_key, thumbnail=True, **kwargs)
 
-    @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None', get_thumbnails='bool')
-    def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
+    @contract(course_key='CourseKey', start='int|None', maxresults='int|None',
+              sort='tuple(str,(int,>=1,<=2))|None', get_thumbnails='bool')
+    def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1,
+                                sort=('displayname', ModuleStoreEnum.SortOrder.ascending),
+                                get_thumbnails=False, **kwargs):
         """
-        Returns a list of static asset (or thumbnail) metadata for a course.
-
-        Args:
             course_key (CourseKey): course identifier
-            start (int): optional - start at this asset number
+            start (int): optional - start at this asset number. Zero-based!
             maxresults (int): optional - return at most this many, -1 means no limit
             sort (array): optional - None means no sort
                 (sort_by (str), sort_order (str))
                 sort_by - one of 'uploadDate' or 'displayname'
-                sort_order - one of 'ascending' or 'descending'
+                sort_order - one of SortOrder.ascending or SortOrder.descending
             get_thumbnails (bool): True if getting thumbnail metadata, else getting asset metadata
 
         Returns:
@@ -382,35 +395,60 @@ class ModuleStoreAssetInterface(object):
             # to distinguish zero assets from "not able to retrieve assets".
             return None
 
-        if get_thumbnails:
-            all_assets = course_assets.get('thumbnails', [])
-        else:
-            all_assets = course_assets.get('assets', [])
-
-        # DO_NEXT: Add start/maxresults/sort functionality as part of https://openedx.atlassian.net/browse/PLAT-74
-        if start and maxresults and sort:
-            pass
+        # Determine the proper sort - with defaults of ('displayname', SortOrder.ascending).
+        sort_field = 'filename'
+        sort_order = ModuleStoreEnum.SortOrder.ascending
+        if sort:
+            if sort[0] == 'uploadDate' and not get_thumbnails:
+                sort_field = 'edited_on'
+            if sort[1] == ModuleStoreEnum.SortOrder.descending:
+                sort_order = ModuleStoreEnum.SortOrder.descending
+
+        info = 'thumbnails' if get_thumbnails else 'assets'
+        all_assets = SortedListWithKey(course_assets.get(info, []), key=itemgetter(sort_field))
+        num_assets = len(all_assets)
+
+        start_idx = start
+        end_idx = min(num_assets, start + maxresults)
+        if maxresults < 0:
+            # No limit on the results.
+            end_idx = num_assets
+
+        step_incr = 1
+        if sort_order == ModuleStoreEnum.SortOrder.descending:
+            # Flip the indices and iterate backwards.
+            step_incr = -1
+            start_idx = (num_assets - 1) - start_idx
+            end_idx = (num_assets - 1) - end_idx
 
         ret_assets = []
-        for asset in all_assets:
+        for idx in xrange(start_idx, end_idx, step_incr):
+            asset = all_assets[idx]
             if get_thumbnails:
                 thumb = AssetThumbnailMetadata(
                     course_key.make_asset_key('thumbnail', asset['filename']),
-                    internal_name=asset['filename'], **kwargs
+                    internal_name=asset['filename'],
+                    **kwargs
                 )
                 ret_assets.append(thumb)
             else:
-                asset = AssetMetadata(
+                new_asset = AssetMetadata(
                     course_key.make_asset_key('asset', asset['filename']),
                     basename=asset['filename'],
-                    edited_on=asset['edit_info']['edited_on'],
+                    internal_name=asset['internal_name'],
+                    locked=asset['locked'],
                     contenttype=asset['contenttype'],
-                    md5=str(asset['md5']), **kwargs
+                    md5=asset['md5'],
+                    curr_version=asset['curr_version'],
+                    prev_version=asset['prev_version'],
+                    edited_on=asset['edited_on'],
+                    edited_by=asset['edited_by'],
+                    **kwargs
                 )
-                ret_assets.append(asset)
+                ret_assets.append(new_asset)
         return ret_assets
 
-    @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None')
+    @contract(course_key='CourseKey', start='int|None', maxresults='int|None', sort='tuple(str,int)|None')
     def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
         """
         Returns a list of static assets for a course.
@@ -423,7 +461,7 @@ class ModuleStoreAssetInterface(object):
             sort (array): optional - None means no sort
                 (sort_by (str), sort_order (str))
                 sort_by - one of 'uploadDate' or 'displayname'
-                sort_order - one of 'ascending' or 'descending'
+                sort_order - one of SortOrder.ascending or SortOrder.descending
 
         Returns:
             List of AssetMetadata objects.

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -370,7 +370,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
         return store.find_asset_thumbnail_metadata(asset_key, **kwargs)
 
     @strip_key
-    @contract(course_key='CourseKey', start=int, maxresults=int, sort='list | None')
+    @contract(course_key='CourseKey', start=int, maxresults=int, sort='tuple|None')
     def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
         """
         Returns a list of static assets for a course.

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -25,6 +25,8 @@ from path import path
 from datetime import datetime
 from pytz import UTC
 from contracts import contract, new_contract
+from operator import itemgetter
+from sortedcontainers import SortedListWithKey
 
 from importlib import import_module
 from xmodule.errortracker import null_error_tracker, exc_info_to_str
@@ -1493,7 +1495,10 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
 
         course_assets, asset_idx = self._find_course_asset(course_key, asset_metadata.asset_id.path, thumbnail)
         info = 'thumbnails' if thumbnail else 'assets'
-        all_assets = course_assets[info]
+        all_assets = SortedListWithKey([], key=itemgetter('filename'))
+        # Assets should be pre-sorted, so add them efficiently without sorting.
+        # extend() will raise a ValueError if the passed-in list is not sorted.
+        all_assets.extend(course_assets[info])
 
         # Set the edited information for assets only - not thumbnails.
         if not thumbnail:
@@ -1502,17 +1507,38 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         # Translate metadata to Mongo format.
         metadata_to_insert = asset_metadata.to_mongo()
         if asset_idx is None:
-            # Append new metadata.
-            # Future optimization: Insert in order & binary search to retrieve.
-            all_assets.append(metadata_to_insert)
+            # Add new metadata sorted into the list.
+            all_assets.add(metadata_to_insert)
         else:
             # Replace existing metadata.
             all_assets[asset_idx] = metadata_to_insert
 
         # Update the document.
-        self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets}})
+        self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets.as_list()}})
         return True
 
+    @contract(source_course_key='CourseKey', dest_course_key='CourseKey')
+    def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
+        """
+        Copy all the course assets from source_course_key to dest_course_key.
+
+        Arguments:
+            source_course_key (CourseKey): identifier of course to copy from
+            dest_course_key (CourseKey): identifier of course to copy to
+        """
+        source_assets = self._find_course_assets(source_course_key)
+        dest_assets = self._find_course_assets(dest_course_key)
+        dest_assets['assets'] = source_assets.get('assets', [])
+        dest_assets['thumbnails'] = source_assets.get('thumbnails', [])
+
+        # Update the document.
+        self.asset_collection.update(
+            {'_id': dest_assets['_id']},
+            {'$set': {'assets': dest_assets['assets'],
+                      'thumbnails': dest_assets['thumbnails']}
+             }
+        )
+
     @contract(asset_key='AssetKey', attr_dict=dict)
     def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id):
         """

common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py

@@ -12,7 +12,7 @@ from xmodule.modulestore import ModuleStoreEnum
 
 from xmodule.modulestore.tests.factories import CourseFactory
 from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
-    MODULESTORE_SETUPS, MongoContentstoreBuilder, XmlModulestoreBuilder, MixedModulestoreBuilder
+    MODULESTORE_SETUPS, MongoContentstoreBuilder, XmlModulestoreBuilder, MixedModulestoreBuilder, MongoModulestoreBuilder
 )
 
 
@@ -63,88 +63,59 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
         Setup assets. Save in store if given
         """
         asset_fields = ('filename', 'internal_name', 'basename', 'locked', 'edited_by', 'edited_on', 'curr_version', 'prev_version')
-        asset1_vals = ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '14', '13')
-        asset2_vals = ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '1', None)
-        asset3_vals = ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, ModuleStoreEnum.UserID.test * 2, datetime.now(pytz.utc), 'AB', 'AA')
-        asset4_vals = ('dog.png', 'PUPY4242X', 'pictures/animals', True, ModuleStoreEnum.UserID.test * 3, datetime.now(pytz.utc), '5', '4')
-        asset5_vals = ('not_here.txt', 'JJJCCC747', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49')
-
-        asset1 = dict(zip(asset_fields[1:], asset1_vals[1:]))
-        asset2 = dict(zip(asset_fields[1:], asset2_vals[1:]))
-        asset3 = dict(zip(asset_fields[1:], asset3_vals[1:]))
-        asset4 = dict(zip(asset_fields[1:], asset4_vals[1:]))
-        non_existent_asset = dict(zip(asset_fields[1:], asset5_vals[1:]))
-
-        # Asset6 and thumbnail6 have equivalent information on purpose.
-        asset6_vals = ('asset.txt', 'JJJCCC747858', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49')
-        asset6 = dict(zip(asset_fields[1:], asset6_vals[1:]))
-
-        asset1_key = course1_key.make_asset_key('asset', asset1_vals[0])
-        asset2_key = course1_key.make_asset_key('asset', asset2_vals[0])
-        asset3_key = course2_key.make_asset_key('asset', asset3_vals[0])
-        asset4_key = course2_key.make_asset_key('asset', asset4_vals[0])
-        asset5_key = course2_key.make_asset_key('asset', asset5_vals[0])
-        asset6_key = course2_key.make_asset_key('asset', asset6_vals[0])
-
-        asset1_md = AssetMetadata(asset1_key, **asset1)
-        asset2_md = AssetMetadata(asset2_key, **asset2)
-        asset3_md = AssetMetadata(asset3_key, **asset3)
-        asset4_md = AssetMetadata(asset4_key, **asset4)
-        asset5_md = AssetMetadata(asset5_key, **non_existent_asset)
-        asset6_md = AssetMetadata(asset6_key, **asset6)
-
-        if store is not None:
-            store.save_asset_metadata(course1_key, asset1_md, ModuleStoreEnum.UserID.test)
-            store.save_asset_metadata(course1_key, asset2_md, ModuleStoreEnum.UserID.test)
-            store.save_asset_metadata(course2_key, asset3_md, ModuleStoreEnum.UserID.test)
-            store.save_asset_metadata(course2_key, asset4_md, ModuleStoreEnum.UserID.test)
-            # 5 & 6 are not saved on purpose!
-
-        return (asset1_md, asset2_md, asset3_md, asset4_md, asset5_md, asset6_md)
+        all_asset_data = (
+            ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '14', '13'),
+            ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '1', None),
+            ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, ModuleStoreEnum.UserID.test * 2, datetime.now(pytz.utc), 'AB', 'AA'),
+            ('dog.png', 'PUPY4242X', 'pictures/animals', True, ModuleStoreEnum.UserID.test * 3, datetime.now(pytz.utc), '5', '4'),
+            ('not_here.txt', 'JJJCCC747', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49'),
+            ('asset.txt', 'JJJCCC747858', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49'),
+            ('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, ModuleStoreEnum.UserID.test * 7, datetime.now(pytz.utc), '1.1', '1.01'),
+            ('weather_patterns.bmp', '928SJXX2EB', 'science', False, ModuleStoreEnum.UserID.test * 8, datetime.now(pytz.utc), '52', '51'),
+            ('demo.swf', 'DFDFGGGG14', 'demos/easy', False, ModuleStoreEnum.UserID.test * 9, datetime.now(pytz.utc), '5', '4'),
+        )
+
+        for i, asset in enumerate(all_asset_data):
+            asset_dict = dict(zip(asset_fields[1:], asset[1:]))
+            if i in (0, 1) and course1_key:
+                asset_key = course1_key.make_asset_key('asset', asset[0])
+                asset_md = AssetMetadata(asset_key, **asset_dict)
+                if store is not None:
+                    store.save_asset_metadata(course1_key, asset_md, asset[4])
+            elif course2_key:
+                asset_key = course2_key.make_asset_key('asset', asset[0])
+                asset_md = AssetMetadata(asset_key, **asset_dict)
+                # Don't save assets 5 and 6.
+                if store is not None and i not in (4, 5):
+                    store.save_asset_metadata(course2_key, asset_md, asset[4])
 
     def setup_thumbnails(self, course1_key, course2_key, store=None):
         """
         Setup thumbs. Save in store if given
         """
         thumbnail_fields = ('filename', 'internal_name')
-        thumbnail1_vals = ('cat_thumb.jpg', 'XYXYXYXYXYXY')
-        thumbnail2_vals = ('kitten_thumb.jpg', '123ABC123ABC')
-        thumbnail3_vals = ('puppy_thumb.jpg', 'ADAM12ADAM12')
-        thumbnail4_vals = ('meerkat_thumb.jpg', 'CHIPSPONCH14')
-        thumbnail5_vals = ('corgi_thumb.jpg', 'RON8LDXFFFF10')
-
-        thumbnail1 = dict(zip(thumbnail_fields[1:], thumbnail1_vals[1:]))
-        thumbnail2 = dict(zip(thumbnail_fields[1:], thumbnail2_vals[1:]))
-        thumbnail3 = dict(zip(thumbnail_fields[1:], thumbnail3_vals[1:]))
-        thumbnail4 = dict(zip(thumbnail_fields[1:], thumbnail4_vals[1:]))
-        non_existent_thumbnail = dict(zip(thumbnail_fields[1:], thumbnail5_vals[1:]))
-
-        # Asset6 and thumbnail6 have equivalent information on purpose.
-        thumbnail6_vals = ('asset.txt', 'JJJCCC747858')
-        thumbnail6 = dict(zip(thumbnail_fields[1:], thumbnail6_vals[1:]))
-
-        thumb1_key = course1_key.make_asset_key('thumbnail', thumbnail1_vals[0])
-        thumb2_key = course1_key.make_asset_key('thumbnail', thumbnail2_vals[0])
-        thumb3_key = course2_key.make_asset_key('thumbnail', thumbnail3_vals[0])
-        thumb4_key = course2_key.make_asset_key('thumbnail', thumbnail4_vals[0])
-        thumb5_key = course2_key.make_asset_key('thumbnail', thumbnail5_vals[0])
-        thumb6_key = course2_key.make_asset_key('thumbnail', thumbnail6_vals[0])
-
-        thumb1_md = AssetThumbnailMetadata(thumb1_key, **thumbnail1)
-        thumb2_md = AssetThumbnailMetadata(thumb2_key, **thumbnail2)
-        thumb3_md = AssetThumbnailMetadata(thumb3_key, **thumbnail3)
-        thumb4_md = AssetThumbnailMetadata(thumb4_key, **thumbnail4)
-        thumb5_md = AssetThumbnailMetadata(thumb5_key, **non_existent_thumbnail)
-        thumb6_md = AssetThumbnailMetadata(thumb6_key, **thumbnail6)
-
-        if store is not None:
-            store.save_asset_thumbnail_metadata(course1_key, thumb1_md, ModuleStoreEnum.UserID.test)
-            store.save_asset_thumbnail_metadata(course1_key, thumb2_md, ModuleStoreEnum.UserID.test)
-            store.save_asset_thumbnail_metadata(course2_key, thumb3_md, ModuleStoreEnum.UserID.test)
-            store.save_asset_thumbnail_metadata(course2_key, thumb4_md, ModuleStoreEnum.UserID.test)
-            # thumb5 and thumb6 are not saved on purpose!
-
-        return (thumb1_md, thumb2_md, thumb3_md, thumb4_md, thumb5_md, thumb6_md)
+        all_thumbnail_data = (
+            ('cat_thumb.jpg', 'XYXYXYXYXYXY'),
+            ('kitten_thumb.jpg', '123ABC123ABC'),
+            ('puppy_thumb.jpg', 'ADAM12ADAM12'),
+            ('meerkat_thumb.jpg', 'CHIPSPONCH14'),
+            ('corgi_thumb.jpg', 'RON8LDXFFFF10'),
+        )
+
+        for i, thumb in enumerate(all_thumbnail_data):
+            thumb_dict = dict(zip(thumbnail_fields[1:], thumb[1:]))
+            if i in (0, 1) and course1_key:
+                thumb_key = course1_key.make_asset_key('thumbnail', thumb[0])
+                thumb_md = AssetThumbnailMetadata(thumb_key, **thumb_dict)
+                if store is not None:
+                    store.save_asset_thumbnail_metadata(course1_key, thumb_md, ModuleStoreEnum.UserID.test)
+            elif course2_key:
+                thumb_key = course2_key.make_asset_key('thumbnail', thumb[0])
+                thumb_md = AssetThumbnailMetadata(thumb_key, **thumb_dict)
+                # Don't save assets 5 and 6.
+                if store is not None and i not in (4, 5):
+                    store.save_asset_thumbnail_metadata(course2_key, thumb_md, ModuleStoreEnum.UserID.test)
+
 
     @ddt.data(*MODULESTORE_SETUPS)
     def test_save_one_and_confirm(self, storebuilder):
@@ -387,11 +358,67 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
                 store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
                 self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 0)
 
-    def test_get_all_assets_with_paging(self):
-        pass
-
-    def test_copy_all_assets(self):
-        pass
+    @ddt.data(*MODULESTORE_SETUPS)
+    def test_get_all_assets_with_paging(self, storebuilder):
+        """
+        Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
+        """
+        # Temporarily only perform this test for Old Mongo - not Split.
+        if not isinstance(storebuilder, MongoModulestoreBuilder):
+            raise unittest.SkipTest
+        with MongoContentstoreBuilder().build() as contentstore:
+            with storebuilder.build(contentstore) as store:
+                course1 = CourseFactory.create(modulestore=store)
+                course2 = CourseFactory.create(modulestore=store)
+                self.setup_assets(course1.id, course2.id, store)
+
+                expected_sorts_by_2 = (
+                    (
+                        ('displayname', ModuleStoreEnum.SortOrder.ascending),
+                        ('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'),
+                        (2, 2, 1)
+                    ),
+                    (
+                        ('displayname', ModuleStoreEnum.SortOrder.descending),
+                        ('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'),
+                        (2, 2, 1)
+                    ),
+                    (
+                        ('uploadDate', ModuleStoreEnum.SortOrder.ascending),
+                        ('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'),
+                        (2, 2, 1)
+                    ),
+                    (
+                        ('uploadDate', ModuleStoreEnum.SortOrder.descending),
+                        ('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'),
+                        (2, 2, 1)
+                    ),
+                )
+                # First, with paging across all sorts.
+                for sort_test in expected_sorts_by_2:
+                    for i in xrange(3):
+                        asset_page = store.get_all_asset_metadata(course2.id, start=2 * i, maxresults=2, sort=sort_test[0])
+                        self.assertEquals(len(asset_page), sort_test[2][i])
+                        self.assertEquals(asset_page[0].asset_id.path, sort_test[1][2 * i])
+                        if sort_test[2][i] == 2:
+                            self.assertEquals(asset_page[1].asset_id.path, sort_test[1][(2 * i) + 1])
+
+                # Now fetch everything.
+                asset_page = store.get_all_asset_metadata(course2.id, start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending))
+                self.assertEquals(len(asset_page), 5)
+                self.assertEquals(asset_page[0].asset_id.path, 'code.tgz')
+                self.assertEquals(asset_page[1].asset_id.path, 'demo.swf')
+                self.assertEquals(asset_page[2].asset_id.path, 'dog.png')
+                self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf')
+                self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp')
+
+                # Some odd conditions.
+                asset_page = store.get_all_asset_metadata(course2.id, start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending))
+                self.assertEquals(len(asset_page), 0)
+                asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending))
+                self.assertEquals(len(asset_page), 0)
+                asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=-12345, sort=('displayname', ModuleStoreEnum.SortOrder.descending))
+                self.assertEquals(len(asset_page), 2)
 
     @ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())]))
     def test_xml_not_yet_implemented(self, storebuilder):
@@ -410,3 +437,30 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
             for method in ['_get_all_asset_metadata', 'get_all_asset_metadata', 'get_all_asset_thumbnail_metadata']:
                 with self.assertRaises(NotImplementedError):
                     getattr(store, method)(course_key)
+
+    @ddt.data(*MODULESTORE_SETUPS)
+    def test_copy_all_assets(self, storebuilder):
+        """
+        Create a course with assets and such, copy it all to another course, and check on it.
+        """
+        with MongoContentstoreBuilder().build() as contentstore:
+            with storebuilder.build(contentstore) as store:
+                course1 = CourseFactory.create(modulestore=store)
+                course2 = CourseFactory.create(modulestore=store)
+                self.setup_assets(course1.id, None, store)
+                self.setup_thumbnails(course1.id, None, store)
+                self.assertEquals(len(store.get_all_asset_metadata(course1.id)), 2)
+                self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course1.id)), 2)
+                self.assertEquals(len(store.get_all_asset_metadata(course2.id)), 0)
+                self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course2.id)), 0)
+                store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
+                self.assertEquals(len(store.get_all_asset_metadata(course1.id)), 2)
+                self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course1.id)), 2)
+                all_assets = store.get_all_asset_metadata(course2.id, sort=('displayname', ModuleStoreEnum.SortOrder.ascending))
+                self.assertEquals(len(all_assets), 2)
+                self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
+                self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
+                all_thumbnails = store.get_all_asset_thumbnail_metadata(course2.id, sort=('uploadDate', ModuleStoreEnum.SortOrder.descending))
+                self.assertEquals(len(all_thumbnails), 2)
+                self.assertEquals(all_thumbnails[0].asset_id.path, 'kitten_thumb.jpg')
+                self.assertEquals(all_thumbnails[1].asset_id.path, 'cat_thumb.jpg')

requirements/edx/base.txt

@@ -76,6 +76,7 @@ scipy==0.14.0
 Shapely==1.2.16
 singledispatch==3.4.0.2
 sorl-thumbnail==11.12
+sortedcontainers==0.9.2
 South==0.7.6
 stevedore==0.14.1
 sure==1.2.3