From 014cb9e564e04977ae4852167f923ac10116e898 Mon Sep 17 00:00:00 2001
From: Calen Pennington <cale@edx.org>
Date: Wed, 30 Apr 2014 10:17:27 -0400
Subject: [PATCH] Make course ids and usage ids opaque to LMS and Studio
[partial commit]
This commit is the base library for OpaqueKeys.
These keys are now objects with a limited interface, and the particular
internal representation is managed by the data storage layer (the
modulestore).
For the LMS, there should be no outward-facing changes to the system.
The keys are, for now, a change to internal representation only. For
Studio, the new serialized form of the keys is used in urls, to allow
for further migration in the future.
Co-Author: Andy Armstrong <andya@edx.org>
Co-Author: Christina Roberts <christina@edx.org>
Co-Author: David Baumgold <db@edx.org>
Co-Author: Diana Huang <dkh@edx.org>
Co-Author: Don Mitchell <dmitchell@edx.org>
Co-Author: Julia Hansbrough <julia@edx.org>
Co-Author: Nimisha Asthagiri <nasthagiri@edx.org>
Co-Author: Sarina Canelake <sarina@edx.org>
[LMS-2370]
---
.../lib/opaque_keys/opaque_keys/__init__.py | 247 ++++++++++++++++++
.../opaque_keys/opaque_keys/tests/__init__.py | 0
.../opaque_keys/tests/test_opaque_keys.py | 182 +++++++++++++
common/lib/opaque_keys/setup.py | 19 ++
requirements/edx/base.txt | 1 +
requirements/edx/local.txt | 1 +
6 files changed, 450 insertions(+)
create mode 100644 common/lib/opaque_keys/opaque_keys/__init__.py
create mode 100644 common/lib/opaque_keys/opaque_keys/tests/__init__.py
create mode 100644 common/lib/opaque_keys/opaque_keys/tests/test_opaque_keys.py
create mode 100644 common/lib/opaque_keys/setup.py
diff --git a/common/lib/opaque_keys/opaque_keys/__init__.py b/common/lib/opaque_keys/opaque_keys/__init__.py
new file mode 100644
index 00000000000..3a33add8a14
--- /dev/null
+++ b/common/lib/opaque_keys/opaque_keys/__init__.py
@@ -0,0 +1,247 @@
+"""
+Defines OpaqueKey class, to be used as the base-class for
+implementing pluggable OpaqueKeys.
+"""
+from abc import ABCMeta, abstractmethod, abstractproperty
+from copy import deepcopy
+from collections import namedtuple
+from functools import total_ordering
+
+from stevedore.enabled import EnabledExtensionManager
+
+
+class InvalidKeyError(Exception):
+ """
+ Raised to indicated that a serialized key isn't valid (wasn't able to be parsed
+ by any available providers).
+ """
+ def __init__(self, key_class, serialized):
+ super(InvalidKeyError, self).__init__(u'{}: {}'.format(key_class, serialized))
+
+
+class OpaqueKeyMetaclass(ABCMeta):
+ """
+ Metaclass for OpaqueKeys. Automatically derives the class from a namedtuple
+ with a fieldset equal to the KEY_FIELDS class attribute, if KEY_FIELDS is set.
+ """
+ def __new__(mcs, name, bases, attrs):
+ if '__slots__' not in attrs:
+ for field in attrs.get('KEY_FIELDS', []):
+ attrs.setdefault(field, None)
+ return super(OpaqueKeyMetaclass, mcs).__new__(mcs, name, bases, attrs)
+
+
+@total_ordering
+class OpaqueKey(object):
+ """
+ A base-class for implementing pluggable opaque keys. Individual key subclasses identify
+ particular types of resources, without specifying the actual form of the key (or
+ its serialization).
+
+ There are two levels of expected subclasses: Key type definitions, and key implementations
+
+ OpaqueKey
+ |
+ KeyType
+ |
+ KeyImplementation
+
+ The KeyType base class must define the class property KEY_TYPE, which identifies
+ which entry_point namespace the keys implementations should be registered with.
+
+ The KeyImplementation classes must define CANONICAL_NAMESPACE and KEY_FIELDS.
+ CANONICAL_NAMESPACE: Identifies the key namespace for the particular
+ key_implementation (when serializing). KeyImplementations must be
+ registered using the CANONICAL_NAMESPACE is their entry_point name,
+ but can also be registered with other names for backwards compatibility.
+ KEY_FIELDS: A list of attribute names that will be used to establish object
+ identity. KeyImplementation instances will compare equal iff all of
+ their KEY_FIELDS match, and will not compare equal to instances
+ of different KeyImplementation classes (even if the KEY_FIELDS match).
+ These fields must be hashable.
+
+ OpaqueKeys will not have optional constructor parameters (due to the implementation of
+ KEY_FIELDS), by default. However, and implementation class can provide a default,
+ as long as it passes that default to a call to super().__init__.
+
+ OpaqueKeys are immutable.
+ """
+ __metaclass__ = OpaqueKeyMetaclass
+ __slots__ = ('_initialized')
+
+ NAMESPACE_SEPARATOR = u':'
+
+ @classmethod
+ @abstractmethod
+ def _from_string(cls, serialized):
+ """
+ Return an instance of `cls` parsed from its `serialized` form.
+
+ Args:
+ cls: The :class:`OpaqueKey` subclass.
+ serialized (unicode): A serialized :class:`OpaqueKey`, with namespace already removed.
+
+ Raises:
+ InvalidKeyError: Should be raised if `serialized` is not a valid serialized key
+ understood by `cls`.
+ """
+ raise NotImplementedError()
+
+ @abstractmethod
+ def _to_string(self):
+ """
+ Return a serialization of `self`.
+
+ This serialization should not include the namespace prefix.
+ """
+ raise NotImplementedError()
+
+ @classmethod
+ def _separate_namespace(cls, serialized):
+ """
+ Return the namespace from a serialized :class:`OpaqueKey`, and
+ the rest of the key.
+
+ Args:
+ serialized (unicode): A serialized :class:`OpaqueKey`.
+
+ Raises:
+ MissingNamespace: Raised when no namespace can be
+ extracted from `serialized`.
+ """
+ namespace, _, rest = serialized.partition(cls.NAMESPACE_SEPARATOR)
+
+ # If ':' isn't found in the string, then the source string
+ # is returned as the first result (i.e. namespace)
+ if namespace == serialized:
+ raise InvalidKeyError(cls, serialized)
+
+ return (namespace, rest)
+
+ def __init__(self, *args, **kwargs):
+ # pylint: disable=no-member
+ if len(args) + len(kwargs) != len(self.KEY_FIELDS):
+ raise TypeError('__init__() takes exactly {} arguments ({} given)'.format(
+ len(self.KEY_FIELDS),
+ len(args) + len(kwargs)
+ ))
+
+ keyed_args = dict(zip(self.KEY_FIELDS, args))
+
+ overlapping_args = keyed_args.viewkeys() & kwargs.viewkeys()
+ if overlapping_args:
+ raise TypeError('__init__() got multiple values for keyword argument {!r}'.format(overlapping_args[0]))
+
+ keyed_args.update(kwargs)
+
+ for key, value in keyed_args.viewitems():
+ if key not in self.KEY_FIELDS:
+ raise TypeError('__init__() got an unexpected argument {!r}'.format(key))
+
+ setattr(self, key, value)
+ self._initialized = True
+
+ def replace(self, **kwargs):
+ existing_values = {key: getattr(self, key) for key in self.KEY_FIELDS} # pylint: disable=no-member
+ existing_values.update(kwargs)
+ return type(self)(**existing_values)
+
+ def __setattr__(self, name, value):
+ if getattr(self, '_initialized', False):
+ raise AttributeError("Can't set {!r}. OpaqueKeys are immutable.".format(name))
+
+ super(OpaqueKey, self).__setattr__(name, value)
+
+ def __delattr__(self, name):
+ raise AttributeError("Can't delete {!r}. OpaqueKeys are immutable.".format(name))
+
+ def __unicode__(self):
+ return self.NAMESPACE_SEPARATOR.join([self.CANONICAL_NAMESPACE, self._to_string()]) # pylint: disable=no-member
+
+ def __copy__(self):
+ return self.replace()
+
+ def __deepcopy__(self, memo):
+ return self.replace(**{
+ key: deepcopy(getattr(self, key), memo) for key in self.KEY_FIELDS # pylint: disable=no-member
+ })
+
+ def __setstate__(self, state_dict):
+ # used by pickle to set fields on an unpickled object
+ for key in state_dict:
+ if key in self.KEY_FIELDS: # pylint: disable=no-member
+ setattr(self, key, state_dict[key])
+
+ def __getstate__(self):
+ # used by pickle to get fields on an unpickled object
+ pickleable_dict = {}
+ for key in self.KEY_FIELDS: # pylint: disable=no-member
+ pickleable_dict[key] = getattr(self, key)
+ return pickleable_dict
+
+ @property
+ def _key(self):
+ """Returns a tuple of key fields"""
+ return tuple(getattr(self, field) for field in self.KEY_FIELDS) # pylint: disable=no-member
+
+ def __eq__(self, other):
+ return (
+ type(self) == type(other) and
+ self._key == other._key # pylint: disable=protected-access
+ )
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __lt__(self, other):
+ if type(self) != type(other):
+ raise TypeError()
+ return self._key < other._key # pylint: disable=protected-access
+
+ def __hash__(self):
+ return hash(self._key)
+
+ def __str__(self):
+ return unicode(self).encode('utf-8')
+
+ def __repr__(self):
+ return '{}({})'.format(
+ self.__class__.__name__,
+ ', '.join(repr(getattr(self, key)) for key in self.KEY_FIELDS) # pylint: disable=no-member
+ )
+
+ def __len__(self):
+ """Return the number of characters in the serialized OpaqueKey"""
+ return len(unicode(self))
+
+ @classmethod
+ def _drivers(cls):
+ """
+ Return a driver manager for all key classes that are
+ subclasses of `cls`.
+ """
+ return EnabledExtensionManager(
+ cls.KEY_TYPE, # pylint: disable=no-member
+ check_func=lambda extension: issubclass(extension.plugin, cls),
+ invoke_on_load=False,
+ )
+
+ @classmethod
+ def from_string(cls, serialized):
+ """
+ Return a :class:`OpaqueKey` object deserialized from
+ the `serialized` argument. This object will be an instance
+ of a subclass of the `cls` argument.
+
+ Args:
+ serialized: A stringified form of a :class:`OpaqueKey`
+ """
+ if serialized is None:
+ raise InvalidKeyError(cls, serialized)
+
+ # pylint: disable=protected-access
+ namespace, rest = cls._separate_namespace(serialized)
+ try:
+ return cls._drivers()[namespace].plugin._from_string(rest)
+ except KeyError:
+ raise InvalidKeyError(cls, serialized)
diff --git a/common/lib/opaque_keys/opaque_keys/tests/__init__.py b/common/lib/opaque_keys/opaque_keys/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/common/lib/opaque_keys/opaque_keys/tests/test_opaque_keys.py b/common/lib/opaque_keys/opaque_keys/tests/test_opaque_keys.py
new file mode 100644
index 00000000000..4f9c8ef5a90
--- /dev/null
+++ b/common/lib/opaque_keys/opaque_keys/tests/test_opaque_keys.py
@@ -0,0 +1,182 @@
+import copy
+import json
+from unittest import TestCase
+from stevedore.extension import Extension
+from mock import Mock
+
+from opaque_keys import OpaqueKey, InvalidKeyError
+
+
+def _mk_extension(name, cls):
+ return Extension(
+ name,
+ Mock(name='entry_point_{}'.format(name)),
+ cls,
+ Mock(name='obj_{}'.format(name)),
+ )
+
+
+class DummyKey(OpaqueKey):
+ """
+ Key type for testing
+ """
+ KEY_TYPE = 'opaque_keys.testing'
+ __slots__ = ()
+
+
+class HexKey(DummyKey):
+ KEY_FIELDS = ('value',)
+ __slots__ = KEY_FIELDS
+
+ def _to_string(self):
+ return hex(self._value)
+
+ @classmethod
+ def _from_string(cls, serialized):
+ if not serialized.startswith('0x'):
+ raise InvalidKeyError(cls, serialized)
+ try:
+ return cls(int(serialized, 16))
+ except (ValueError, TypeError):
+ raise InvalidKeyError(cls, serialized)
+
+
+class Base10Key(DummyKey):
+ KEY_FIELDS = ('value',)
+ # Deliberately not using __slots__, to test both cases
+
+ def _to_string(self):
+ return unicode(self._value)
+
+ @classmethod
+ def _from_string(cls, serialized):
+ try:
+ return cls(int(serialized))
+ except (ValueError, TypeError):
+ raise InvalidKeyError(cls, serialized)
+
+
+class DictKey(DummyKey):
+ KEY_FIELDS = ('value',)
+ __slots__ = KEY_FIELDS
+
+ def _to_string(self):
+ return json.dumps(self._value)
+
+ @classmethod
+ def _from_string(cls, serialized):
+ try:
+ return cls(json.loads(serialized))
+ except (ValueError, TypeError):
+ raise InvalidKeyError(cls, serialized)
+
+
+class KeyTests(TestCase):
+ def test_namespace_from_string(self):
+ hex_key = DummyKey.from_string('hex:0x10')
+ self.assertIsInstance(hex_key, HexKey)
+ self.assertEquals(hex_key.value, 16)
+
+ base_key = DummyKey.from_string('base10:15')
+ self.assertIsInstance(base_key, Base10Key)
+ self.assertEquals(base_key.value, 15)
+
+ def test_unknown_namespace(self):
+ with self.assertRaises(InvalidKeyError):
+ DummyKey.from_string('no_namespace:0x10')
+
+ def test_no_namespace_from_string(self):
+ with self.assertRaises(InvalidKeyError):
+ DummyKey.from_string('0x10')
+
+ with self.assertRaises(InvalidKeyError):
+ DummyKey.from_string('15')
+
+ def test_immutability(self):
+ key = HexKey(10)
+
+ with self.assertRaises(AttributeError):
+ key.value = 11 # pylint: disable=attribute-defined-outside-init
+
+ def test_equality(self):
+ self.assertEquals(DummyKey.from_string('hex:0x10'), DummyKey.from_string('hex:0x10'))
+ self.assertNotEquals(DummyKey.from_string('hex:0x10'), DummyKey.from_string('base10:16'))
+
+ def test_constructor(self):
+ with self.assertRaises(TypeError):
+ HexKey()
+
+ with self.assertRaises(TypeError):
+ HexKey(foo='bar')
+
+ with self.assertRaises(TypeError):
+ HexKey(10, 20)
+
+ with self.assertRaises(TypeError):
+ HexKey(value=10, bar=20)
+
+ self.assertEquals(HexKey(10).value, 10)
+ self.assertEquals(HexKey(value=10).value, 10)
+
+ def test_replace(self):
+ hex10 = HexKey(10)
+ hex11 = hex10.replace(value=11)
+ hex_copy = hex10.replace()
+
+ self.assertNotEquals(id(hex10), id(hex11))
+ self.assertNotEquals(id(hex10), id(hex_copy))
+ self.assertNotEquals(hex10, hex11)
+ self.assertEquals(hex10, hex_copy)
+ self.assertEquals(HexKey(10), hex10)
+ self.assertEquals(HexKey(11), hex11)
+
+ def test_copy(self):
+ original = DictKey({'foo': 'bar'})
+ copied = copy.copy(original)
+ deep = copy.deepcopy(original)
+
+ self.assertEquals(original, copied)
+ self.assertNotEquals(id(original), id(copied))
+ self.assertEquals(id(original.value), id(copied.value))
+
+ self.assertEquals(original, deep)
+ self.assertNotEquals(id(original), id(deep))
+ self.assertNotEquals(id(original.value), id(deep.value))
+
+ self.assertEquals(copy.deepcopy([original]), [original])
+
+ def test_subclass(self):
+ with self.assertRaises(InvalidKeyError):
+ HexKey.from_string('base10:15')
+
+ with self.assertRaises(InvalidKeyError):
+ Base10Key.from_string('hex:0x10')
+
+ def test_ordering(self):
+ ten = HexKey(value=10)
+ eleven = HexKey(value=11)
+
+ self.assertLess(ten, eleven)
+ self.assertLessEqual(ten, ten)
+ self.assertLessEqual(ten, eleven)
+ self.assertGreater(eleven, ten)
+ self.assertGreaterEqual(eleven, eleven)
+ self.assertGreaterEqual(eleven, ten)
+
+ def test_non_ordering(self):
+ # Verify that different key types aren't comparable
+ ten = HexKey(value=10)
+ twelve = Base10Key(value=12)
+
+ # pylint: disable=pointless-statement
+ with self.assertRaises(TypeError):
+ ten < twelve
+
+ with self.assertRaises(TypeError):
+ ten > twelve
+
+ with self.assertRaises(TypeError):
+ ten <= twelve
+
+ with self.assertRaises(TypeError):
+ ten >= twelve
diff --git a/common/lib/opaque_keys/setup.py b/common/lib/opaque_keys/setup.py
new file mode 100644
index 00000000000..eaf0ec48971
--- /dev/null
+++ b/common/lib/opaque_keys/setup.py
@@ -0,0 +1,19 @@
+from setuptools import setup
+
+setup(
+ name="opaque_keys",
+ version="0.1",
+ packages=[
+ "opaque_keys",
+ ],
+ install_requires=[
+ "stevedore"
+ ],
+ entry_points={
+ 'opaque_keys.testing': [
+ 'base10 = opaque_keys.tests.test_opaque_keys:Base10Key',
+ 'hex = opaque_keys.tests.test_opaque_keys:HexKey',
+ 'dict = opaque_keys.tests.test_opaque_keys:DictKey',
+ ]
+ }
+)
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index 466cadcd660..02f3a7ad968 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -70,6 +70,7 @@ Shapely==1.2.16
singledispatch==3.4.0.2
sorl-thumbnail==11.12
South==0.7.6
+stevedore==0.14.1
sure==1.2.3
sympy==0.7.1
xmltodict==0.4.1
diff --git a/requirements/edx/local.txt b/requirements/edx/local.txt
index 0e775d04e3f..e56d8dc8ace 100644
--- a/requirements/edx/local.txt
+++ b/requirements/edx/local.txt
@@ -3,6 +3,7 @@
-e common/lib/calc
-e common/lib/capa
-e common/lib/chem
+-e common/lib/opaque_keys
-e common/lib/sandbox-packages
-e common/lib/symmath
-e common/lib/xmodule
--
GitLab