Merge pull request #26065 from edx/feanil/update_cookie_metrics

Add more cookie logging code.

openedx/core/lib/request_utils.py

@@ -102,18 +102,104 @@ class CookieMonitoringMiddleware(MiddlewareMixin):
 
         Don't log contents of cookies because that might cause a security issue.
         We just want to see if any cookies are growing out of control.
+
+        A useful NRQL Query:
+            SELECT count(*), max(`cookies.max.group.size`) from Transaction FACET
+            `cookies.max.group.name`
+
+            SELECT * FROM Transaction WHERE cookies_total_size > 6000
+
+        Attributes that are added by this middleware:
+
+        cookies.<N>.name: The name of the Nth largest cookie
+        cookies.<N>.size: The size of the Nth largest cookie
+        cookies..group.<N>.name: The name of the Nth largest cookie.
+        cookies.group.<N>.size: The size of the Nth largest cookie group.
+        cookies.max.name: The name of the largest cookie sent by the user.
+        cookies.max.size: The size of the largest cookie sent by the user.
+        cookies.max.group.name: The name of the largest group of cookies. A single cookie
+            counts as a group of one for this calculation.
+        cookies.max.group.size: The sum total size of all the cookies in the largest group.
+        cookies_total_size: The sum total size of all cookies in this request.
+
+        Related Settings:
+
+        - `request_utils.capture_cookie_sizes` is the waffle flag that control whether this
+            middleware logs anything or not.
+
+        - TOP_N_COOKIES_CAPTURED(Default: 5) controls how many cookies to log.
+        - TOP_N_COOKIE_GROUPS_CAPTURED(Default: 5): controls how many cookie groups to capture.
+
+
         """
         if not CAPTURE_COOKIE_SIZES.is_enabled():
             return
 
-        cookie_names_to_size = {
-            name: len(value)
-            for name, value in request.COOKIES.items()
-        }
-        for name, size in cookie_names_to_size.items():
-            attribute_name = 'cookies.{}.size'.format(name)
-            set_custom_attribute(attribute_name, size)
-            log.debug(u'%s = %d', attribute_name, size)
+        # Capture the N largest cookies
+        top_n_cookies_captured = getattr(settings, "TOP_N_COOKIES_CAPTURED", 5)
+        top_n_cookie_groups_captured = getattr(settings, "TOP_N_COOKIE_GROUPS_CAPTURED", 5)
+
+        cookie_names_to_size = {}
+        cookie_groups_to_size = {}
+
+        for name, value in request.COOKIES.items():
+            # Get cookie size for all cookies.
+            cookie_size = len(value)
+            cookie_names_to_size[name] = cookie_size
+
+            # Group cookies by their prefix seperated by a period or underscore
+            grouping_name = re.split('[._]', name, 1)[0]
+            if grouping_name and grouping_name != name:
+                # Add or update the size for this group.
+                cookie_groups_to_size[grouping_name] = cookie_groups_to_size.get(grouping_name, 0) + cookie_size
+
+        max_cookie_name = max(cookie_names_to_size, key=lambda name: cookie_names_to_size[name])
+        max_cookie_size = cookie_names_to_size[max_cookie_name]
+
+        max_group_cookie_name = max(cookie_groups_to_size, key=lambda name: cookie_groups_to_size[name])
+        max_group_cookie_size = cookie_groups_to_size[max_group_cookie_name]
+
+        # If a single cookies is bigger than any group of cookies, we want max_group... to reflect that.
+        # Treating an individual cookie as a group of 1 for calculating the max.
+        if max_group_cookie_size < max_cookie_size:
+            max_group_cookie_name = max_cookie_name
+            max_group_cookie_size = max_cookie_size
+
+        # Log only the top N biggest cookies.
+        top_n_cookies = sorted(
+            cookie_names_to_size,
+            key=lambda x: cookie_names_to_size[x],
+            reverse=True,
+        )[:top_n_cookies_captured]
+        for index, name in enumerate(top_n_cookies, start=1):
+            size = cookie_names_to_size[name]
+            name_attribute = 'cookies.{}.name'.format(index)
+            size_attribute = 'cookies.{}.size'.format(index)
+
+            set_custom_attribute(name_attribute, name)
+            set_custom_attribute(size_attribute, size)
+            log.debug(u'%s = %d', name, size)
+
+        # Log only the top N biggest groups.
+        top_n_cookie_groups = sorted(
+            cookie_groups_to_size,
+            key=lambda x: cookie_groups_to_size[x],
+            reverse=True,
+        )[:top_n_cookie_groups_captured]
+
+        for index, name in enumerate(top_n_cookie_groups, start=1):
+            size = cookie_groups_to_size[name]
+            name_attribute = 'cookies.group.{}.name'.format(index)
+            size_attribute = 'cookies.group.{}.size'.format(index)
+
+            set_custom_attribute(name_attribute, name)
+            set_custom_attribute(size_attribute, size)
+            log.debug(u'%s = %d', name, size)
+
+        set_custom_attribute('cookies.max.name', max_cookie_name)
+        set_custom_attribute('cookies.max.size', max_cookie_size)
+        set_custom_attribute('cookies.max.group.name', max_group_cookie_name)
+        set_custom_attribute('cookies.max.group.size', max_group_cookie_size)
 
         total_cookie_size = sum(cookie_names_to_size.values())
         set_custom_attribute('cookies_total_size', total_cookie_size)

openedx/core/lib/tests/test_request_utils.py

@@ -2,12 +2,18 @@
 
 
 import unittest
+from unittest.mock import Mock, patch, call
 
 from django.conf import settings
 from django.core.exceptions import SuspiciousOperation
 from django.test.client import RequestFactory
 
-from openedx.core.lib.request_utils import get_request_or_stub, course_id_from_url, safe_get_host
+from openedx.core.lib.request_utils import (
+    get_request_or_stub,
+    course_id_from_url,
+    safe_get_host,
+    CookieMonitoringMiddleware,
+)
 
 
 class RequestUtilTestCase(unittest.TestCase):
@@ -83,3 +89,78 @@ class RequestUtilTestCase(unittest.TestCase):
         self.assertEqual(course_id.org, org)
         self.assertEqual(course_id.course, course)
         self.assertEqual(course_id.run, run)
+
+    @patch("openedx.core.lib.request_utils.CAPTURE_COOKIE_SIZES")
+    @patch("openedx.core.lib.request_utils.set_custom_attribute")
+    def test_cookie_monitoring(self, mock_set_custom_attribute, mock_capture_cookie_sizes):
+
+        mock_capture_cookie_sizes.is_enabled.return_value = True
+        middleware = CookieMonitoringMiddleware()
+
+        mock_request = Mock()
+        mock_request.COOKIES = {
+            "a": "." * 100,
+            "_b": "." * 13,
+            "_c_": "." * 13,
+            "a.b": "." * 10,
+            "a.c": "." * 10,
+            "b.": "." * 13,
+            "b_a": "." * 15,
+            "b_c": "." * 15,
+        }
+
+        middleware.process_request(mock_request)
+
+        mock_set_custom_attribute.assert_has_calls([
+            call('cookies.1.name', 'a'),
+            call('cookies.1.size', 100),
+            call('cookies.2.name', 'b_a'),
+            call('cookies.2.size', 15),
+            call('cookies.3.name', 'b_c'),
+            call('cookies.3.size', 15),
+            call('cookies.4.name', '_b'),
+            call('cookies.4.size', 13),
+            call('cookies.5.name', '_c_'),
+            call('cookies.5.size', 13),
+            call('cookies.group.1.name', 'b'),
+            call('cookies.group.1.size', 43),
+            call('cookies.group.2.name', 'a'),
+            call('cookies.group.2.size', 20),
+            call('cookies.max.name', 'a'),
+            call('cookies.max.size', 100),
+            call('cookies.max.group.name', 'a'),
+            call('cookies.max.group.size', 100),
+            call('cookies_total_size', 189),
+        ])
+
+    @patch("openedx.core.lib.request_utils.CAPTURE_COOKIE_SIZES")
+    @patch("openedx.core.lib.request_utils.set_custom_attribute")
+    def test_cookie_monitoring_max_group(self, mock_set_custom_attribute, mock_capture_cookie_sizes):
+
+        mock_capture_cookie_sizes.is_enabled.return_value = True
+        middleware = CookieMonitoringMiddleware()
+
+        mock_request = Mock()
+        mock_request.COOKIES = {
+            "a": "." * 10,
+            "b_a": "." * 15,
+            "b_c": "." * 20,
+        }
+
+        middleware.process_request(mock_request)
+
+        mock_set_custom_attribute.assert_has_calls([
+            call('cookies.1.name', 'b_c'),
+            call('cookies.1.size', 20),
+            call('cookies.2.name', 'b_a'),
+            call('cookies.2.size', 15),
+            call('cookies.3.name', 'a'),
+            call('cookies.3.size', 10),
+            call('cookies.group.1.name', 'b'),
+            call('cookies.group.1.size', 35),
+            call('cookies.max.name', 'b_c'),
+            call('cookies.max.size', 20),
+            call('cookies.max.group.name', 'b'),
+            call('cookies.max.group.size', 35),
+            call('cookies_total_size', 45)
+        ])