kinto 20.2.0__tar.gz → 20.3.0__tar.gz

{kinto-20.2.0 → kinto-20.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kinto
-Version: 20.2.0
+Version: 20.3.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation

{kinto-20.2.0 → kinto-20.3.0}/docs/api/1.x/history.rst

@@ -195,3 +195,21 @@ It is possible to exclude certain resources from being tracked by history using
 
     kinto.history.exclude_resources = /buckets/preview
                                       /buckets/signed/collections/certificates
+
+Or certain users:
+
+.. code-block:: ini
+
+    kinto.history.exclude_user_ids = account:quicksuggest
+
+In order to limit the size of the history length of each resource:
+
+.. code-block:: ini
+
+    kinto.history.auto_trim_max_count = 100
+
+In order to limit the size of the history length of each resource only for certain users:
+
+.. code-block:: ini
+
+    kinto.history.auto_trim_user_ids = account:quicksuggest

{kinto-20.2.0 → kinto-20.3.0}/kinto/core/storage/memory.py

@@ -288,13 +288,26 @@ class Storage(MemoryBasedStorage):
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
         parent_id_match = re.compile(parent_id.replace("*", ".*"))
-        by_parent_id = {
+
+        timestamps_by_parent_id = {
             pid: resources
-            for pid, resources in self._cemetery.items()
+            for pid, resources in self._timestamps.items()
             if parent_id_match.match(pid)
         }
+        if resource_name is not None:
+            for pid, resources in timestamps_by_parent_id.items():
+                del self._timestamps[pid][resource_name]
+        else:
+            for pid, resources in timestamps_by_parent_id.items():
+                del self._timestamps[pid]
+
         num_deleted = 0
-        for pid, resources in by_parent_id.items():
+        tombstones_by_parent_id = {
+            pid: resources
+            for pid, resources in self._cemetery.items()
+            if parent_id_match.match(pid)
+        }
+        for pid, resources in tombstones_by_parent_id.items():
             if resource_name is not None:
                 resources = {resource_name: resources[resource_name]}
             for resource, resource_objects in resources.items():

{kinto-20.2.0 → kinto-20.3.0}/kinto/core/storage/postgresql/__init__.py

@@ -79,7 +79,7 @@ class Storage(StorageBase, MigratorMixin):
 
     # MigratorMixin attributes.
     name = "storage"
-    schema_version = 22
+    schema_version = 23
     schema_file = os.path.join(HERE, "schema.sql")
     migrations_directory = os.path.join(HERE, "migrations")
 

kinto-20.3.0/kinto/core/storage/postgresql/migrations/migration_022_023.sql

@@ -0,0 +1,5 @@
+CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
+    ON objects(resource_name, parent_id, deleted);
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');

{kinto-20.2.0 → kinto-20.3.0}/kinto/core/storage/postgresql/schema.sql

@@ -47,7 +47,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_objects_parent_id_resource_name_last_modif
     ON objects(parent_id, resource_name, last_modified DESC);
 CREATE INDEX IF NOT EXISTS idx_objects_last_modified_epoch
     ON objects(as_epoch(last_modified));
-
+CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
+    ON objects(resource_name, parent_id, deleted);
 
 CREATE TABLE IF NOT EXISTS timestamps (
   parent_id TEXT NOT NULL COLLATE "C",
@@ -131,4 +132,4 @@ INSERT INTO metadata (name, value) VALUES ('created_at', NOW()::TEXT);
 
 -- Set storage schema version.
 -- Should match ``kinto.core.storage.postgresql.PostgreSQL.schema_version``
-INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '22');
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');

{kinto-20.2.0 → kinto-20.3.0}/kinto/core/storage/testing.py

@@ -1296,6 +1296,9 @@ class DeletedObjectsTest:
         self.create_object(parent_id="/abc/a", resource_name="c")
         self.create_object(parent_id="/efg", resource_name="c")
 
+        all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
+        self.assertEqual(set(all_timestamps.keys()), {"/abc/a", "/efg"})
+
         before1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
         # Different parent_id with object.
         before2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
@@ -1305,11 +1308,15 @@ class DeletedObjectsTest:
         self.storage.delete_all(parent_id="/abc/*", resource_name=None, with_deleted=False)
         self.storage.purge_deleted(parent_id="/abc/*", resource_name=None)
 
+        all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
+        self.assertEqual(set(all_timestamps.keys()), {"/efg", "/ijk"})
+
+        time.sleep(0.002)  # make sure we don't recreate timestamps at same msec.
         after1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
         after2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
         after3 = self.storage.resource_timestamp(parent_id="/ijk", resource_name="c")
 
-        self.assertNotEqual(before1, after1)
+        self.assertNotEqual(before1, after1)  # timestamp was removed, it will differ.
         self.assertEqual(before2, after2)
         self.assertEqual(before3, after3)
 

{kinto-20.2.0 → kinto-20.3.0}/kinto/plugins/history/listener.py

@@ -1,8 +1,13 @@
+import logging
 from datetime import datetime, timezone
 
 from pyramid.settings import asbool, aslist
 
-from kinto.core.utils import instance_uri
+from kinto.core.storage import Filter, Sort
+from kinto.core.utils import COMPARISON, instance_uri
+
+
+logger = logging.getLogger(__name__)
 
 
 def on_resource_changed(event):
@@ -14,15 +19,26 @@ def on_resource_changed(event):
     payload = event.payload
     resource_name = payload["resource_name"]
     event_uri = payload["uri"]
-
-    bucket_id = None
-    bucket_uri = None
-    collection_uri = None
+    user_id = payload["user_id"]
 
     storage = event.request.registry.storage
     permission = event.request.registry.permission
     settings = event.request.registry.settings
 
+    excluded_user_ids = aslist(settings.get("history.exclude_user_ids", ""))
+    if user_id in excluded_user_ids:
+        logger.info(f"History entries for user {user_id!r} are disabled in config")
+        return
+
+    trim_history_max = int(settings.get("history.auto_trim_max_count", "-1"))
+    is_trim_enabled = trim_history_max > 0
+    trim_user_ids = aslist(settings.get("history.auto_trim_user_ids", ""))
+    is_trim_by_user_enabled = len(trim_user_ids) > 0
+
+    bucket_id = None
+    bucket_uri = None
+    collection_uri = None
+
     excluded_resources = aslist(settings.get("history.exclude_resources", ""))
 
     targets = []
@@ -38,6 +54,7 @@ def on_resource_changed(event):
         bucket_uri = instance_uri(event.request, "bucket", id=bucket_id)
 
         if bucket_uri in excluded_resources:
+            logger.info(f"History entries for bucket {bucket_uri!r} are disabled in config")
             continue
 
         if "collection_id" in payload:
@@ -46,6 +63,9 @@ def on_resource_changed(event):
                 event.request, "collection", bucket_id=bucket_id, id=collection_id
             )
             if collection_uri in excluded_resources:
+                logger.info(
+                    f"History entries for collection {collection_uri!r} are disabled in config"
+                )
                 continue
 
         # On POST .../records, the URI does not contain the newly created
@@ -59,6 +79,7 @@ def on_resource_changed(event):
         uri = "/".join(parts)
 
         if uri in excluded_resources:
+            logger.info(f"History entries for record {uri!r} are disabled in config")
             continue
 
         targets.append((uri, target))
@@ -109,6 +130,40 @@ def on_resource_changed(event):
         # Note: this will be rolledback if the transaction is rolledback.
         entry = storage.create(parent_id=bucket_uri, resource_name="history", obj=attrs)
 
+        # If enabled, we trim history by resource.
+        # This means that we will only keep the last `auto_trim_max_count` history entries
+        # for this same type of object (eg. `collection`, `record`).
+        #
+        # If trim by user is enabled, we only trim if the user matches the config
+        # and we only delete the history entries of this user.
+        # This means that if a user touches X different types of objects, we will keep
+        # ``(X * auto_trim_max_count)`` entries.
+        if is_trim_enabled and (not is_trim_by_user_enabled or user_id in trim_user_ids):
+            filters = [
+                Filter("resource_name", resource_name, COMPARISON.EQ),
+            ]
+            if is_trim_by_user_enabled:
+                filters.append(Filter("user_id", user_id, COMPARISON.EQ))
+
+            # Identify the oldest entry to keep.
+            previous_entries = storage.list_all(
+                parent_id=bucket_uri,
+                resource_name="history",
+                filters=filters,
+                sorting=[Sort("last_modified", -1)],
+                limit=trim_history_max + 1,
+            )
+            # And delete all older ones.
+            if len(previous_entries) > trim_history_max:
+                trim_before_timestamp = previous_entries[-1]["last_modified"]
+                deleted = storage.delete_all(
+                    parent_id=bucket_uri,
+                    resource_name="history",
+                    filters=filters
+                    + [Filter("last_modified", trim_before_timestamp, COMPARISON.MAX)],
+                )
+                logger.debug(f"Trimmed {len(deleted)} old history entries.")
+
         # Without explicit permissions, the ACLs on the history entries will
         # fully depend on the inherited permission tree (eg. bucket:read, bucket:write).
         # This basically means that if user loose the permissions on the related

{kinto-20.2.0 → kinto-20.3.0}/kinto.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kinto
-Version: 20.2.0
+Version: 20.3.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation

{kinto-20.2.0 → kinto-20.3.0}/kinto.egg-info/SOURCES.txt

@@ -240,6 +240,7 @@ kinto/core/storage/postgresql/migrations/migration_018_019.sql
 kinto/core/storage/postgresql/migrations/migration_019_020.sql
 kinto/core/storage/postgresql/migrations/migration_020_021.sql
 kinto/core/storage/postgresql/migrations/migration_021_022.sql
+kinto/core/storage/postgresql/migrations/migration_022_023.sql
 kinto/core/views/__init__.py
 kinto/core/views/batch.py
 kinto/core/views/errors.py

{kinto-20.2.0 → kinto-20.3.0}/tests/plugins/test_history.py

@@ -670,6 +670,152 @@ class ExcludeResourcesTest(HistoryWebTest):
         assert len(entries) == 0  # nothing.
 
 
+class ExcludeUserIdTest(HistoryWebTest):
+    joan_headers = get_user_headers("joan")
+    joan_principal = "basicauth:64942e918f6481e3101f3a22a87a3206480923ccc0d4387cff9cb1ae0af21217"
+
+    @classmethod
+    def get_app_settings(cls, extras=None):
+        settings = super().get_app_settings(extras)
+        settings["history.exclude_user_ids"] = cls.joan_principal
+        return settings
+
+    def setUp(self):
+        self.app.put_json("/buckets/bid", headers=self.headers)
+        self.app.put_json(
+            "/buckets/bid/collections/cid",
+            {"permissions": {"write": ["system.Authenticated"]}},
+            headers=self.headers,
+        )
+
+    def test_entries_exclude_joan(self):
+        resp = self.app.get("/buckets/bid/history", headers=self.headers)
+        entries = resp.json["data"]
+        self.assertEqual(len(entries), 2)  # creations
+
+        self.app.post("/buckets/bid/collections/cid/records", headers=self.headers)
+        self.app.post("/buckets/bid/collections/cid/records", headers=self.joan_headers)
+
+        resp = self.app.get("/buckets/bid/history", headers=self.headers)
+        entries = resp.json["data"]
+        self.assertEqual(len(entries), 3)  # not 4
+        self.assertEqual(set(e["user_id"] for e in entries), {self.principal})
+
+
+class TrimHistoryTest(HistoryWebTest):
+    @classmethod
+    def get_app_settings(cls, extras=None):
+        settings = super().get_app_settings(extras)
+        settings["history.auto_trim_max_count"] = "5"
+        return settings
+
+    def test_history_length_is_limited_by_resource(self):
+        for i in range(10):
+            self.app.put_json("/buckets/bid", {"data": {"i": i}}, headers=self.headers)
+            self.app.put(f"/buckets/bid/collections/cid-{i}", headers=self.headers)
+
+        resp = self.app.get("/buckets/bid/history", headers=self.headers)
+        by_uri = {}
+        for entry in resp.json["data"]:
+            by_uri.setdefault(entry["uri"], []).append(entry)
+
+        self.assertEqual(
+            {
+                "/buckets/bid": 5,
+                "/buckets/bid/collections/cid-5": 1,
+                "/buckets/bid/collections/cid-6": 1,
+                "/buckets/bid/collections/cid-7": 1,
+                "/buckets/bid/collections/cid-8": 1,
+                "/buckets/bid/collections/cid-9": 1,
+            },
+            {uri: len(entries) for uri, entries in by_uri.items()},
+        )
+
+
+class TrimUserIdTest(HistoryWebTest):
+    joan_headers = get_user_headers("joan")
+    joan_principal = "basicauth:64942e918f6481e3101f3a22a87a3206480923ccc0d4387cff9cb1ae0af21217"
+
+    @classmethod
+    def get_app_settings(cls, extras=None):
+        settings = super().get_app_settings(extras)
+        settings["history.auto_trim_user_ids"] = cls.joan_principal
+        settings["history.auto_trim_max_count"] = "5"
+        return settings
+
+    def setUp(self):
+        self.app.put_json(
+            "/buckets/bid",
+            {"permissions": {"write": ["system.Authenticated"]}},
+            headers=self.headers,
+        )
+        self.app.put_json(
+            "/buckets/bid/collections/cid",
+            headers=self.headers,
+        )
+
+    def test_unlisted_user_entries_are_not_limited(self):
+        for i in range(10):
+            self.app.put(f"/buckets/bid/groups/gid-{i}", headers=self.headers)
+
+        resp = self.app.get(f"/buckets/bid/history?user_id={self.principal}", headers=self.headers)
+
+        self.assertEqual(len(resp.json["data"]), 12)  # bucket + collection + 10 groups
+
+    def test_listed_user_entries_are_limited(self):
+        for i in range(10):
+            self.app.put(f"/buckets/bid/groups/gid-{i}", headers=self.joan_headers)
+
+        resp = self.app.get(
+            f"/buckets/bid/history?user_id={self.joan_principal}", headers=self.joan_headers
+        )
+
+        self.assertEqual(len(resp.json["data"]), 5)
+
+    def test_trim_is_by_user_and_by_resource_name(self):
+        for headers in (self.headers, self.joan_headers):
+            for i in range(10):
+                self.app.patch_json(
+                    "/buckets/bid/collections/cid", {"data": {"attr": i}}, headers=headers
+                )
+
+        for headers in (self.headers, self.joan_headers):
+            for i in range(3):
+                # Delete all records in it.
+                self.app.delete("/buckets/bid/collections/cid/records", headers=headers)
+                for j in range(10):
+                    # Recreate new ones with different ids
+                    self.app.put(
+                        f"/buckets/bid/collections/cid/records/rid-{i * 10 + j}", headers=headers
+                    )
+
+        resp = self.app.get("/buckets/bid/history", headers=self.headers)
+
+        by_user_id = {}
+        for entry in resp.json["data"]:
+            by_user_id.setdefault(entry["user_id"], {}).setdefault(entry["uri"], []).append(entry)
+        self.assertEqual(
+            {
+                "/buckets/bid/collections/cid": 5,  # Out of 10 on the collection
+                "/buckets/bid/collections/cid/records/rid-29": 1,
+                "/buckets/bid/collections/cid/records/rid-28": 1,
+                "/buckets/bid/collections/cid/records/rid-27": 1,
+                "/buckets/bid/collections/cid/records/rid-26": 1,
+                "/buckets/bid/collections/cid/records/rid-25": 1,
+            },
+            {k: len(v) for k, v in by_user_id[self.joan_principal].items()},
+        )
+        # Other is not trimmed.
+        self.assertEqual(
+            max(len(by_uri) for by_uri in by_user_id[self.principal].values()),
+            11,  # 1 creation + 10 patch
+        )
+        self.assertEqual(
+            len(by_user_id[self.principal].keys()),
+            32,  # 1 bucket + 1 collection + 30 records
+        )
+
+
 class DisabledExplicitPermissionsTest(HistoryWebTest, unittest.TestCase):
     @classmethod
     def get_app_settings(cls, extras=None):