kinto 20.2.0__py3-none-any.whl → 20.3.0__py3-none-any.whl

kinto/core/storage/memory.py

@@ -288,13 +288,26 @@ class Storage(MemoryBasedStorage):
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
         parent_id_match = re.compile(parent_id.replace("*", ".*"))
-        by_parent_id = {
+
+        timestamps_by_parent_id = {
             pid: resources
-            for pid, resources in self._cemetery.items()
+            for pid, resources in self._timestamps.items()
             if parent_id_match.match(pid)
         }
+        if resource_name is not None:
+            for pid, resources in timestamps_by_parent_id.items():
+                del self._timestamps[pid][resource_name]
+        else:
+            for pid, resources in timestamps_by_parent_id.items():
+                del self._timestamps[pid]
+
         num_deleted = 0
-        for pid, resources in by_parent_id.items():
+        tombstones_by_parent_id = {
+            pid: resources
+            for pid, resources in self._cemetery.items()
+            if parent_id_match.match(pid)
+        }
+        for pid, resources in tombstones_by_parent_id.items():
             if resource_name is not None:
                 resources = {resource_name: resources[resource_name]}
             for resource, resource_objects in resources.items():

kinto/core/storage/postgresql/__init__.py

@@ -79,7 +79,7 @@ class Storage(StorageBase, MigratorMixin):
 
     # MigratorMixin attributes.
     name = "storage"
-    schema_version = 22
+    schema_version = 23
     schema_file = os.path.join(HERE, "schema.sql")
     migrations_directory = os.path.join(HERE, "migrations")
 

kinto/core/storage/postgresql/migrations/migration_022_023.sql

@@ -0,0 +1,5 @@
+CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
+    ON objects(resource_name, parent_id, deleted);
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');

kinto/core/storage/postgresql/schema.sql

@@ -47,7 +47,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_objects_parent_id_resource_name_last_modif
     ON objects(parent_id, resource_name, last_modified DESC);
 CREATE INDEX IF NOT EXISTS idx_objects_last_modified_epoch
     ON objects(as_epoch(last_modified));
-
+CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
+    ON objects(resource_name, parent_id, deleted);
 
 CREATE TABLE IF NOT EXISTS timestamps (
   parent_id TEXT NOT NULL COLLATE "C",
@@ -131,4 +132,4 @@ INSERT INTO metadata (name, value) VALUES ('created_at', NOW()::TEXT);
 
 -- Set storage schema version.
 -- Should match ``kinto.core.storage.postgresql.PostgreSQL.schema_version``
-INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '22');
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');

kinto/core/storage/testing.py

@@ -1296,6 +1296,9 @@ class DeletedObjectsTest:
         self.create_object(parent_id="/abc/a", resource_name="c")
         self.create_object(parent_id="/efg", resource_name="c")
 
+        all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
+        self.assertEqual(set(all_timestamps.keys()), {"/abc/a", "/efg"})
+
         before1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
         # Different parent_id with object.
         before2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
@@ -1305,11 +1308,15 @@ class DeletedObjectsTest:
         self.storage.delete_all(parent_id="/abc/*", resource_name=None, with_deleted=False)
         self.storage.purge_deleted(parent_id="/abc/*", resource_name=None)
 
+        all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
+        self.assertEqual(set(all_timestamps.keys()), {"/efg", "/ijk"})
+
+        time.sleep(0.002)  # make sure we don't recreate timestamps at same msec.
         after1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
         after2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
         after3 = self.storage.resource_timestamp(parent_id="/ijk", resource_name="c")
 
-        self.assertNotEqual(before1, after1)
+        self.assertNotEqual(before1, after1)  # timestamp was removed, it will differ.
         self.assertEqual(before2, after2)
         self.assertEqual(before3, after3)
 

kinto/plugins/history/listener.py

@@ -1,8 +1,13 @@
+import logging
 from datetime import datetime, timezone
 
 from pyramid.settings import asbool, aslist
 
-from kinto.core.utils import instance_uri
+from kinto.core.storage import Filter, Sort
+from kinto.core.utils import COMPARISON, instance_uri
+
+
+logger = logging.getLogger(__name__)
 
 
 def on_resource_changed(event):
@@ -14,15 +19,26 @@ def on_resource_changed(event):
     payload = event.payload
     resource_name = payload["resource_name"]
     event_uri = payload["uri"]
-
-    bucket_id = None
-    bucket_uri = None
-    collection_uri = None
+    user_id = payload["user_id"]
 
     storage = event.request.registry.storage
     permission = event.request.registry.permission
     settings = event.request.registry.settings
 
+    excluded_user_ids = aslist(settings.get("history.exclude_user_ids", ""))
+    if user_id in excluded_user_ids:
+        logger.info(f"History entries for user {user_id!r} are disabled in config")
+        return
+
+    trim_history_max = int(settings.get("history.auto_trim_max_count", "-1"))
+    is_trim_enabled = trim_history_max > 0
+    trim_user_ids = aslist(settings.get("history.auto_trim_user_ids", ""))
+    is_trim_by_user_enabled = len(trim_user_ids) > 0
+
+    bucket_id = None
+    bucket_uri = None
+    collection_uri = None
+
     excluded_resources = aslist(settings.get("history.exclude_resources", ""))
 
     targets = []
@@ -38,6 +54,7 @@ def on_resource_changed(event):
         bucket_uri = instance_uri(event.request, "bucket", id=bucket_id)
 
         if bucket_uri in excluded_resources:
+            logger.info(f"History entries for bucket {bucket_uri!r} are disabled in config")
             continue
 
         if "collection_id" in payload:
@@ -46,6 +63,9 @@ def on_resource_changed(event):
                 event.request, "collection", bucket_id=bucket_id, id=collection_id
             )
             if collection_uri in excluded_resources:
+                logger.info(
+                    f"History entries for collection {collection_uri!r} are disabled in config"
+                )
                 continue
 
         # On POST .../records, the URI does not contain the newly created
@@ -59,6 +79,7 @@ def on_resource_changed(event):
         uri = "/".join(parts)
 
         if uri in excluded_resources:
+            logger.info(f"History entries for record {uri!r} are disabled in config")
             continue
 
         targets.append((uri, target))
@@ -109,6 +130,40 @@ def on_resource_changed(event):
         # Note: this will be rolledback if the transaction is rolledback.
         entry = storage.create(parent_id=bucket_uri, resource_name="history", obj=attrs)
 
+        # If enabled, we trim history by resource.
+        # This means that we will only keep the last `auto_trim_max_count` history entries
+        # for this same type of object (eg. `collection`, `record`).
+        #
+        # If trim by user is enabled, we only trim if the user matches the config
+        # and we only delete the history entries of this user.
+        # This means that if a user touches X different types of objects, we will keep
+        # ``(X * auto_trim_max_count)`` entries.
+        if is_trim_enabled and (not is_trim_by_user_enabled or user_id in trim_user_ids):
+            filters = [
+                Filter("resource_name", resource_name, COMPARISON.EQ),
+            ]
+            if is_trim_by_user_enabled:
+                filters.append(Filter("user_id", user_id, COMPARISON.EQ))
+
+            # Identify the oldest entry to keep.
+            previous_entries = storage.list_all(
+                parent_id=bucket_uri,
+                resource_name="history",
+                filters=filters,
+                sorting=[Sort("last_modified", -1)],
+                limit=trim_history_max + 1,
+            )
+            # And delete all older ones.
+            if len(previous_entries) > trim_history_max:
+                trim_before_timestamp = previous_entries[-1]["last_modified"]
+                deleted = storage.delete_all(
+                    parent_id=bucket_uri,
+                    resource_name="history",
+                    filters=filters
+                    + [Filter("last_modified", trim_before_timestamp, COMPARISON.MAX)],
+                )
+                logger.debug(f"Trimmed {len(deleted)} old history entries.")
+
         # Without explicit permissions, the ACLs on the history entries will
         # fully depend on the inherited permission tree (eg. bucket:read, bucket:write).
         # This basically means that if user loose the permissions on the related

{kinto-20.2.0.dist-info → kinto-20.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kinto
-Version: 20.2.0
+Version: 20.3.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation

{kinto-20.2.0.dist-info → kinto-20.3.0.dist-info}/RECORD

@@ -61,14 +61,14 @@ kinto/core/resource/viewset.py,sha256=Wo7mQwmI08IGnSetaqGF66fCqYPB1pDUdZa3U92NIi
 kinto/core/storage/__init__.py,sha256=Bo9q5PCDQ7KkBuBAHt7UoZFr5WfVWwEbFJjGsuq4Oo4,13704
 kinto/core/storage/exceptions.py,sha256=o10f7LohwyCHOTlR-dOdnB4us_MdCGOJUxZO8HZ3Akc,1304
 kinto/core/storage/generators.py,sha256=rxWN9hOfOsB-PLeryhPGO-aE670sivr3LFRIExImpxc,1829
-kinto/core/storage/memory.py,sha256=1M3kq6OC9MD-WR-nwm34U0R7AYcg54ZEAfGzmZX9_fY,19392
-kinto/core/storage/testing.py,sha256=Jg72tGWVS3pWNhIGY8YUTp99_wqSwL6-6TMhc2InpLM,77977
+kinto/core/storage/memory.py,sha256=DR6gpqSYh4oGPBX9x4-dGLNITZi33T0VODsNWsPtgho,19875
+kinto/core/storage/testing.py,sha256=4hGR6xc54M6H81IQb8XP7NP_iBP-X8m8PqKqixDs5nU,78411
 kinto/core/storage/utils.py,sha256=BHpohIKVOCtURjRbUT7O5AEhIKfSEFv-pfgRzq8Q5zs,1082
-kinto/core/storage/postgresql/__init__.py,sha256=GCnf1vWSrxGT1Dd9n9pLTK2064ukctLpKNHVQUMKd0Y,40157
+kinto/core/storage/postgresql/__init__.py,sha256=HMlUfTBsnsUQkLEhuhItU-9fgSOf9wTH6J09-1HW1fA,40157
 kinto/core/storage/postgresql/client.py,sha256=JTRxHK-8ipTXgs0E4PyiFiun-32yuzqFGsWTi-OuFfA,4270
 kinto/core/storage/postgresql/migrator.py,sha256=MQ_5aSrDi9-s2wlyyFyfhYP6HreCXjtlJzBI4b85u1I,3524
 kinto/core/storage/postgresql/pool.py,sha256=lOtclVagFqzzWbVxrGoWeKylpHlKdFgGz3Ef6cgGNJU,2219
-kinto/core/storage/postgresql/schema.sql,sha256=4q0NpjaX2GoiuGNnIVpMVuox9vQHGhh64qE4Z4a2FdQ,4095
+kinto/core/storage/postgresql/schema.sql,sha256=qRmx8NSwqedwozOi-Nn-pnipubxUWLCcYTRDMkt2B80,4216
 kinto/core/storage/postgresql/migrations/migration_001_002.sql,sha256=GVJIs8MGmZEyp1i0KjsmGKv1pLlBRckn0EWX6Kl6uQE,428
 kinto/core/storage/postgresql/migrations/migration_002_003.sql,sha256=zlSZpG_2L-wd8KXh3szmbYtoGjAOOwy2gH7RFMUd61w,1203
 kinto/core/storage/postgresql/migrations/migration_003_004.sql,sha256=OZSbH6Yt1PA2zba8iQWIguaTsnn3v7bFF-rbg2_teXY,530
@@ -90,6 +90,7 @@ kinto/core/storage/postgresql/migrations/migration_018_019.sql,sha256=XK6ex2jwQQ
 kinto/core/storage/postgresql/migrations/migration_019_020.sql,sha256=yDQDdzU65cgctKIeo1YqMrFi7aU2VGdBhVUpVQAQOgM,306
 kinto/core/storage/postgresql/migrations/migration_020_021.sql,sha256=cEgfGNDLH3RyLswxy9YZazZtOvidxNsi57z7SGR8VsQ,2369
 kinto/core/storage/postgresql/migrations/migration_021_022.sql,sha256=MUIAfgbBDQLy8JfklEs6ekK93nr1buVRtmBAHaaMXug,1993
+kinto/core/storage/postgresql/migrations/migration_022_023.sql,sha256=c1Fw10sxtaDqYJpxk1P9eZCtXP3uSmBY7-3IjZZLwjU,231
 kinto/core/views/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kinto/core/views/batch.py,sha256=sWNn67YqfTbiqwpbwbm1QyumcVGi8aNhlT5AtLToElI,5657
 kinto/core/views/errors.py,sha256=2uTy85UBQL1EDIMotzFBhVl14RmWwb3rupbsB0mycbs,6099
@@ -127,7 +128,7 @@ kinto/plugins/admin/build/assets/ttcn-cfg-B9xdYoR4.js,sha256=jTZ_XHKOChJZEt-FzJb
 kinto/plugins/admin/public/help.html,sha256=1hol7z5Sv0Xn3mYyEfPQWFOsrR24htlKhmnGA3rH8fs,958
 kinto/plugins/default_bucket/__init__.py,sha256=7TmBzFgW0gmYsetXr6Kqt9317XZ3PiB9gyCr-IBfmGg,7276
 kinto/plugins/history/__init__.py,sha256=s-RMNWaZlmBNd4sNsgJ-hDvMW4pPKUZ-sdZYubb0Kdo,1015
-kinto/plugins/history/listener.py,sha256=Tq5ZHpIOIzQs9yPXA1exEftPoYCuFQJvgxbaIb6XBrM,5033
+kinto/plugins/history/listener.py,sha256=WuMky5f3Ek6NaRQODTO-kLSD9mftJwCvxHeQ7Vy4_JA,7628
 kinto/plugins/history/views.py,sha256=NoBP-S7epeH5TLZZbIqfBmwMA2KaWmxP7lqPAS11BTU,2293
 kinto/plugins/openid/__init__.py,sha256=1Iv5SCa6vwEvoJkmGd45-TYm_mxz2okFj6u2VTNuVrk,4863
 kinto/plugins/openid/utils.py,sha256=n3KGS-ogXR2sg6j4QtdPe_DtEsqD7AGVT2K7vhl8yE8,273
@@ -140,9 +141,9 @@ kinto/views/contribute.py,sha256=PJoIMLj9_IszSjgZkaCd_TUjekDgNqjpmVTmRN9ztaA,983
 kinto/views/groups.py,sha256=jOq5fX0-4lwZE8k1q5HME2tU7x9052rtBPF7YqcJ-Qg,3181
 kinto/views/permissions.py,sha256=F0_eKx201WyLonXJ5vLdGKa9RcFKjvAihrEEhU1JuLw,9069
 kinto/views/records.py,sha256=lYfACW2L8qcQoyYBD5IX-fTPjFWmGp7GjHq_U4InlyE,5037
-kinto-20.2.0.dist-info/licenses/LICENSE,sha256=oNEIMTuTJzppR5ZEyi86yvvtSagveMYXTYFn56zF0Uk,561
-kinto-20.2.0.dist-info/METADATA,sha256=Yyclbux2PXFDz3EScffA46qvaSPWAv8HzBktTyPYOrc,8731
-kinto-20.2.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-kinto-20.2.0.dist-info/entry_points.txt,sha256=3KlqBWPKY81mrCe_oX0I5s1cRO7Q53nCLbnVr5P9LH4,85
-kinto-20.2.0.dist-info/top_level.txt,sha256=EG_YmbZL6FAug9VwopG7JtF9SvH_r0DEnFp-3twPPys,6
-kinto-20.2.0.dist-info/RECORD,,
+kinto-20.3.0.dist-info/licenses/LICENSE,sha256=oNEIMTuTJzppR5ZEyi86yvvtSagveMYXTYFn56zF0Uk,561
+kinto-20.3.0.dist-info/METADATA,sha256=WFcv90kV4fQrRr1nR8GQknnmY6Zo9_n1wk1Z-M7nYXg,8731
+kinto-20.3.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+kinto-20.3.0.dist-info/entry_points.txt,sha256=3KlqBWPKY81mrCe_oX0I5s1cRO7Q53nCLbnVr5P9LH4,85
+kinto-20.3.0.dist-info/top_level.txt,sha256=EG_YmbZL6FAug9VwopG7JtF9SvH_r0DEnFp-3twPPys,6
+kinto-20.3.0.dist-info/RECORD,,