kinto 20.5.0__py3-none-any.whl → 20.6.0__py3-none-any.whl

kinto/__main__.py

@@ -33,6 +33,7 @@ def main(args=None):
         "flush-cache",
         "version",
         "create-user",
+        "purge-deleted",
     )
     subparsers = parser.add_subparsers(
         title="subcommands",
@@ -128,6 +129,18 @@ def main(args=None):
             subparser.add_argument(
                 "-p", "--password", help="Superuser password", required=False, default=None
             )
+        elif command == "purge-deleted":
+            subparser.add_argument(
+                "resources",  # No '--' → positional
+                nargs="+",  # Accepts one or more
+                help="List of resources (e.g. record bucket group)",
+                default=["record"],
+            )
+            subparser.add_argument(
+                "max-retained",
+                help="The maximum number of tombstones to keep per resource and per parent",
+                type=int,
+            )
 
     # Parse command-line arguments
     parsed_args = vars(parser.parse_args(args))
@@ -208,6 +221,12 @@ def main(args=None):
         env = bootstrap(config_file, options={"command": "create-user"})
         return accounts_scripts.create_user(env, username=username, password=password)
 
+    elif which_command == "purge-deleted":
+        env = bootstrap(config_file)
+        return core_scripts.purge_deleted(
+            env, parsed_args["resources"], parsed_args["max-retained"]
+        )
+
     elif which_command == "start":
         pserve_argv = ["pserve"]
 

kinto/core/scripts.py

@@ -28,6 +28,21 @@ def migrate(env, dry_run=False):
                 getattr(registry, backend).initialize_schema(dry_run=dry_run)
 
 
+def purge_deleted(env, resource_names, max_retained):
+    logger.info("Keep only %r tombstones per parent and resource." % max_retained)
+
+    registry = env["registry"]
+
+    count = 0
+    for resource_name in resource_names:
+        count += registry.storage.purge_deleted(
+            resource_name=resource_name, parent_id="*", max_retained=max_retained
+        )
+
+    logger.info("%s tombstone(s) deleted." % count)
+    return 0
+
+
 def flush_cache(env):
     registry = env["registry"]
     registry.cache.flush()

kinto/core/storage/__init__.py

@@ -264,6 +264,7 @@ class StorageBase:
         resource_name,
         parent_id,
         before=None,
+        max_retained=None,
         id_field=DEFAULT_ID_FIELD,
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
@@ -273,7 +274,8 @@ class StorageBase:
         :param str resource_name: the resource name.
         :param str parent_id: the resource parent.
 
-        :param int before: Optionnal timestamp to limit deletion (exclusive)
+        :param int before: Optional timestamp to limit deletion (exclusive).
+        :param int max_count: Optional maximum of tombstones to keep per collection.
 
         :returns: The number of deleted objects.
         :rtype: int

kinto/core/storage/memory.py

@@ -284,9 +284,13 @@ class Storage(MemoryBasedStorage):
         resource_name,
         parent_id,
         before=None,
+        max_retained=None,
         id_field=DEFAULT_ID_FIELD,
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
+        if max_retained is not None and before is not None:
+            raise ValueError("`before` and `max_retained` are exclusive arguments. Pick one.")
+
         parent_id_match = re.compile(parent_id.replace("*", ".*"))
 
         timestamps_by_parent_id = {
@@ -312,7 +316,20 @@ class Storage(MemoryBasedStorage):
                 resources = {resource_name: resources[resource_name]}
             for resource, resource_objects in resources.items():
                 if before is None:
-                    kept = {}
+                    if max_retained is None:
+                        kept = {}
+                    else:
+                        kept = {
+                            key: value
+                            for i, (key, value) in enumerate(
+                                sorted(
+                                    resource_objects.items(),
+                                    key=lambda i: i[1]["last_modified"],
+                                    reverse=True,
+                                )
+                            )
+                            if i < max_retained
+                        }
                 else:
                     kept = {
                         key: value

kinto/core/storage/postgresql/__init__.py

@@ -598,6 +598,7 @@ class Storage(StorageBase, MigratorMixin):
         resource_name,
         parent_id,
         before=None,
+        max_retained=None,
         id_field=DEFAULT_ID_FIELD,
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
@@ -608,9 +609,39 @@ class Storage(StorageBase, MigratorMixin):
               {resource_name_filter}
               {conditions_filter}
         """
+
+        if max_retained is not None:
+            if before is not None:
+                raise ValueError("`before` and `max_retained` are exclusive arguments. Pick one.")
+
+            delete_tombstones = """
+            WITH ranked AS (
+                SELECT
+                    id AS objid,
+                    parent_id,
+                    resource_name,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY parent_id, resource_name
+                        ORDER BY last_modified DESC
+                    ) AS rn
+                FROM objects
+            )
+            DELETE FROM objects
+            WHERE id IN (
+                SELECT objid
+                FROM ranked
+                WHERE
+                    {parent_id_filter}
+                    {resource_name_filter}
+                    AND rn > :max_retained
+            )
+            """
+
         id_field = id_field or self.id_field
         modified_field = modified_field or self.modified_field
-        placeholders = dict(parent_id=parent_id, resource_name=resource_name)
+        placeholders = dict(
+            parent_id=parent_id, resource_name=resource_name, max_retained=max_retained
+        )
         # Safe strings
         safeholders = defaultdict(str)
         # Handle parent_id as a regex only if it contains *

kinto/core/storage/testing.py

@@ -1342,6 +1342,60 @@ class DeletedObjectsTest:
         count = self.storage.count_all(**self.storage_kw)
         self.assertEqual(count, 0)
 
+    def test_purge_deleted_does_not_support_before_and_max_retained(self):
+        self.assertRaises(
+            ValueError,
+            self.storage.purge_deleted,
+            resource_name="r",
+            parent_id="p",
+            before=42,
+            max_retained=1,
+        )
+
+    def test_purge_deleted_remove_with_max_count_per_collection(self):
+        cid1_kw = dict(parent_id="cid1", resource_name="one")
+        for i in range(5):
+            record = self.create_object(**cid1_kw)
+            self.storage.delete(object_id=record["id"], **cid1_kw)
+        cid2_kw = dict(parent_id="cid2", resource_name="one")
+        for i in range(4):
+            record = self.create_object(**cid2_kw)
+            self.storage.delete(object_id=record["id"], **cid2_kw)
+        cid1_other_kw = dict(parent_id="cid1", resource_name="other")
+        for i in range(5):
+            record = self.create_object(**cid1_other_kw)
+            self.storage.delete(object_id=record["id"], **cid1_other_kw)
+
+        # Consistency checks first.
+        objects_c1_before = self.storage.list_all(include_deleted=True, **cid1_kw)
+        self.assertEqual(len(objects_c1_before), 5)
+        objects_c2_before = self.storage.list_all(include_deleted=True, **cid2_kw)
+        self.assertEqual(len(objects_c2_before), 4)
+        objects_c1_other_before = self.storage.list_all(include_deleted=True, **cid1_other_kw)
+        self.assertEqual(len(objects_c1_other_before), 5)
+
+        num_removed = self.storage.purge_deleted(
+            resource_name="one", parent_id="*", max_retained=3
+        )
+
+        self.assertEqual(num_removed, 3)  # 2 for one/cid1, 1 for one/cid2, 0 for other/cid1
+
+        # It kept 3 tombstones for each resource/parent.
+        objects = self.storage.list_all(include_deleted=True, **cid1_kw)
+        self.assertEqual(len(objects), 3)
+        self.assertEqual(
+            min(obj["last_modified"] for obj in objects), objects_c1_before[2]["last_modified"]
+        )
+
+        objects = self.storage.list_all(include_deleted=True, **cid2_kw)
+        self.assertEqual(len(objects), 3)
+        self.assertNotEqual(
+            min(obj["last_modified"] for obj in objects), objects_c2_before[2]["last_modified"]
+        )
+
+        objects = self.storage.list_all(include_deleted=True, **cid1_other_kw)
+        self.assertEqual(len(objects), 5)
+
     #
     # Sorting
     #

{kinto-20.5.0.dist-info → kinto-20.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kinto
-Version: 20.5.0
+Version: 20.6.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation

{kinto-20.5.0.dist-info → kinto-20.6.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 kinto/__init__.py,sha256=XG7cOyHN0ipQeOSPMW-XFax7uFgAjnCd3Slq0jSFGC4,3299
-kinto/__main__.py,sha256=0vBSUT2pAMg78HyRpfGbk3R6Z_fpg12pcGi3pi1XJ1c,7181
+kinto/__main__.py,sha256=6XDjWYGtJ36qepn9Of4xL9zc35TfdSeEvHh3s26Aw9c,7913
 kinto/authorization.py,sha256=i3ttdPTFGzE9CqUQmfC4rR_6dDZJu0jWJMLGl_jFzIE,4919
 kinto/contribute.json,sha256=HT9QVB8rA8jWIREQoqWfMibfJXMAzbRsixW8F6O6cQY,792
 kinto/events.py,sha256=NMPvKUdbi25aYHhu9svzQsrEZMa9nyO4mtuMZC5871Q,85
@@ -16,7 +16,7 @@ kinto/core/initialization.py,sha256=ZjmCKc8MYZXv63W2mv--fY8rXSLAnJa7RtCYdfK4jsg,
 kinto/core/metrics.py,sha256=Y6Mt4PUzy2-oudeGr_oCmtX8nIR4SZkzUlPxr58jr-g,2619
 kinto/core/openapi.py,sha256=92sZviff4NCxN0jMnu5lPUnF5iQbrKMGy7Cegf-VAME,3876
 kinto/core/schema.py,sha256=d5L5TQynRYJPkZ8Mu2X7F72xEh6SKDbrHK1CNTdOf2E,3646
-kinto/core/scripts.py,sha256=5HSq5QAuin7HuU6icNYkPisny-4JpcdBvjf8X4JImrE,978
+kinto/core/scripts.py,sha256=02SXVjo579W82AsDF8dyVCRxYVcrMFkjjaNVIgLChh0,1412
 kinto/core/statsd.py,sha256=2f4s2opiHVdrA02ZlBa5pxIHaEjPuG8tdVLsmdII27s,64
 kinto/core/testing.py,sha256=kZ-75EiiZwTNDBpHZyKBBHute6jEmUwXd7nRMK9kwr4,5908
 kinto/core/utils.py,sha256=EDlZfHsgcaUvxg-AG_dn9IcQCq7j7RxpBjnFrx8b11E,17069
@@ -58,13 +58,13 @@ kinto/core/resource/__init__.py,sha256=B-1X3nPzIZOiN8Dc_tSersAP-Wn4jXf3GcrctN2Ck
 kinto/core/resource/model.py,sha256=xjZ6shnhelXCdWvgw6yeOWXodxiKMm9iyDqLTk0i8Bs,15626
 kinto/core/resource/schema.py,sha256=EhPKDMlBjx60hXztMvywfo4IrRfwaZ3V-7sY_sl-BYk,16126
 kinto/core/resource/viewset.py,sha256=Wo7mQwmI08IGnSetaqGF66fCqYPB1pDUdZa3U92NIic,7613
-kinto/core/storage/__init__.py,sha256=Bo9q5PCDQ7KkBuBAHt7UoZFr5WfVWwEbFJjGsuq4Oo4,13704
+kinto/core/storage/__init__.py,sha256=xELryXjUvmYon4qS3GJB6nCgVcLyGJoV0rJ5EWjGiVY,13816
 kinto/core/storage/exceptions.py,sha256=o10f7LohwyCHOTlR-dOdnB4us_MdCGOJUxZO8HZ3Akc,1304
 kinto/core/storage/generators.py,sha256=rxWN9hOfOsB-PLeryhPGO-aE670sivr3LFRIExImpxc,1829
-kinto/core/storage/memory.py,sha256=DR6gpqSYh4oGPBX9x4-dGLNITZi33T0VODsNWsPtgho,19875
-kinto/core/storage/testing.py,sha256=4hGR6xc54M6H81IQb8XP7NP_iBP-X8m8PqKqixDs5nU,78411
+kinto/core/storage/memory.py,sha256=--08jHpnFMtcyz1CD10ZFni36qe-Wzrwprl-VVI84IE,20630
+kinto/core/storage/testing.py,sha256=5t61Li0M6XXCNu0vRMaT-_GaBNz7TPZ27eXTO1Bf9vI,80766
 kinto/core/storage/utils.py,sha256=BHpohIKVOCtURjRbUT7O5AEhIKfSEFv-pfgRzq8Q5zs,1082
-kinto/core/storage/postgresql/__init__.py,sha256=HMlUfTBsnsUQkLEhuhItU-9fgSOf9wTH6J09-1HW1fA,40157
+kinto/core/storage/postgresql/__init__.py,sha256=egK7K5oQgRF03LxdQAzPj4L4yeysDVkkPBKZg5Ab4mQ,41111
 kinto/core/storage/postgresql/client.py,sha256=JTRxHK-8ipTXgs0E4PyiFiun-32yuzqFGsWTi-OuFfA,4270
 kinto/core/storage/postgresql/migrator.py,sha256=MQ_5aSrDi9-s2wlyyFyfhYP6HreCXjtlJzBI4b85u1I,3524
 kinto/core/storage/postgresql/pool.py,sha256=lOtclVagFqzzWbVxrGoWeKylpHlKdFgGz3Ef6cgGNJU,2219
@@ -141,9 +141,9 @@ kinto/views/contribute.py,sha256=PJoIMLj9_IszSjgZkaCd_TUjekDgNqjpmVTmRN9ztaA,983
 kinto/views/groups.py,sha256=jOq5fX0-4lwZE8k1q5HME2tU7x9052rtBPF7YqcJ-Qg,3181
 kinto/views/permissions.py,sha256=F0_eKx201WyLonXJ5vLdGKa9RcFKjvAihrEEhU1JuLw,9069
 kinto/views/records.py,sha256=lYfACW2L8qcQoyYBD5IX-fTPjFWmGp7GjHq_U4InlyE,5037
-kinto-20.5.0.dist-info/licenses/LICENSE,sha256=oNEIMTuTJzppR5ZEyi86yvvtSagveMYXTYFn56zF0Uk,561
-kinto-20.5.0.dist-info/METADATA,sha256=8hgp9glzPTbNRIWX6r2RXIy5xX_-V72uFHyjo27XlxQ,8731
-kinto-20.5.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-kinto-20.5.0.dist-info/entry_points.txt,sha256=3KlqBWPKY81mrCe_oX0I5s1cRO7Q53nCLbnVr5P9LH4,85
-kinto-20.5.0.dist-info/top_level.txt,sha256=EG_YmbZL6FAug9VwopG7JtF9SvH_r0DEnFp-3twPPys,6
-kinto-20.5.0.dist-info/RECORD,,
+kinto-20.6.0.dist-info/licenses/LICENSE,sha256=oNEIMTuTJzppR5ZEyi86yvvtSagveMYXTYFn56zF0Uk,561
+kinto-20.6.0.dist-info/METADATA,sha256=ajUNuc134EmgReYQWZIKBRKtn6gWQEeIEEj8O1MQ3KU,8731
+kinto-20.6.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+kinto-20.6.0.dist-info/entry_points.txt,sha256=3KlqBWPKY81mrCe_oX0I5s1cRO7Q53nCLbnVr5P9LH4,85
+kinto-20.6.0.dist-info/top_level.txt,sha256=EG_YmbZL6FAug9VwopG7JtF9SvH_r0DEnFp-3twPPys,6
+kinto-20.6.0.dist-info/RECORD,,