kinto 20.5.0__tar.gz → 20.6.0__tar.gz

{kinto-20.5.0 → kinto-20.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kinto
-Version: 20.5.0
+Version: 20.6.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation

{kinto-20.5.0 → kinto-20.6.0}/docs/commandline.rst

@@ -96,3 +96,23 @@ For example:
 ::
 
     kinto flush-cache --ini kinto.ini
+
+
+Purge Deleted
+-------------
+
+Delete tombstones from the database, and keep a certain number of them per collection.
+
+This is **not** recommended in production, unless you really know what you are doing.
+
+It will break partial synchronization on clients that didn't have the chance to apply these deletions locally.
+
+::
+
+    usage: kinto purge-deleted [-h] [--ini INI_FILE] [-q] [-v] resources [resources ...] max-retained
+
+For example:
+
+::
+
+    kinto --ini=config/postgresql.ini purge-deleted bucket,collection,record 10000

{kinto-20.5.0 → kinto-20.6.0}/kinto/__main__.py

@@ -33,6 +33,7 @@ def main(args=None):
         "flush-cache",
         "version",
         "create-user",
+        "purge-deleted",
     )
     subparsers = parser.add_subparsers(
         title="subcommands",
@@ -128,6 +129,18 @@ def main(args=None):
             subparser.add_argument(
                 "-p", "--password", help="Superuser password", required=False, default=None
             )
+        elif command == "purge-deleted":
+            subparser.add_argument(
+                "resources",  # No '--' → positional
+                nargs="+",  # Accepts one or more
+                help="List of resources (e.g. record bucket group)",
+                default=["record"],
+            )
+            subparser.add_argument(
+                "max-retained",
+                help="The maximum number of tombstones to keep per resource and per parent",
+                type=int,
+            )
 
     # Parse command-line arguments
     parsed_args = vars(parser.parse_args(args))
@@ -208,6 +221,12 @@ def main(args=None):
         env = bootstrap(config_file, options={"command": "create-user"})
         return accounts_scripts.create_user(env, username=username, password=password)
 
+    elif which_command == "purge-deleted":
+        env = bootstrap(config_file)
+        return core_scripts.purge_deleted(
+            env, parsed_args["resources"], parsed_args["max-retained"]
+        )
+
     elif which_command == "start":
         pserve_argv = ["pserve"]
 

{kinto-20.5.0 → kinto-20.6.0}/kinto/core/scripts.py

@@ -28,6 +28,21 @@ def migrate(env, dry_run=False):
                 getattr(registry, backend).initialize_schema(dry_run=dry_run)
 
 
+def purge_deleted(env, resource_names, max_retained):
+    logger.info("Keep only %r tombstones per parent and resource." % max_retained)
+
+    registry = env["registry"]
+
+    count = 0
+    for resource_name in resource_names:
+        count += registry.storage.purge_deleted(
+            resource_name=resource_name, parent_id="*", max_retained=max_retained
+        )
+
+    logger.info("%s tombstone(s) deleted." % count)
+    return 0
+
+
 def flush_cache(env):
     registry = env["registry"]
     registry.cache.flush()

{kinto-20.5.0 → kinto-20.6.0}/kinto/core/storage/__init__.py

@@ -264,6 +264,7 @@ class StorageBase:
         resource_name,
         parent_id,
         before=None,
+        max_retained=None,
         id_field=DEFAULT_ID_FIELD,
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
@@ -273,7 +274,8 @@ class StorageBase:
         :param str resource_name: the resource name.
         :param str parent_id: the resource parent.
 
-        :param int before: Optionnal timestamp to limit deletion (exclusive)
+        :param int before: Optional timestamp to limit deletion (exclusive).
+        :param int max_count: Optional maximum of tombstones to keep per collection.
 
         :returns: The number of deleted objects.
         :rtype: int

{kinto-20.5.0 → kinto-20.6.0}/kinto/core/storage/memory.py

@@ -284,9 +284,13 @@ class Storage(MemoryBasedStorage):
         resource_name,
         parent_id,
         before=None,
+        max_retained=None,
         id_field=DEFAULT_ID_FIELD,
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
+        if max_retained is not None and before is not None:
+            raise ValueError("`before` and `max_retained` are exclusive arguments. Pick one.")
+
         parent_id_match = re.compile(parent_id.replace("*", ".*"))
 
         timestamps_by_parent_id = {
@@ -312,7 +316,20 @@ class Storage(MemoryBasedStorage):
                 resources = {resource_name: resources[resource_name]}
             for resource, resource_objects in resources.items():
                 if before is None:
-                    kept = {}
+                    if max_retained is None:
+                        kept = {}
+                    else:
+                        kept = {
+                            key: value
+                            for i, (key, value) in enumerate(
+                                sorted(
+                                    resource_objects.items(),
+                                    key=lambda i: i[1]["last_modified"],
+                                    reverse=True,
+                                )
+                            )
+                            if i < max_retained
+                        }
                 else:
                     kept = {
                         key: value

{kinto-20.5.0 → kinto-20.6.0}/kinto/core/storage/postgresql/__init__.py

@@ -598,6 +598,7 @@ class Storage(StorageBase, MigratorMixin):
         resource_name,
         parent_id,
         before=None,
+        max_retained=None,
         id_field=DEFAULT_ID_FIELD,
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
@@ -608,9 +609,39 @@ class Storage(StorageBase, MigratorMixin):
               {resource_name_filter}
               {conditions_filter}
         """
+
+        if max_retained is not None:
+            if before is not None:
+                raise ValueError("`before` and `max_retained` are exclusive arguments. Pick one.")
+
+            delete_tombstones = """
+            WITH ranked AS (
+                SELECT
+                    id AS objid,
+                    parent_id,
+                    resource_name,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY parent_id, resource_name
+                        ORDER BY last_modified DESC
+                    ) AS rn
+                FROM objects
+            )
+            DELETE FROM objects
+            WHERE id IN (
+                SELECT objid
+                FROM ranked
+                WHERE
+                    {parent_id_filter}
+                    {resource_name_filter}
+                    AND rn > :max_retained
+            )
+            """
+
         id_field = id_field or self.id_field
         modified_field = modified_field or self.modified_field
-        placeholders = dict(parent_id=parent_id, resource_name=resource_name)
+        placeholders = dict(
+            parent_id=parent_id, resource_name=resource_name, max_retained=max_retained
+        )
         # Safe strings
         safeholders = defaultdict(str)
         # Handle parent_id as a regex only if it contains *

{kinto-20.5.0 → kinto-20.6.0}/kinto/core/storage/testing.py

@@ -1342,6 +1342,60 @@ class DeletedObjectsTest:
         count = self.storage.count_all(**self.storage_kw)
         self.assertEqual(count, 0)
 
+    def test_purge_deleted_does_not_support_before_and_max_retained(self):
+        self.assertRaises(
+            ValueError,
+            self.storage.purge_deleted,
+            resource_name="r",
+            parent_id="p",
+            before=42,
+            max_retained=1,
+        )
+
+    def test_purge_deleted_remove_with_max_count_per_collection(self):
+        cid1_kw = dict(parent_id="cid1", resource_name="one")
+        for i in range(5):
+            record = self.create_object(**cid1_kw)
+            self.storage.delete(object_id=record["id"], **cid1_kw)
+        cid2_kw = dict(parent_id="cid2", resource_name="one")
+        for i in range(4):
+            record = self.create_object(**cid2_kw)
+            self.storage.delete(object_id=record["id"], **cid2_kw)
+        cid1_other_kw = dict(parent_id="cid1", resource_name="other")
+        for i in range(5):
+            record = self.create_object(**cid1_other_kw)
+            self.storage.delete(object_id=record["id"], **cid1_other_kw)
+
+        # Consistency checks first.
+        objects_c1_before = self.storage.list_all(include_deleted=True, **cid1_kw)
+        self.assertEqual(len(objects_c1_before), 5)
+        objects_c2_before = self.storage.list_all(include_deleted=True, **cid2_kw)
+        self.assertEqual(len(objects_c2_before), 4)
+        objects_c1_other_before = self.storage.list_all(include_deleted=True, **cid1_other_kw)
+        self.assertEqual(len(objects_c1_other_before), 5)
+
+        num_removed = self.storage.purge_deleted(
+            resource_name="one", parent_id="*", max_retained=3
+        )
+
+        self.assertEqual(num_removed, 3)  # 2 for one/cid1, 1 for one/cid2, 0 for other/cid1
+
+        # It kept 3 tombstones for each resource/parent.
+        objects = self.storage.list_all(include_deleted=True, **cid1_kw)
+        self.assertEqual(len(objects), 3)
+        self.assertEqual(
+            min(obj["last_modified"] for obj in objects), objects_c1_before[2]["last_modified"]
+        )
+
+        objects = self.storage.list_all(include_deleted=True, **cid2_kw)
+        self.assertEqual(len(objects), 3)
+        self.assertNotEqual(
+            min(obj["last_modified"] for obj in objects), objects_c2_before[2]["last_modified"]
+        )
+
+        objects = self.storage.list_all(include_deleted=True, **cid1_other_kw)
+        self.assertEqual(len(objects), 5)
+
     #
     # Sorting
     #

{kinto-20.5.0 → kinto-20.6.0}/kinto.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kinto
-Version: 20.5.0
+Version: 20.6.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation

{kinto-20.5.0 → kinto-20.6.0}/tests/core/test_scripts.py

@@ -45,3 +45,20 @@ class InitSchemaTest(unittest.TestCase):
         scripts.flush_cache({"registry": self.registry})
         reg = self.registry
         reg.cache.flush.assert_called_with()
+
+
+class PurgeDeletedTest(unittest.TestCase):
+    def setUp(self):
+        self.registry = mock.MagicMock()
+
+    def test_purge_deleted(self):
+        code = scripts.purge_deleted(
+            {"registry": self.registry}, resource_names=["A", "B"], max_retained=42
+        )
+        assert code == 0
+        self.registry.storage.purge_deleted.assert_any_call(
+            parent_id="*", resource_name="A", max_retained=42
+        )
+        self.registry.storage.purge_deleted.assert_any_call(
+            parent_id="*", resource_name="B", max_retained=42
+        )

{kinto-20.5.0 → kinto-20.6.0}/tests/test_main.py

@@ -312,3 +312,37 @@ class TestMain(unittest.TestCase):
             res = main(["flush-cache", "--ini", TEMP_KINTO_INI])
             assert res == 0
             assert mocked_cache_script.call_count == 1
+
+    def test_cli_purge_deleted_runs_purge_deleted_script(self):
+        with mock.patch("kinto.core.scripts.purge_deleted") as purge_deleted:
+            purge_deleted.return_value = mock.sentinel.purge_deleted
+            main(
+                [
+                    "init",
+                    "--ini",
+                    TEMP_KINTO_INI,
+                    "--backend",
+                    "memory",
+                    "--cache-backend",
+                    "memory",
+                ]
+            )
+            res = main(["purge-deleted", "--ini", TEMP_KINTO_INI, "record,bucket", "42"])
+            assert res == mock.sentinel.purge_deleted
+            assert purge_deleted.call_count == 1
+
+    def test_cli_purge_deleted_fails_if_no_max_retained(self):
+        with mock.patch("kinto.core.scripts.purge_deleted"):
+            main(
+                [
+                    "init",
+                    "--ini",
+                    TEMP_KINTO_INI,
+                    "--backend",
+                    "memory",
+                    "--cache-backend",
+                    "memory",
+                ]
+            )
+            with self.assertRaises(SystemExit):
+                main(["purge-deleted", "--ini", TEMP_KINTO_INI, "record,bucket"])