kinto 20.0.0__py3-none-any.whl → 20.4.0__py3-none-any.whl

kinto/config/kinto.tpl

@@ -245,7 +245,7 @@ keys = root, kinto
 keys = console
 
 [formatters]
-keys = color
+keys = color, json
 
 [logger_root]
 level = INFO
@@ -258,10 +258,13 @@ qualname = kinto
 propagate = 0
 
 [handler_console]
-class = StreamHandler
+class = kinto.core.StreamHandlerWithRequestID
 args = (sys.stderr,)
 level = NOTSET
 formatter = color
 
 [formatter_color]
 class = logging_color_formatter.ColorFormatter
+
+[formatter_json]
+class = kinto.core.JsonLogFormatter

kinto/core/__init__.py

@@ -151,6 +151,20 @@ class JsonLogFormatter(dockerflow_logging.JsonLogFormatter):
         self.logger_name = logger_name
 
 
+class StreamHandlerWithRequestID(logging.StreamHandler):
+    """
+    A custom StreamHandler that adds the Dockerflow's `RequestIdLogFilter`.
+
+    Defining a custom handler seems to be the only way to bypass the fact that
+    ``logging.config.fileConfig()`` does not load filters from ``.ini`` files.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        filter_ = dockerflow_logging.RequestIdLogFilter()
+        self.addFilter(filter_)
+
+
 def get_user_info(request):
     # Default user info (shown in hello view for example).
     user_info = {"id": request.prefixed_userid, "principals": request.prefixed_principals}

kinto/core/initialization.py

@@ -4,9 +4,9 @@ import re
 import urllib.parse
 import warnings
 from datetime import datetime
-from secrets import token_hex
 
 from dateutil import parser as dateparser
+from dockerflow.logging import get_or_generate_request_id, request_id_context
 from pyramid.events import ApplicationCreated, NewRequest, NewResponse
 from pyramid.exceptions import ConfigurationError
 from pyramid.httpexceptions import (
@@ -374,12 +374,15 @@ def setup_logging(config):
                 message="Invalid URL path.",
             )
 
+        rid = get_or_generate_request_id(headers=request.headers)
+        request_id_context.set(rid)
+
         request.log_context(
             agent=request.headers.get("User-Agent"),
             path=request_path,
             method=request.method,
             lang=request.headers.get("Accept-Language"),
-            rid=request.headers.get("X-Request-Id", token_hex(16)),
+            rid=rid,
             errno=0,
         )
         qs = dict(errors.request_GET(request))

kinto/core/storage/__init__.py

@@ -87,6 +87,21 @@ class StorageBase:
         """
         raise NotImplementedError
 
+    def all_resources_timestamps(self, resource_name):
+        """Get the highest timestamp of every objects in this `resource_name` for
+        each `parent_id`.
+
+        .. note::
+
+            This should take deleted objects into account.
+
+        :param str resource_name: the resource name.
+
+        :returns: the latest timestamp of the resource by `parent_id`.
+        :rtype: dict[str, int]
+        """
+        raise NotImplementedError
+
     def create(
         self,
         resource_name,

kinto/core/storage/memory.py

@@ -153,6 +153,10 @@ class Storage(MemoryBasedStorage):
             raise exceptions.ReadonlyError(message=error_msg)
         return self.bump_and_store_timestamp(resource_name, parent_id)
 
+    @synchronized
+    def all_resources_timestamps(self, resource_name):
+        return {k: v[resource_name] for k, v in self._timestamps.items() if resource_name in v}
+
     def bump_and_store_timestamp(
         self, resource_name, parent_id, obj=None, modified_field=None, last_modified=None
     ):
@@ -284,13 +288,26 @@ class Storage(MemoryBasedStorage):
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
         parent_id_match = re.compile(parent_id.replace("*", ".*"))
-        by_parent_id = {
+
+        timestamps_by_parent_id = {
             pid: resources
-            for pid, resources in self._cemetery.items()
+            for pid, resources in self._timestamps.items()
             if parent_id_match.match(pid)
         }
+        if resource_name is not None:
+            for pid, resources in timestamps_by_parent_id.items():
+                del self._timestamps[pid][resource_name]
+        else:
+            for pid, resources in timestamps_by_parent_id.items():
+                del self._timestamps[pid]
+
         num_deleted = 0
-        for pid, resources in by_parent_id.items():
+        tombstones_by_parent_id = {
+            pid: resources
+            for pid, resources in self._cemetery.items()
+            if parent_id_match.match(pid)
+        }
+        for pid, resources in tombstones_by_parent_id.items():
             if resource_name is not None:
                 resources = {resource_name: resources[resource_name]}
             for resource, resource_objects in resources.items():

kinto/core/storage/postgresql/__init__.py

@@ -79,7 +79,7 @@ class Storage(StorageBase, MigratorMixin):
 
     # MigratorMixin attributes.
     name = "storage"
-    schema_version = 22
+    schema_version = 23
     schema_file = os.path.join(HERE, "schema.sql")
     migrations_directory = os.path.join(HERE, "migrations")
 
@@ -247,6 +247,36 @@ class Storage(StorageBase, MigratorMixin):
 
         return obj.last_epoch
 
+    def all_resources_timestamps(self, resource_name):
+        query = """
+        WITH existing_timestamps AS (
+          -- Timestamp of latest object by parent_id.
+          (
+            SELECT parent_id, MAX(last_modified) AS last_modified
+            FROM objects
+            WHERE resource_name = :resource_name
+            GROUP BY parent_id
+          )
+          -- Timestamp of resources without sub-objects.
+          UNION
+          (
+            SELECT parent_id, last_modified
+            FROM timestamps
+            WHERE resource_name = :resource_name
+          )
+        )
+        SELECT parent_id, MAX(as_epoch(last_modified)) AS last_modified
+          FROM existing_timestamps
+          GROUP BY parent_id
+          ORDER BY last_modified DESC
+        """
+        with self.client.connect(readonly=True) as conn:
+            result = conn.execute(sa.text(query), dict(resource_name=resource_name))
+            rows = result.fetchmany(self._max_fetch_size + 1)
+
+        results = {r[0]: r[1] for r in rows}
+        return results
+
     @deprecate_kwargs({"collection_id": "resource_name", "record": "obj"})
     def create(
         self,

kinto/core/storage/postgresql/migrations/migration_022_023.sql

@@ -0,0 +1,5 @@
+CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
+    ON objects(resource_name, parent_id, deleted);
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');

kinto/core/storage/postgresql/schema.sql

@@ -47,7 +47,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_objects_parent_id_resource_name_last_modif
     ON objects(parent_id, resource_name, last_modified DESC);
 CREATE INDEX IF NOT EXISTS idx_objects_last_modified_epoch
     ON objects(as_epoch(last_modified));
-
+CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
+    ON objects(resource_name, parent_id, deleted);
 
 CREATE TABLE IF NOT EXISTS timestamps (
   parent_id TEXT NOT NULL COLLATE "C",
@@ -131,4 +132,4 @@ INSERT INTO metadata (name, value) VALUES ('created_at', NOW()::TEXT);
 
 -- Set storage schema version.
 -- Should match ``kinto.core.storage.postgresql.PostgreSQL.schema_version``
-INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '22');
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');

kinto/core/storage/testing.py

@@ -783,6 +783,39 @@ class TimestampsTest:
         after = self.storage.resource_timestamp(**self.storage_kw)
         self.assertTrue(before < after)
 
+    def test_all_timestamps_by_parent_id(self):
+        self.storage.create(obj={"id": "main"}, resource_name="bucket", parent_id="")
+        self.storage.create(obj={"id": "cid1"}, resource_name="collection", parent_id="/main")
+        self.storage.create(obj={"id": "cid2"}, resource_name="collection", parent_id="/main")
+        self.storage.create(obj={}, resource_name="record", parent_id="/main/cid2")
+        self.storage.create(obj={}, resource_name="record", parent_id="/main/cid2")
+
+        self.assertEqual(
+            {
+                "": self.storage.resource_timestamp(resource_name="bucket", parent_id=""),
+            },
+            self.storage.all_resources_timestamps(resource_name="bucket"),
+        )
+        self.assertEqual(
+            {
+                "/main": self.storage.resource_timestamp(
+                    resource_name="collection", parent_id="/main"
+                ),
+            },
+            self.storage.all_resources_timestamps(resource_name="collection"),
+        )
+        self.assertEqual(
+            {
+                "/main/cid1": self.storage.resource_timestamp(
+                    resource_name="record", parent_id="/main/cid1"
+                ),
+                "/main/cid2": self.storage.resource_timestamp(
+                    resource_name="record", parent_id="/main/cid2"
+                ),
+            },
+            self.storage.all_resources_timestamps(resource_name="record"),
+        )
+
     @skip_if_ci
     def test_timestamps_are_unique(self):  # pragma: no cover
         obtained = []
@@ -1263,6 +1296,9 @@ class DeletedObjectsTest:
         self.create_object(parent_id="/abc/a", resource_name="c")
         self.create_object(parent_id="/efg", resource_name="c")
 
+        all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
+        self.assertEqual(set(all_timestamps.keys()), {"/abc/a", "/efg"})
+
         before1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
         # Different parent_id with object.
         before2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
@@ -1272,11 +1308,15 @@ class DeletedObjectsTest:
         self.storage.delete_all(parent_id="/abc/*", resource_name=None, with_deleted=False)
         self.storage.purge_deleted(parent_id="/abc/*", resource_name=None)
 
+        all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
+        self.assertEqual(set(all_timestamps.keys()), {"/efg", "/ijk"})
+
+        time.sleep(0.002)  # make sure we don't recreate timestamps at same msec.
         after1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
         after2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
         after3 = self.storage.resource_timestamp(parent_id="/ijk", resource_name="c")
 
-        self.assertNotEqual(before1, after1)
+        self.assertNotEqual(before1, after1)  # timestamp was removed, it will differ.
         self.assertEqual(before2, after2)
         self.assertEqual(before3, after3)
 

kinto/plugins/admin/VERSION

@@ -1 +1 @@
-3.6.0
+3.7.1

kinto/plugins/admin/build/VERSION

@@ -1 +1 @@
-3.6.0
+3.7.1