kinto 20.0.0__tar.gz → 20.4.0__tar.gz

{kinto-20.0.0 → kinto-20.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: kinto
-Version: 20.0.0
+Version: 20.4.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation
@@ -70,6 +70,7 @@ Provides-Extra: dev
 Requires-Dist: build; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
 
 Kinto
 =====

{kinto-20.0.0 → kinto-20.4.0}/constraints.txt

@@ -73,7 +73,7 @@ markupsafe==2.1.4
     # via werkzeug
 msgpack==1.0.7
     # via bravado-core
-newrelic==10.6.0
+newrelic==10.8.1
     # via -r constraints.in
 packaging==23.2
     # via
@@ -88,7 +88,7 @@ plaster==1.1.2
     #   pyramid
 plaster-pastedeploy==1.0.1
     # via pyramid
-playwright==1.50.0
+playwright==1.51.0
     # via -r constraints.in
 pluggy==1.5.0
     # via pytest
@@ -116,7 +116,7 @@ pytest==8.3.5
     #   pytest-cov
 pytest-cache==1.0
     # via -r constraints.in
-pytest-cov==6.0.0
+pytest-cov==6.1.1
     # via -r constraints.in
 python-dateutil==2.9.0.post0
     # via
@@ -149,7 +149,7 @@ rpds-py==0.17.1
     # via
     #   jsonschema
     #   referencing
-ruff==0.9.9
+ruff==0.11.4
     # via -r constraints.in
 sentry-sdk==2.8.0
     # via -r constraints.in
@@ -162,7 +162,7 @@ six==1.16.0
     #   rfc3339-validator
 soupsieve==2.5
     # via beautifulsoup4
-sqlalchemy==2.0.38
+sqlalchemy==2.0.40
     # via
     #   -r constraints.in
     #   sentry-sdk

{kinto-20.0.0 → kinto-20.4.0}/docs/api/1.x/history.rst

@@ -195,3 +195,21 @@ It is possible to exclude certain resources from being tracked by history using
 
     kinto.history.exclude_resources = /buckets/preview
                                       /buckets/signed/collections/certificates
+
+Or certain users:
+
+.. code-block:: ini
+
+    kinto.history.exclude_user_ids = account:quicksuggest
+
+In order to limit the size of the history length of each resource:
+
+.. code-block:: ini
+
+    kinto.history.auto_trim_max_count = 100
+
+In order to limit the size of the history length of each resource only for certain users:
+
+.. code-block:: ini
+
+    kinto.history.auto_trim_user_ids = account:quicksuggest

{kinto-20.0.0 → kinto-20.4.0}/docs/configuration/settings.rst

@@ -456,6 +456,19 @@ Example output:
     {"Pid": 19240, "Type": "root", "Timestamp": 1489067817834153984, "Severity": 4, "Hostname": "pluo", "Logger": "%", "EnvVersion": "2.0", "Fields": {"perm": "read", "userid": "ldap:john@corp.com", "message": "Permission not granted.", "uri": "/buckets/123"}}
 
 
+Request IDs
+:::::::::::
+
+In order to add requests IDs to log messages, set the provided handler:
+
+.. code-block:: ini
+
+    [handler_console]
+    class = kinto.core.StreamHandlerWithRequestID
+
+It will read the value from the ``X-Request-Id`` request header if present, or generate a random string when the request is received otherwise.
+
+
 .. _handling-exceptions-with-sentry:
 
 Handling exceptions with Sentry

{kinto-20.0.0 → kinto-20.4.0}/docs/requirements.txt

@@ -8,4 +8,4 @@ mock==5.2.0
 webtest==3.0.4
 pyramid==2.0.2
 python-rapidjson==1.20
-SQLAlchemy==2.0.38
+SQLAlchemy==2.0.40

{kinto-20.0.0 → kinto-20.4.0}/kinto/config/kinto.tpl

@@ -245,7 +245,7 @@ keys = root, kinto
 keys = console
 
 [formatters]
-keys = color
+keys = color, json
 
 [logger_root]
 level = INFO
@@ -258,10 +258,13 @@ qualname = kinto
 propagate = 0
 
 [handler_console]
-class = StreamHandler
+class = kinto.core.StreamHandlerWithRequestID
 args = (sys.stderr,)
 level = NOTSET
 formatter = color
 
 [formatter_color]
 class = logging_color_formatter.ColorFormatter
+
+[formatter_json]
+class = kinto.core.JsonLogFormatter

{kinto-20.0.0 → kinto-20.4.0}/kinto/core/__init__.py

@@ -151,6 +151,20 @@ class JsonLogFormatter(dockerflow_logging.JsonLogFormatter):
         self.logger_name = logger_name
 
 
+class StreamHandlerWithRequestID(logging.StreamHandler):
+    """
+    A custom StreamHandler that adds the Dockerflow's `RequestIdLogFilter`.
+
+    Defining a custom handler seems to be the only way to bypass the fact that
+    ``logging.config.fileConfig()`` does not load filters from ``.ini`` files.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        filter_ = dockerflow_logging.RequestIdLogFilter()
+        self.addFilter(filter_)
+
+
 def get_user_info(request):
     # Default user info (shown in hello view for example).
     user_info = {"id": request.prefixed_userid, "principals": request.prefixed_principals}

{kinto-20.0.0 → kinto-20.4.0}/kinto/core/initialization.py

@@ -4,9 +4,9 @@ import re
 import urllib.parse
 import warnings
 from datetime import datetime
-from secrets import token_hex
 
 from dateutil import parser as dateparser
+from dockerflow.logging import get_or_generate_request_id, request_id_context
 from pyramid.events import ApplicationCreated, NewRequest, NewResponse
 from pyramid.exceptions import ConfigurationError
 from pyramid.httpexceptions import (
@@ -374,12 +374,15 @@ def setup_logging(config):
                 message="Invalid URL path.",
             )
 
+        rid = get_or_generate_request_id(headers=request.headers)
+        request_id_context.set(rid)
+
         request.log_context(
             agent=request.headers.get("User-Agent"),
             path=request_path,
             method=request.method,
             lang=request.headers.get("Accept-Language"),
-            rid=request.headers.get("X-Request-Id", token_hex(16)),
+            rid=rid,
             errno=0,
         )
         qs = dict(errors.request_GET(request))

{kinto-20.0.0 → kinto-20.4.0}/kinto/core/storage/__init__.py

@@ -87,6 +87,21 @@ class StorageBase:
         """
         raise NotImplementedError
 
+    def all_resources_timestamps(self, resource_name):
+        """Get the highest timestamp of every objects in this `resource_name` for
+        each `parent_id`.
+
+        .. note::
+
+            This should take deleted objects into account.
+
+        :param str resource_name: the resource name.
+
+        :returns: the latest timestamp of the resource by `parent_id`.
+        :rtype: dict[str, int]
+        """
+        raise NotImplementedError
+
     def create(
         self,
         resource_name,

{kinto-20.0.0 → kinto-20.4.0}/kinto/core/storage/memory.py

@@ -153,6 +153,10 @@ class Storage(MemoryBasedStorage):
             raise exceptions.ReadonlyError(message=error_msg)
         return self.bump_and_store_timestamp(resource_name, parent_id)
 
+    @synchronized
+    def all_resources_timestamps(self, resource_name):
+        return {k: v[resource_name] for k, v in self._timestamps.items() if resource_name in v}
+
     def bump_and_store_timestamp(
         self, resource_name, parent_id, obj=None, modified_field=None, last_modified=None
     ):
@@ -284,13 +288,26 @@ class Storage(MemoryBasedStorage):
         modified_field=DEFAULT_MODIFIED_FIELD,
     ):
         parent_id_match = re.compile(parent_id.replace("*", ".*"))
-        by_parent_id = {
+
+        timestamps_by_parent_id = {
             pid: resources
-            for pid, resources in self._cemetery.items()
+            for pid, resources in self._timestamps.items()
             if parent_id_match.match(pid)
         }
+        if resource_name is not None:
+            for pid, resources in timestamps_by_parent_id.items():
+                del self._timestamps[pid][resource_name]
+        else:
+            for pid, resources in timestamps_by_parent_id.items():
+                del self._timestamps[pid]
+
         num_deleted = 0
-        for pid, resources in by_parent_id.items():
+        tombstones_by_parent_id = {
+            pid: resources
+            for pid, resources in self._cemetery.items()
+            if parent_id_match.match(pid)
+        }
+        for pid, resources in tombstones_by_parent_id.items():
             if resource_name is not None:
                 resources = {resource_name: resources[resource_name]}
             for resource, resource_objects in resources.items():

{kinto-20.0.0 → kinto-20.4.0}/kinto/core/storage/postgresql/__init__.py

@@ -79,7 +79,7 @@ class Storage(StorageBase, MigratorMixin):
 
     # MigratorMixin attributes.
     name = "storage"
-    schema_version = 22
+    schema_version = 23
     schema_file = os.path.join(HERE, "schema.sql")
     migrations_directory = os.path.join(HERE, "migrations")
 
@@ -247,6 +247,36 @@ class Storage(StorageBase, MigratorMixin):
 
         return obj.last_epoch
 
+    def all_resources_timestamps(self, resource_name):
+        query = """
+        WITH existing_timestamps AS (
+          -- Timestamp of latest object by parent_id.
+          (
+            SELECT parent_id, MAX(last_modified) AS last_modified
+            FROM objects
+            WHERE resource_name = :resource_name
+            GROUP BY parent_id
+          )
+          -- Timestamp of resources without sub-objects.
+          UNION
+          (
+            SELECT parent_id, last_modified
+            FROM timestamps
+            WHERE resource_name = :resource_name
+          )
+        )
+        SELECT parent_id, MAX(as_epoch(last_modified)) AS last_modified
+          FROM existing_timestamps
+          GROUP BY parent_id
+          ORDER BY last_modified DESC
+        """
+        with self.client.connect(readonly=True) as conn:
+            result = conn.execute(sa.text(query), dict(resource_name=resource_name))
+            rows = result.fetchmany(self._max_fetch_size + 1)
+
+        results = {r[0]: r[1] for r in rows}
+        return results
+
     @deprecate_kwargs({"collection_id": "resource_name", "record": "obj"})
     def create(
         self,

kinto-20.4.0/kinto/core/storage/postgresql/migrations/migration_022_023.sql

@@ -0,0 +1,5 @@
+CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
+    ON objects(resource_name, parent_id, deleted);
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');

{kinto-20.0.0 → kinto-20.4.0}/kinto/core/storage/postgresql/schema.sql

@@ -47,7 +47,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_objects_parent_id_resource_name_last_modif
     ON objects(parent_id, resource_name, last_modified DESC);
 CREATE INDEX IF NOT EXISTS idx_objects_last_modified_epoch
     ON objects(as_epoch(last_modified));
-
+CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
+    ON objects(resource_name, parent_id, deleted);
 
 CREATE TABLE IF NOT EXISTS timestamps (
   parent_id TEXT NOT NULL COLLATE "C",
@@ -131,4 +132,4 @@ INSERT INTO metadata (name, value) VALUES ('created_at', NOW()::TEXT);
 
 -- Set storage schema version.
 -- Should match ``kinto.core.storage.postgresql.PostgreSQL.schema_version``
-INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '22');
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');

{kinto-20.0.0 → kinto-20.4.0}/kinto/core/storage/testing.py

@@ -783,6 +783,39 @@ class TimestampsTest:
         after = self.storage.resource_timestamp(**self.storage_kw)
         self.assertTrue(before < after)
 
+    def test_all_timestamps_by_parent_id(self):
+        self.storage.create(obj={"id": "main"}, resource_name="bucket", parent_id="")
+        self.storage.create(obj={"id": "cid1"}, resource_name="collection", parent_id="/main")
+        self.storage.create(obj={"id": "cid2"}, resource_name="collection", parent_id="/main")
+        self.storage.create(obj={}, resource_name="record", parent_id="/main/cid2")
+        self.storage.create(obj={}, resource_name="record", parent_id="/main/cid2")
+
+        self.assertEqual(
+            {
+                "": self.storage.resource_timestamp(resource_name="bucket", parent_id=""),
+            },
+            self.storage.all_resources_timestamps(resource_name="bucket"),
+        )
+        self.assertEqual(
+            {
+                "/main": self.storage.resource_timestamp(
+                    resource_name="collection", parent_id="/main"
+                ),
+            },
+            self.storage.all_resources_timestamps(resource_name="collection"),
+        )
+        self.assertEqual(
+            {
+                "/main/cid1": self.storage.resource_timestamp(
+                    resource_name="record", parent_id="/main/cid1"
+                ),
+                "/main/cid2": self.storage.resource_timestamp(
+                    resource_name="record", parent_id="/main/cid2"
+                ),
+            },
+            self.storage.all_resources_timestamps(resource_name="record"),
+        )
+
     @skip_if_ci
     def test_timestamps_are_unique(self):  # pragma: no cover
         obtained = []
@@ -1263,6 +1296,9 @@ class DeletedObjectsTest:
         self.create_object(parent_id="/abc/a", resource_name="c")
         self.create_object(parent_id="/efg", resource_name="c")
 
+        all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
+        self.assertEqual(set(all_timestamps.keys()), {"/abc/a", "/efg"})
+
         before1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
         # Different parent_id with object.
         before2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
@@ -1272,11 +1308,15 @@ class DeletedObjectsTest:
         self.storage.delete_all(parent_id="/abc/*", resource_name=None, with_deleted=False)
         self.storage.purge_deleted(parent_id="/abc/*", resource_name=None)
 
+        all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
+        self.assertEqual(set(all_timestamps.keys()), {"/efg", "/ijk"})
+
+        time.sleep(0.002)  # make sure we don't recreate timestamps at same msec.
         after1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
         after2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
         after3 = self.storage.resource_timestamp(parent_id="/ijk", resource_name="c")
 
-        self.assertNotEqual(before1, after1)
+        self.assertNotEqual(before1, after1)  # timestamp was removed, it will differ.
         self.assertEqual(before2, after2)
         self.assertEqual(before3, after3)
 

kinto-20.4.0/kinto/plugins/admin/VERSION

@@ -0,0 +1 @@
+3.7.1

kinto-20.4.0/kinto/plugins/admin/build/VERSION

@@ -0,0 +1 @@
+3.7.1