kinto 23.2.1__py3-none-any.whl

kinto/core/storage/postgresql/client.py

@@ -0,0 +1,120 @@
+import contextlib
+import logging
+import warnings
+from collections import defaultdict
+
+import transaction as zope_transaction
+
+from kinto.core.storage import exceptions
+from kinto.core.utils import sqlalchemy
+
+
+logger = logging.getLogger(__name__)
+
+BLACKLISTED_SETTINGS = [
+    "backend",
+    "max_fetch_size",
+    "max_size_bytes",
+    "prefix",
+    "hosts",
+]
+
+
+class PostgreSQLClient:
+    def __init__(self, session_factory, commit_manually, invalidate):
+        self.session_factory = session_factory
+        self.commit_manually = commit_manually
+        self.invalidate = invalidate
+
+    @contextlib.contextmanager
+    def connect(self, readonly=False, force_commit=False):
+        """
+        Pulls a connection from the pool when context is entered and
+        returns it when context is exited.
+
+        A COMMIT is performed on the current transaction if everything went
+        well. Otherwise transaction is ROLLBACK, and everything cleaned up.
+        """
+        commit_manually = self.commit_manually and not readonly
+        session = None
+        try:
+            # Pull connection from pool.
+            session = self.session_factory()
+            # Start context
+            yield session
+            if not readonly and not self.commit_manually:
+                # Mark session as dirty.
+                self.invalidate(session)
+            # Success
+            if commit_manually:
+                session.commit()
+            elif force_commit:
+                # Commit like would do a succesful request.
+                zope_transaction.commit()
+
+        except sqlalchemy.exc.IntegrityError as e:
+            logger.error(e, exc_info=True)
+            if commit_manually:  # pragma: no branch
+                session.rollback()
+            raise exceptions.IntegrityError(original=e) from e
+        except sqlalchemy.exc.SQLAlchemyError as e:
+            logger.error(e, exc_info=True)
+            if session and commit_manually:
+                session.rollback()
+            raise exceptions.BackendError(original=e) from e
+        finally:
+            if session and self.commit_manually:
+                # Give back to pool if commit done manually.
+                session.close()
+
+
+# Reuse existing client if same URL.
+_CLIENTS = defaultdict(dict)
+
+
+def create_from_config(config, prefix="", with_transaction=True):
+    """Create a PostgreSQLClient client using settings in the provided config."""
+    if sqlalchemy is None:
+        message = (
+            "PostgreSQL SQLAlchemy dependency missing. "
+            "Refer to installation section in documentation."
+        )
+        raise ImportWarning(message)
+
+    from sqlalchemy.orm import scoped_session, sessionmaker
+    from zope.sqlalchemy import invalidate, register
+
+    settings = {**config.get_settings()}
+
+    # Custom Kinto settings, unsupported by SQLAlchemy.
+    blacklist = [prefix + setting for setting in BLACKLISTED_SETTINGS]
+    filtered_settings = {k: v for k, v in settings.items() if k not in blacklist}
+    transaction_per_request = with_transaction and filtered_settings.pop(
+        "transaction_per_request", False
+    )
+    url = filtered_settings[prefix + "url"]
+    existing_client = _CLIENTS[transaction_per_request].get(url)
+    if existing_client:
+        msg = f"Reuse existing PostgreSQL connection. Parameters {prefix}* will be ignored."
+        warnings.warn(msg)
+        return existing_client
+
+    # Initialize SQLAlchemy engine from filtered_settings.
+    poolclass_key = prefix + "poolclass"
+    filtered_settings.setdefault(
+        poolclass_key, ("kinto.core.storage.postgresql.pool.QueuePoolWithMaxBacklog")
+    )
+    filtered_settings[poolclass_key] = config.maybe_dotted(filtered_settings[poolclass_key])
+    engine = sqlalchemy.engine_from_config(filtered_settings, prefix=prefix, url=url)
+
+    # Initialize thread-safe session factory.
+    session_factory = scoped_session(sessionmaker(bind=engine))
+    if transaction_per_request:
+        # Plug with Pyramid transaction manager
+        register(session_factory)
+
+    # Store one client per URI.
+    commit_manually = not transaction_per_request
+    client = PostgreSQLClient(session_factory, commit_manually, invalidate)
+    _CLIENTS[transaction_per_request][url] = client
+    return client

kinto/core/storage/postgresql/migrations/migration_001_002.sql

@@ -0,0 +1,10 @@
+ALTER FUNCTION as_epoch(TIMESTAMP) IMMUTABLE;
+
+DROP INDEX IF EXISTS idx_records_last_modified_epoch;
+CREATE INDEX idx_records_last_modified_epoch ON records(as_epoch(last_modified));
+
+DROP INDEX IF EXISTS idx_deleted_last_modified_epoch;
+CREATE INDEX idx_deleted_last_modified_epoch ON deleted(as_epoch(last_modified));
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '2');

kinto/core/storage/postgresql/migrations/migration_002_003.sql

@@ -0,0 +1,33 @@
+ALTER TABLE records DROP CONSTRAINT records_id_user_id_resource_name_last_modified_key CASCADE;
+CREATE UNIQUE INDEX idx_records_user_id_resource_name_last_modified
+    ON records(user_id, resource_name, last_modified DESC);
+
+ALTER TABLE deleted DROP CONSTRAINT deleted_id_user_id_resource_name_last_modified_key CASCADE;
+CREATE UNIQUE INDEX idx_deleted_user_id_resource_name_last_modified
+    ON deleted(user_id, resource_name, last_modified DESC);
+
+CREATE OR REPLACE FUNCTION resource_timestamp(uid VARCHAR, resource VARCHAR)
+RETURNS TIMESTAMP AS $$
+DECLARE
+    ts_records TIMESTAMP;
+    ts_deleted TIMESTAMP;
+BEGIN
+    SELECT last_modified INTO ts_records
+      FROM records
+     WHERE user_id = uid
+       AND resource_name = resource
+     ORDER BY last_modified DESC LIMIT 1;
+
+    SELECT last_modified INTO ts_deleted
+      FROM deleted
+     WHERE user_id = uid
+       AND resource_name = resource
+     ORDER BY last_modified DESC LIMIT 1;
+
+    -- Latest of records/deleted or current if empty
+    RETURN coalesce(greatest(ts_deleted, ts_records), localtimestamp);
+END;
+$$ LANGUAGE plpgsql;
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '3');

kinto/core/storage/postgresql/migrations/migration_003_004.sql

@@ -0,0 +1,18 @@
+ALTER TABLE records
+    ALTER COLUMN id DROP DEFAULT,
+    ALTER COLUMN id SET DATA TYPE TEXT,
+    ALTER COLUMN user_id SET DATA TYPE TEXT,
+    ALTER COLUMN resource_name SET DATA TYPE TEXT;
+
+ALTER TABLE deleted
+    ALTER COLUMN id DROP DEFAULT,
+    ALTER COLUMN id SET DATA TYPE TEXT,
+    ALTER COLUMN user_id SET DATA TYPE TEXT,
+    ALTER COLUMN resource_name SET DATA TYPE TEXT;
+
+
+DROP EXTENSION IF EXISTS "uuid-ossp";
+
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '4');

kinto/core/storage/postgresql/migrations/migration_004_005.sql

@@ -0,0 +1,20 @@
+DROP INDEX IF EXISTS idx_records_user_id;
+DROP INDEX IF EXISTS idx_records_resource_name;
+DROP INDEX IF EXISTS idx_records_last_modified;
+DROP INDEX IF EXISTS idx_records_id;
+
+ALTER TABLE records
+    ADD PRIMARY KEY (id, user_id, resource_name);
+
+
+DROP INDEX IF EXISTS idx_deleted_id;
+DROP INDEX IF EXISTS idx_deleted_user_id;
+DROP INDEX IF EXISTS idx_deleted_resource_name;
+DROP INDEX IF EXISTS idx_deleted_last_modified;
+
+ALTER TABLE deleted
+    ADD PRIMARY KEY (id, user_id, resource_name);
+
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '5');

kinto/core/storage/postgresql/migrations/migration_005_006.sql

@@ -0,0 +1,11 @@
+--
+-- Switch records data column to JSONB.
+-- (requires PostgreSQL 9.4+)
+--
+ALTER TABLE records
+    ALTER COLUMN data DROP DEFAULT,
+    ALTER COLUMN data SET DATA TYPE JSONB USING data::TEXT::JSONB,
+    ALTER COLUMN data SET DEFAULT '{}'::JSONB;
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '6');

kinto/core/storage/postgresql/migrations/migration_006_007.sql

@@ -0,0 +1,74 @@
+ALTER TABLE records RENAME COLUMN resource_name TO collection_id;
+ALTER TABLE records RENAME COLUMN user_id TO parent_id;
+
+ALTER INDEX idx_records_user_id_resource_name_last_modified
+    RENAME TO idx_records_parent_id_collection_id_last_modified;
+
+ALTER TABLE deleted RENAME COLUMN resource_name TO collection_id;
+ALTER TABLE deleted RENAME COLUMN user_id TO parent_id;
+
+ALTER INDEX idx_deleted_user_id_resource_name_last_modified
+    RENAME TO idx_deleted_parent_id_collection_id_last_modified;
+
+ALTER FUNCTION resource_timestamp(VARCHAR, VARCHAR)
+    RENAME TO collection_timestamp;
+
+
+CREATE OR REPLACE FUNCTION collection_timestamp(uid VARCHAR, resource VARCHAR)
+RETURNS TIMESTAMP AS $$
+DECLARE
+    ts_records TIMESTAMP;
+    ts_deleted TIMESTAMP;
+BEGIN
+    --
+    -- This is fast because an index was created for ``parent_id``,
+    -- ``collection_id``, and ``last_modified`` with descending sorting order.
+    --
+    SELECT last_modified INTO ts_records
+      FROM records
+     WHERE parent_id = uid
+       AND collection_id = resource
+     ORDER BY last_modified DESC LIMIT 1;
+
+    SELECT last_modified INTO ts_deleted
+      FROM deleted
+     WHERE parent_id = uid
+       AND collection_id = resource
+     ORDER BY last_modified DESC LIMIT 1;
+
+    -- Latest of records/deleted or current if empty
+    RETURN coalesce(greatest(ts_deleted, ts_records), localtimestamp);
+END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION bump_timestamp()
+RETURNS trigger AS $$
+DECLARE
+    previous TIMESTAMP;
+    current TIMESTAMP;
+BEGIN
+    --
+    -- This bumps the current timestamp to 1 msec in the future if the previous
+    -- timestamp is equal to the current one (or higher if was bumped already).
+    --
+    -- If a bunch of requests from the same user on the same collection
+    -- arrive in the same millisecond, the unicity constraint can raise
+    -- an error (operation is cancelled).
+    -- See https://github.com/mozilla-services/cliquet/issues/25
+    --
+    previous := collection_timestamp(NEW.parent_id, NEW.collection_id);
+    current := localtimestamp;
+
+    IF previous >= current THEN
+        current := previous + INTERVAL '1 milliseconds';
+    END IF;
+
+    NEW.last_modified := current;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '7');

kinto/core/storage/postgresql/migrations/migration_007_008.sql

@@ -0,0 +1,66 @@
+--
+-- Helper that returns the current collection timestamp.
+--
+CREATE OR REPLACE FUNCTION collection_timestamp(uid VARCHAR, resource VARCHAR)
+RETURNS TIMESTAMP AS $$
+DECLARE
+    ts_records TIMESTAMP;
+    ts_deleted TIMESTAMP;
+BEGIN
+    --
+    -- This is fast because an index was created for ``parent_id``,
+    -- ``collection_id``, and ``last_modified`` with descending sorting order.
+    --
+    SELECT last_modified INTO ts_records
+      FROM records
+     WHERE parent_id = uid
+       AND collection_id = resource
+     ORDER BY last_modified DESC LIMIT 1;
+
+    SELECT last_modified INTO ts_deleted
+      FROM deleted
+     WHERE parent_id = uid
+       AND collection_id = resource
+     ORDER BY last_modified DESC LIMIT 1;
+
+    -- Latest of records/deleted or current if empty
+    RETURN coalesce(greatest(ts_deleted, ts_records), clock_timestamp());
+END;
+$$ LANGUAGE plpgsql;
+
+--
+-- Triggers to set last_modified on INSERT/UPDATE
+--
+DROP TRIGGER IF EXISTS tgr_records_last_modified ON records;
+DROP TRIGGER IF EXISTS tgr_deleted_last_modified ON deleted;
+
+CREATE OR REPLACE FUNCTION bump_timestamp()
+RETURNS trigger AS $$
+DECLARE
+    previous TIMESTAMP;
+    current TIMESTAMP;
+BEGIN
+    --
+    -- This bumps the current timestamp to 1 msec in the future if the previous
+    -- timestamp is equal to the current one (or higher if was bumped already).
+    --
+    -- If a bunch of requests from the same user on the same collection
+    -- arrive in the same millisecond, the unicity constraint can raise
+    -- an error (operation is cancelled).
+    -- See https://github.com/mozilla-services/cliquet/issues/25
+    --
+    previous := collection_timestamp(NEW.parent_id, NEW.collection_id);
+    current := clock_timestamp();
+
+    IF previous >= current THEN
+        current := previous + INTERVAL '1 milliseconds';
+    END IF;
+
+    NEW.last_modified := current;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '8');

kinto/core/storage/postgresql/migrations/migration_008_009.sql

@@ -0,0 +1,41 @@
+CREATE OR REPLACE FUNCTION from_epoch(epoch BIGINT) RETURNS TIMESTAMP AS $$
+BEGIN
+    RETURN TIMESTAMP WITH TIME ZONE 'epoch' + epoch * INTERVAL '1 millisecond';
+END;
+$$ LANGUAGE plpgsql
+IMMUTABLE;
+
+
+CREATE OR REPLACE FUNCTION bump_timestamp()
+RETURNS trigger AS $$
+DECLARE
+    previous TIMESTAMP;
+    current TIMESTAMP;
+
+BEGIN
+    --
+    -- This bumps the current timestamp to 1 msec in the future if the previous
+    -- timestamp is equal to the current one (or higher if was bumped already).
+    --
+    -- If a bunch of requests from the same user on the same collection
+    -- arrive in the same millisecond, the unicity constraint can raise
+    -- an error (operation is cancelled).
+    -- See https://github.com/mozilla-services/cliquet/issues/25
+    --
+    previous := collection_timestamp(NEW.parent_id, NEW.collection_id);
+
+    IF NEW.last_modified IS NULL THEN
+        current := clock_timestamp();
+        IF previous >= current THEN
+            current := previous + INTERVAL '1 milliseconds';
+        END IF;
+        NEW.last_modified := current;
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '9');

kinto/core/storage/postgresql/migrations/migration_009_010.sql

@@ -0,0 +1,98 @@
+CREATE TABLE IF NOT EXISTS timestamps (
+  parent_id TEXT NOT NULL,
+  collection_id TEXT NOT NULL,
+  last_modified TIMESTAMP NOT NULL,
+  PRIMARY KEY (parent_id, collection_id)
+);
+
+
+CREATE OR REPLACE FUNCTION collection_timestamp(uid VARCHAR, resource VARCHAR)
+RETURNS TIMESTAMP AS $$
+DECLARE
+    ts TIMESTAMP;
+BEGIN
+    ts := NULL;
+
+    SELECT last_modified INTO ts
+      FROM timestamps
+     WHERE parent_id = uid
+       AND collection_id = resource;
+
+    IF ts IS NULL THEN
+      ts := clock_timestamp();
+      INSERT INTO timestamps (parent_id, collection_id, last_modified)
+      VALUES (uid, resource, ts);
+    END IF;
+
+    RETURN ts;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS tgr_records_last_modified ON records;
+DROP TRIGGER IF EXISTS tgr_deleted_last_modified ON deleted;
+
+CREATE OR REPLACE FUNCTION bump_timestamp()
+RETURNS trigger AS $$
+DECLARE
+    previous TIMESTAMP;
+    current TIMESTAMP;
+
+BEGIN
+    previous := NULL;
+    SELECT last_modified INTO previous
+      FROM timestamps
+     WHERE parent_id = NEW.parent_id
+       AND collection_id = NEW.collection_id;
+
+    --
+    -- This bumps the current timestamp to 1 msec in the future if the previous
+    -- timestamp is equal to the current one (or higher if was bumped already).
+    --
+    -- If a bunch of requests from the same user on the same collection
+    -- arrive in the same millisecond, the unicity constraint can raise
+    -- an error (operation is cancelled).
+    -- See https://github.com/mozilla-services/cliquet/issues/25
+    --
+    current := clock_timestamp();
+    IF previous IS NOT NULL AND previous >= current THEN
+        current := previous + INTERVAL '1 milliseconds';
+    END IF;
+
+
+    IF NEW.last_modified IS NULL THEN
+        -- If record does not carry last-modified, assign it to current.
+        NEW.last_modified := current;
+    ELSE
+        -- Use record last-modified as collection timestamp.
+        IF previous IS NULL OR NEW.last_modified > previous THEN
+            current := NEW.last_modified;
+        END IF;
+    END IF;
+
+    --
+    -- Upsert current collection timestamp.
+    --
+    WITH upsert AS (
+        UPDATE timestamps SET last_modified = current
+         WHERE parent_id = NEW.parent_id AND collection_id = NEW.collection_id
+        RETURNING *
+    )
+    INSERT INTO timestamps (parent_id, collection_id, last_modified)
+    SELECT NEW.parent_id, NEW.collection_id, current
+    WHERE NOT EXISTS (SELECT * FROM upsert);
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER tgr_records_last_modified
+BEFORE INSERT OR UPDATE ON records
+FOR EACH ROW EXECUTE PROCEDURE bump_timestamp();
+
+CREATE TRIGGER tgr_deleted_last_modified
+BEFORE INSERT OR UPDATE ON deleted
+FOR EACH ROW EXECUTE PROCEDURE bump_timestamp();
+
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '10');

kinto/core/storage/postgresql/migrations/migration_010_011.sql

@@ -0,0 +1,14 @@
+DROP TRIGGER IF EXISTS tgr_records_last_modified ON records;
+DROP TRIGGER IF EXISTS tgr_deleted_last_modified ON deleted;
+
+CREATE TRIGGER tgr_records_last_modified
+BEFORE INSERT OR UPDATE ON records
+FOR EACH ROW EXECUTE PROCEDURE bump_timestamp();
+
+CREATE TRIGGER tgr_deleted_last_modified
+BEFORE INSERT OR UPDATE ON deleted
+FOR EACH ROW EXECUTE PROCEDURE bump_timestamp();
+
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '11');

kinto/core/storage/postgresql/migrations/migration_011_012.sql

@@ -0,0 +1,9 @@
+-- Select all existing records and delete their tombstone if any.
+DELETE FROM deleted d
+USING records r
+WHERE d.id = r.id
+AND   d.parent_id = r.parent_id
+AND   d.collection_id = r.collection_id;
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '12');

kinto/core/storage/postgresql/migrations/migration_012_013.sql

@@ -0,0 +1,71 @@
+--
+-- Triggers to set last_modified on INSERT/UPDATE
+--
+DROP TRIGGER IF EXISTS tgr_records_last_modified ON records;
+DROP TRIGGER IF EXISTS tgr_deleted_last_modified ON deleted;
+
+CREATE OR REPLACE FUNCTION bump_timestamp()
+RETURNS trigger AS $$
+DECLARE
+    previous TIMESTAMP;
+    current TIMESTAMP;
+
+BEGIN
+    previous := NULL;
+    SELECT last_modified INTO previous
+      FROM timestamps
+     WHERE parent_id = NEW.parent_id
+       AND collection_id = NEW.collection_id;
+
+    --
+    -- This bumps the current timestamp to 1 msec in the future if the previous
+    -- timestamp is equal to the current one (or higher if was bumped already).
+    --
+    -- If a bunch of requests from the same user on the same collection
+    -- arrive in the same millisecond, the unicity constraint can raise
+    -- an error (operation is cancelled).
+    -- See https://github.com/mozilla-services/cliquet/issues/25
+    --
+    current := clock_timestamp();
+    IF previous IS NOT NULL AND previous >= current THEN
+        current := previous + INTERVAL '1 milliseconds';
+    END IF;
+
+    IF NEW.last_modified IS NULL OR
+       (previous IS NOT NULL AND as_epoch(NEW.last_modified) = as_epoch(previous)) THEN
+        -- If record does not carry last-modified, or if the one specified
+        -- is equal to previous, assign it to current (i.e. bump it).
+        NEW.last_modified := current;
+    ELSE
+        -- Use record last-modified as collection timestamp.
+        IF previous IS NULL OR NEW.last_modified > previous THEN
+            current := NEW.last_modified;
+        END IF;
+    END IF;
+
+    --
+    -- Upsert current collection timestamp.
+    --
+    WITH upsert AS (
+        UPDATE timestamps SET last_modified = current
+         WHERE parent_id = NEW.parent_id AND collection_id = NEW.collection_id
+        RETURNING *
+    )
+    INSERT INTO timestamps (parent_id, collection_id, last_modified)
+    SELECT NEW.parent_id, NEW.collection_id, current
+    WHERE NOT EXISTS (SELECT * FROM upsert);
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER tgr_records_last_modified
+BEFORE INSERT OR UPDATE ON records
+FOR EACH ROW EXECUTE PROCEDURE bump_timestamp();
+
+CREATE TRIGGER tgr_deleted_last_modified
+BEFORE INSERT OR UPDATE ON deleted
+FOR EACH ROW EXECUTE PROCEDURE bump_timestamp();
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '13');

kinto/core/storage/postgresql/migrations/migration_013_014.sql

@@ -0,0 +1,14 @@
+DROP TRIGGER IF EXISTS tgr_records_last_modified ON records;
+DROP TRIGGER IF EXISTS tgr_deleted_last_modified ON deleted;
+
+CREATE TRIGGER tgr_records_last_modified
+BEFORE INSERT OR UPDATE OF data ON records
+FOR EACH ROW EXECUTE PROCEDURE bump_timestamp();
+
+CREATE TRIGGER tgr_deleted_last_modified
+BEFORE INSERT ON deleted
+FOR EACH ROW EXECUTE PROCEDURE bump_timestamp();
+
+
+-- Bump storage schema version.
+INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '14');