kinto 23.2.1__py3-none-any.whl

kinto/core/cache/postgresql/__init__.py

@@ -0,0 +1,178 @@
+import logging
+import os
+
+from kinto.core.cache import CacheBase
+from kinto.core.storage.postgresql.client import create_from_config
+from kinto.core.utils import json
+from kinto.core.utils import sqlalchemy as sa
+
+
+logger = logging.getLogger(__name__)
+
+
+class Cache(CacheBase):
+    """Cache backend using PostgreSQL.
+
+    Enable in configuration::
+
+        kinto.cache_backend = kinto.core.cache.postgresql
+
+    Database location URI can be customized::
+
+        kinto.cache_url = postgresql://user:pass@db.server.lan:5432/dbname
+
+    Alternatively, username and password could also rely on system user ident
+    or even specified in :file:`~/.pgpass` (*see PostgreSQL documentation*).
+
+    .. note::
+
+        Some tables and indices are created when ``kinto migrate`` is run.
+        This requires some privileges on the database, or some error will
+        be raised.
+
+        **Alternatively**, the schema can be initialized outside the
+        python application, using the SQL file located in
+        :file:`kinto/core/cache/postgresql/schema.sql`. This allows to
+        distinguish schema manipulation privileges from schema usage.
+
+
+    A connection pool is enabled by default::
+
+        kinto.cache_pool_size = 10
+        kinto.cache_maxoverflow = 10
+        kinto.cache_max_backlog = -1
+        kinto.cache_pool_recycle = -1
+        kinto.cache_pool_timeout = 30
+        kinto.cache_poolclass =
+            kinto.core.storage.postgresql.pool.QueuePoolWithMaxBacklog
+
+    The ``max_backlog``  limits the number of threads that can be in the queue
+    waiting for a connection.  Once this limit has been reached, any further
+    attempts to acquire a connection will be rejected immediately, instead of
+    locking up all threads by keeping them waiting in the queue.
+
+    See `dedicated section in SQLAlchemy documentation
+    <http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html>`_
+    for default values and behaviour.
+
+    .. note::
+
+        Using a `dedicated connection pool <http://pgpool.net>`_ is still
+        recommended to allow load balancing, replication or limit the number
+        of connections used in a multi-process deployment.
+
+    :noindex:
+    """  # NOQA
+
+    def __init__(self, client, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = client
+
+    def initialize_schema(self, dry_run=False):
+        # Check if cache table exists.
+        query = """
+        SELECT 1
+          FROM information_schema.tables
+         WHERE table_name = 'cache';
+        """
+        with self.client.connect(readonly=True) as conn:
+            result = conn.execute(sa.text(query))
+            if result.rowcount > 0:
+                logger.info("PostgreSQL cache schema is up-to-date.")
+                return
+
+        # Create schema
+        here = os.path.dirname(__file__)
+        sql_file = os.path.join(here, "schema.sql")
+
+        if dry_run:
+            logger.info(f"Create cache schema from '{sql_file}'")
+            return
+
+        # Since called outside request, force commit.
+        with open(sql_file) as f:
+            schema = f.read()
+        with self.client.connect(force_commit=True) as conn:
+            conn.execute(sa.text(schema))
+        logger.info("Created PostgreSQL cache tables")
+
+    def flush(self):
+        query = """
+        DELETE FROM cache;
+        """
+        # Since called outside request (e.g. tests), force commit.
+        with self.client.connect(force_commit=True) as conn:
+            conn.execute(sa.text(query))
+        logger.debug("Flushed PostgreSQL cache tables")
+
+    def ttl(self, key):
+        query = """
+        SELECT EXTRACT(SECOND FROM (ttl - now())) AS ttl
+          FROM cache
+         WHERE key = :key
+           AND ttl IS NOT NULL;
+        """
+        with self.client.connect(readonly=True) as conn:
+            result = conn.execute(sa.text(query), dict(key=self.prefix + key))
+            if result.rowcount > 0:
+                return result.fetchone().ttl
+        return -1
+
+    def expire(self, key, ttl):
+        query = """
+        UPDATE cache SET ttl = sec2ttl(:ttl) WHERE key = :key;
+        """
+        with self.client.connect() as conn:
+            conn.execute(sa.text(query), dict(ttl=ttl, key=self.prefix + key))
+
+    def set(self, key, value, ttl):
+        if isinstance(value, bytes):
+            raise TypeError("a string-like object is required, not 'bytes'")
+
+        query = """
+        INSERT INTO cache (key, value, ttl)
+        VALUES (:key, :value, sec2ttl(:ttl))
+        ON CONFLICT (key) DO UPDATE
+        SET value = :value,
+            ttl = sec2ttl(:ttl);
+        """
+        value = json.dumps(value)
+        with self.client.connect() as conn:
+            conn.execute(sa.text(query), dict(key=self.prefix + key, value=value, ttl=ttl))
+
+    def get(self, key):
+        purge = """
+        DELETE FROM cache c
+        USING (
+            SELECT key
+            FROM cache
+            WHERE ttl IS NOT NULL AND now() > ttl
+            ORDER BY key ASC
+            FOR UPDATE
+        ) del
+        WHERE del.key = c.key;"""
+        query = "SELECT value FROM cache WHERE key = :key AND now() < ttl;"
+        with self.client.connect() as conn:
+            conn.execute(sa.text(purge))
+            result = conn.execute(sa.text(query), dict(key=self.prefix + key))
+            if result.rowcount > 0:
+                self.metrics_backend.count_hit()
+                value = result.fetchone().value
+                return json.loads(value)
+            self.metrics_backend.count_miss()
+            return None
+
+    def delete(self, key):
+        query = "DELETE FROM cache WHERE key = :key RETURNING value;"
+        with self.client.connect() as conn:
+            result = conn.execute(sa.text(query), dict(key=self.prefix + key))
+            if result.rowcount > 0:
+                value = result.fetchone().value
+                return json.loads(value)
+        return None
+
+
+def load_from_config(config):
+    settings = config.get_settings()
+    client = create_from_config(config, prefix="cache_", with_transaction=False)
+    return Cache(client=client, cache_prefix=settings["cache_prefix"])

kinto/core/cache/postgresql/schema.sql

@@ -0,0 +1,23 @@
+--
+-- Automated script, we do not need NOTICE and WARNING
+--
+SET client_min_messages TO ERROR;
+
+CREATE TABLE IF NOT EXISTS cache (
+    key VARCHAR(256) PRIMARY KEY,
+    value TEXT NOT NULL,
+    ttl TIMESTAMP DEFAULT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_cache_ttl ON cache(ttl);
+
+
+CREATE OR REPLACE FUNCTION sec2ttl(seconds FLOAT)
+RETURNS TIMESTAMP AS $$
+BEGIN
+    IF seconds IS NULL THEN
+        RETURN NULL;
+    END IF;
+    RETURN now() + (seconds || ' SECOND')::INTERVAL;
+END;
+$$ LANGUAGE plpgsql;

kinto/core/cache/testing.py

@@ -0,0 +1,208 @@
+import time
+from unittest import mock
+
+import pytest
+from pyramid import testing
+
+from kinto.core.cache import heartbeat
+from kinto.core.storage import exceptions
+
+
+class CacheTest:
+    backend = None
+    settings = {}
+
+    def setUp(self):
+        super().setUp()
+        self.cache = self.backend.load_from_config(self._get_config())
+        self.cache.initialize_schema()
+        self.request = None
+        self.client_error_patcher = None
+
+    def _get_config(self, settings=None):
+        """Mock Pyramid config object."""
+        if settings is None:
+            settings = self.settings
+        config = testing.setUp()
+        config.add_settings(settings)
+        return config
+
+    def tearDown(self):
+        mock.patch.stopall()
+        super().tearDown()
+        self.cache.flush()
+
+    def get_backend_prefix(self, prefix):
+        settings_prefix = {**self.settings}
+        settings_prefix["cache_prefix"] = prefix
+        config_prefix = self._get_config(settings=settings_prefix)
+
+        # initiating cache backend with prefix:
+        backend_prefix = self.backend.load_from_config(config_prefix)
+
+        return backend_prefix
+
+    def test_backend_error_is_raised_anywhere(self):
+        self.client_error_patcher.start()
+        calls = [
+            (self.cache.flush,),
+            (self.cache.ttl, ""),
+            (self.cache.expire, "", 0),
+            (self.cache.get, ""),
+            (self.cache.set, "", "", 42),
+            (self.cache.delete, ""),
+        ]
+        for call in calls:
+            self.assertRaises(exceptions.BackendError, *call)
+
+    def test_initialize_schema_is_idempotent(self):
+        self.cache.initialize_schema()
+        self.cache.initialize_schema()  # not raising.
+
+    def test_ping_returns_false_if_unavailable(self):
+        self.client_error_patcher.start()
+        ping = heartbeat(self.cache)
+        self.assertFalse(ping(self.request))
+        with mock.patch("kinto.core.cache.random.SystemRandom.random", return_value=0.6):
+            self.assertFalse(ping(self.request))
+        with mock.patch("kinto.core.cache.random.SystemRandom.random", return_value=0.4):
+            self.assertFalse(ping(self.request))
+
+    def test_ping_returns_true_if_available(self):
+        ping = heartbeat(self.cache)
+        with mock.patch("kinto.core.cache.random.random", return_value=0.6):
+            self.assertTrue(ping(self.request))
+        with mock.patch("kinto.core.cache.random.random", return_value=0.4):
+            self.assertTrue(ping(self.request))
+
+    def test_ping_logs_error_if_unavailable(self):
+        self.client_error_patcher.start()
+        ping = heartbeat(self.cache)
+
+        with mock.patch("kinto.core.cache.logger.exception") as exc_handler:
+            self.assertFalse(ping(self.request))
+
+        self.assertTrue(exc_handler.called)
+
+    def test_set_adds_the_object(self):
+        stored = "toto"
+        self.cache.set("foobar", stored, 42)
+        retrieved = self.cache.get("foobar")
+        self.assertEqual(retrieved, stored)
+
+    def test_values_remains_python_dict(self):
+        def setget(k, v):
+            self.cache.set(k, v, 42)
+            return (self.cache.get(k), v)
+
+        self.assertEqual(*setget("foobar", 3))
+        self.assertEqual(*setget("foobar", ["a"]))
+        self.assertEqual(*setget("foobar", {"b": [1, 2]}))
+        self.assertEqual(*setget("foobar", 3.14))
+
+    def test_bytes_cannot_be_stored_in_the_cache(self):
+        with pytest.raises(TypeError):
+            self.cache.set("test", b"foo", 42)
+
+    def test_delete_removes_the_object(self):
+        self.cache.set("foobar", "toto", 42)
+        returned = self.cache.delete("foobar")
+        self.assertEqual(returned, "toto")
+        missing = self.cache.get("foobar")
+        self.assertIsNone(missing)
+
+    def test_delete_does_not_fail_if_object_is_unknown(self):
+        returned = self.cache.delete("foobar")
+        self.assertIsNone(returned)
+
+    def test_expire_expires_the_value(self):
+        self.cache.set("foobar", "toto", 42)
+        self.cache.expire("foobar", 0.01)
+        time.sleep(0.02)
+        retrieved = self.cache.get("foobar")
+        self.assertIsNone(retrieved)
+
+    def test_set_with_ttl_expires_the_value(self):
+        self.cache.set("foobar", "toto", 0.01)
+        time.sleep(0.02)
+        retrieved = self.cache.get("foobar")
+        self.assertIsNone(retrieved)
+
+    def test_ttl_return_the_time_to_live(self):
+        self.cache.set("foobar", "toto", 42)
+        self.cache.expire("foobar", 10)
+        ttl = self.cache.ttl("foobar")
+        self.assertGreater(ttl, 0)
+        self.assertLessEqual(ttl, 10)
+
+    def test_ttl_return_none_if_unknown(self):
+        ttl = self.cache.ttl("unknown")
+        self.assertTrue(ttl < 0)
+
+    def test_cache_prefix_is_set(self):
+        backend_prefix = self.get_backend_prefix(prefix="prefix_")
+
+        # Set the value
+        backend_prefix.set("key", "foo", 42)
+
+        # Validate that it was set with the prefix.
+        obtained = self.cache.get("prefix_key")
+        self.assertEqual(obtained, "foo")
+
+    def test_cache_when_prefix_is_not_set(self):
+        backend_prefix = self.get_backend_prefix(prefix="")
+
+        # Set a value
+        backend_prefix.set("key", "foo", 42)
+
+        # Validate that it was set with no prefix
+        obtained = self.cache.get("key")
+        self.assertEqual(obtained, "foo")
+
+    def test_prefix_value_use_to_get_data(self):
+        backend_prefix = self.get_backend_prefix(prefix="prefix_")
+
+        # Set the value with the prefix
+        self.cache.set("prefix_key", "foo", 42)
+
+        # Validate that the prefix was added
+        obtained = backend_prefix.get("key")
+        self.assertEqual(obtained, "foo")
+
+    def test_prefix_value_use_to_delete_data(self):
+        backend_prefix = self.get_backend_prefix(prefix="prefix_")
+        # Set the value
+        self.cache.set("prefix_key", "foo", 42)
+
+        # Delete the value
+        backend_prefix.delete("key")
+
+        # Validate that the value was deleted
+        obtained = self.cache.get("prefix_key")
+        self.assertEqual(obtained, None)
+
+    def test_prefix_value_used_with_ttl(self):
+        backend_prefix = self.get_backend_prefix(prefix="prefix_")
+
+        self.cache.set("prefix_key", "foo", 10)
+
+        # Validate that the ttl add the prefix to the key.
+        obtained = backend_prefix.ttl("key")
+        self.assertLessEqual(obtained, 10)
+        self.assertGreater(obtained, 9)
+
+    def test_prefix_value_used_with_expire(self):
+        backend_prefix = self.get_backend_prefix(prefix="prefix_")
+
+        self.cache.set("prefix_foobar", "toto", 10)
+
+        # expiring the ttl of key
+        backend_prefix.expire("foobar", 0)
+
+        # Make sure the TTL was set accordingly.
+        ttl = self.cache.ttl("prefix_foobar")
+        self.assertLessEqual(ttl, 0)
+
+        # The object should have expired
+        retrieved = self.cache.get("prefix_foobar")
+        self.assertIsNone(retrieved)

kinto/core/cornice/__init__.py

@@ -0,0 +1,93 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+import logging
+from functools import partial
+
+from pyramid.events import NewRequest
+from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound
+from pyramid.security import NO_PERMISSION_REQUIRED
+from pyramid.settings import asbool, aslist
+
+from kinto.core.cornice.errors import Errors  # NOQA
+from kinto.core.cornice.pyramidhook import (
+    handle_exceptions,
+    register_resource_views,
+    register_service_views,
+    wrap_request,
+)
+from kinto.core.cornice.renderer import CorniceRenderer
+from kinto.core.cornice.service import Service  # NOQA
+from kinto.core.cornice.util import ContentTypePredicate, current_service
+
+
+logger = logging.getLogger("cornice")
+
+
+def set_localizer_for_languages(event, available_languages, default_locale_name):
+    """
+    Sets the current locale based on the incoming Accept-Language header, if
+    present, and sets a localizer attribute on the request object based on
+    the current locale.
+
+    To be used as an event handler, this function needs to be partially applied
+    with the available_languages and default_locale_name arguments. The
+    resulting function will be an event handler which takes an event object as
+    its only argument.
+    """
+    request = event.request
+    if request.accept_language:
+        accepted = request.accept_language.lookup(available_languages, default=default_locale_name)
+        request._LOCALE_ = accepted
+
+
+def setup_localization(config):
+    """
+    Setup localization based on the available_languages and
+    pyramid.default_locale_name settings.
+
+    These settings are named after suggestions from the "Internationalization
+    and Localization" section of the Pyramid documentation.
+    """
+    try:
+        config.add_translation_dirs("colander:locale/")
+        settings = config.get_settings()
+        available_languages = aslist(settings["available_languages"])
+        default_locale_name = settings.get("pyramid.default_locale_name", "en")
+        set_localizer = partial(
+            set_localizer_for_languages,
+            available_languages=available_languages,
+            default_locale_name=default_locale_name,
+        )
+        config.add_subscriber(set_localizer, NewRequest)
+    except ImportError:  # pragma: no cover
+        # add_translation_dirs raises an ImportError if colander is not
+        # installed
+        pass
+
+
+def includeme(config):
+    """Include the Cornice definitions"""
+    # attributes required to maintain services
+    config.registry.cornice_services = {}
+
+    settings = config.get_settings()
+
+    # localization request subscriber must be set before first call
+    # for request.localizer (in wrap_request)
+    if settings.get("available_languages"):
+        setup_localization(config)
+
+    config.add_directive("add_cornice_service", register_service_views)
+    config.add_directive("add_cornice_resource", register_resource_views)
+    config.add_subscriber(wrap_request, NewRequest)
+    config.add_renderer("cornicejson", CorniceRenderer())
+    config.add_view_predicate("content_type", ContentTypePredicate)
+    config.add_request_method(current_service, reify=True)
+
+    if asbool(settings.get("handle_exceptions", True)):
+        config.add_view(handle_exceptions, context=Exception, permission=NO_PERMISSION_REQUIRED)
+        config.add_view(handle_exceptions, context=HTTPNotFound, permission=NO_PERMISSION_REQUIRED)
+        config.add_view(
+            handle_exceptions, context=HTTPForbidden, permission=NO_PERMISSION_REQUIRED
+        )

kinto/core/cornice/cors.py

@@ -0,0 +1,144 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+import fnmatch
+import functools
+
+from pyramid.settings import asbool
+
+
+CORS_PARAMETERS = (
+    "cors_headers",
+    "cors_enabled",
+    "cors_origins",
+    "cors_credentials",
+    "cors_max_age",
+    "cors_expose_all_headers",
+)
+
+
+def get_cors_preflight_view(service):
+    """Return a view for the OPTION method.
+
+    Checks that the User-Agent is authorized to do a request to the server, and
+    to this particular service, and add the various checks that are specified
+    in http://www.w3.org/TR/cors/#resource-processing-model.
+    """
+
+    def _preflight_view(request):
+        response = request.response
+        origin = request.headers.get("Origin")
+        supported_headers = service.cors_supported_headers_for()
+
+        if not origin:
+            request.errors.add("header", "Origin", "this header is mandatory")
+
+        requested_method = request.headers.get("Access-Control-Request-Method")
+        if not requested_method:
+            request.errors.add(
+                "header", "Access-Control-Request-Method", "this header is mandatory"
+            )
+
+        if not (requested_method and origin):
+            return
+
+        requested_headers = request.headers.get("Access-Control-Request-Headers", ())
+
+        if requested_headers:
+            requested_headers = map(str.strip, requested_headers.split(","))
+
+        if requested_method not in service.cors_supported_methods:
+            request.errors.add("header", "Access-Control-Request-Method", "Method not allowed")
+
+        if not service.cors_expose_all_headers:
+            for h in requested_headers:
+                if h.lower() not in [s.lower() for s in supported_headers]:
+                    request.errors.add(
+                        "header", "Access-Control-Request-Headers", 'Header "%s" not allowed' % h
+                    )
+
+        supported_headers = set(supported_headers) | set(requested_headers)
+
+        response.headers["Access-Control-Allow-Headers"] = ",".join(supported_headers)
+
+        response.headers["Access-Control-Allow-Methods"] = ",".join(service.cors_supported_methods)
+
+        max_age = service.cors_max_age_for(requested_method)
+        if max_age is not None:
+            response.headers["Access-Control-Max-Age"] = str(max_age)
+
+        return None
+
+    return _preflight_view
+
+
+def _get_method(request):
+    """Return what's supposed to be the method for CORS operations.
+    (e.g if the verb is options, look at the A-C-Request-Method header,
+    otherwise return the HTTP verb).
+    """
+    if request.method == "OPTIONS":
+        method = request.headers.get("Access-Control-Request-Method", request.method)
+    else:
+        method = request.method
+    return method
+
+
+def ensure_origin(service, request, response=None, **kwargs):
+    """Ensure that the origin header is set and allowed."""
+    response = response or request.response
+
+    # Don't check this twice.
+    if not request.info.get("cors_checked", False):
+        method = _get_method(request)
+
+        origin = request.headers.get("Origin")
+
+        if not origin:
+            always_cors = asbool(request.registry.settings.get("cornice.always_cors"))
+            # With this setting, if the service origins has "*", then
+            # always return CORS headers.
+            origins = getattr(service, "cors_origins", [])
+            if always_cors and "*" in origins:
+                origin = "*"
+
+        if origin:
+            if not any([fnmatch.fnmatchcase(origin, o) for o in service.cors_origins_for(method)]):
+                request.errors.add("header", "Origin", "%s not allowed" % origin)
+            elif service.cors_support_credentials_for(method):
+                response.headers["Access-Control-Allow-Origin"] = origin
+            else:
+                if any([o == "*" for o in service.cors_origins_for(method)]):
+                    response.headers["Access-Control-Allow-Origin"] = "*"
+                else:
+                    response.headers["Access-Control-Allow-Origin"] = origin
+        request.info["cors_checked"] = True
+    return response
+
+
+def get_cors_validator(service):
+    return functools.partial(ensure_origin, service)
+
+
+def apply_cors_post_request(service, request, response):
+    """Handles CORS-related post-request things.
+
+    Add some response headers, such as the Expose-Headers and the
+    Allow-Credentials ones.
+    """
+    response = ensure_origin(service, request, response)
+    method = _get_method(request)
+
+    if (
+        service.cors_support_credentials_for(method)
+        and "Access-Control-Allow-Credentials" not in response.headers
+    ):
+        response.headers["Access-Control-Allow-Credentials"] = "true"
+
+    if request.method != "OPTIONS":
+        # Which headers are exposed?
+        supported_headers = service.cors_supported_headers_for(request.method)
+        if supported_headers:
+            response.headers["Access-Control-Expose-Headers"] = ", ".join(supported_headers)
+
+    return response

kinto/core/cornice/errors.py

@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+import json
+
+from pyramid.i18n import TranslationString
+
+
+class Errors(list):
+    """Holds Request errors"""
+
+    def __init__(self, status=400, localizer=None):
+        self.status = status
+        self.localizer = localizer
+        super(Errors, self).__init__()
+
+    def add(self, location, name=None, description=None, **kw):
+        """Registers a new error."""
+        allowed = ("body", "querystring", "url", "header", "path", "cookies", "method")
+        if location != "" and location not in allowed:
+            raise ValueError("%r not in %s" % (location, allowed))
+
+        if isinstance(description, TranslationString) and self.localizer:
+            description = self.localizer.translate(description)
+
+        self.append(dict(location=location, name=name, description=description, **kw))
+
+    @classmethod
+    def from_json(cls, string):
+        """Transforms a json string into an `Errors` instance"""
+        obj = json.loads(string.decode())
+        return Errors.from_list(obj.get("errors", []))
+
+    @classmethod
+    def from_list(cls, obj):
+        """Transforms a python list into an `Errors` instance"""
+        errors = Errors()
+        for error in obj:
+            errors.add(**error)
+        return errors