queryapigate 0.5.0__py3-none-any.whl

queryapigate/params.py

@@ -0,0 +1,275 @@
+"""Parameter definitions of a saved query: checking the definitions, and enforcing them on request values.
+
+A saved query declares its parameters in ``query_parameters``. The short form is just a type
+(``{"id": "int"}``); the full form adds rules::
+
+    {
+        "rating":     {"type": "str", "enum": ["G", "PG", "R"], "default": "PG", "description": "MPAA rating"},
+        "max_length": {"type": "int", "min": 1, "max": 600},
+        "title":      {"type": "str", "required": false, "min_length": 2, "pattern": "[A-Za-z ]+"}
+    }
+
+Rules: ``type`` (int, float, str, bool), ``required`` (default: true unless a ``default`` is given), ``default``,
+``enum``, ``min``/``max`` (numbers), ``min_length``/``max_length``/``pattern`` (text) and ``description``.
+An optional parameter that is neither supplied nor defaulted is bound as NULL.
+
+Definitions are validated strictly when a query is saved. At run time they are read leniently, so older saved
+files with unusual definitions keep working.
+"""
+import math
+import re
+
+from .errors import ApiError
+
+_TYPES = {'int': 'integer', 'integer': 'integer', 'float': 'number', 'number': 'number',
+          'str': 'string', 'string': 'string', 'bool': 'boolean', 'boolean': 'boolean'}
+_RULE_KEYS = {'type', 'required', 'default', 'enum', 'min', 'max', 'min_length', 'max_length', 'pattern',
+              'description'}
+_NAME_RE = re.compile(r'^[A-Za-z_]\w*$')
+_MAX_PATTERN_LENGTH = 500
+
+
+class _Invalid(Exception):
+    """A value or a definition that breaks a rule; the message completes the sentence '<name> ...'."""
+
+
+def _to_bool(text):
+    lowered = text.strip().lower()
+    if lowered in ('true', '1', 'yes'):
+        return True
+    if lowered in ('false', '0', 'no'):
+        return False
+    raise ValueError(text)
+
+
+def _is_number(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
+
+
+def _plain(spec):
+    return {'type': None, 'required': True, 'has_default': False, 'default': None, 'enum': None, 'min': None,
+            'max': None, 'min_length': None, 'max_length': None, 'pattern': None, 'description': None, **spec}
+
+
+def _coerce(type_name, value):
+    """Convert a value (usually a query-string text) to the declared type, or raise _Invalid."""
+    if type_name == 'integer':
+        if isinstance(value, str):
+            try:
+                return int(value)
+            except ValueError:
+                raise _Invalid('must be an integer') from None
+        if isinstance(value, int) and not isinstance(value, bool):
+            return value
+        raise _Invalid('must be an integer')
+    if type_name == 'number':
+        if isinstance(value, str):
+            try:
+                value = float(value)
+            except ValueError:
+                raise _Invalid('must be a number') from None
+        if _is_number(value):
+            return value
+        raise _Invalid('must be a number')
+    if type_name == 'boolean':
+        if isinstance(value, str):
+            try:
+                return _to_bool(value)
+            except ValueError:
+                raise _Invalid('must be true or false') from None
+        if isinstance(value, bool):
+            return value
+        raise _Invalid('must be true or false')
+    if type_name == 'string':
+        if isinstance(value, str):
+            return value
+        raise _Invalid('must be text')
+    if isinstance(value, (str, bool)) or _is_number(value):  # untyped: any scalar
+        return value
+    raise _Invalid('must be text, a number or true/false')
+
+
+def _check(spec, value):
+    """Return the value converted to the declared type after applying every rule, or raise _Invalid."""
+    value = _coerce(spec['type'], value)
+    if spec['enum'] is not None and value not in spec['enum']:
+        raise _Invalid('must be one of: ' + ', '.join(str(v) for v in spec['enum']))
+    if _is_number(value):
+        if spec['min'] is not None and value < spec['min']:
+            raise _Invalid(f"must be at least {spec['min']:g}")
+        if spec['max'] is not None and value > spec['max']:
+            raise _Invalid(f"must be at most {spec['max']:g}")
+    if isinstance(value, str):
+        if spec['min_length'] is not None and len(value) < spec['min_length']:
+            raise _Invalid(f"must be at least {spec['min_length']} characters long")
+        if spec['max_length'] is not None and len(value) > spec['max_length']:
+            raise _Invalid(f"must be at most {spec['max_length']} characters long")
+        if spec['pattern'] is not None and not re.fullmatch(spec['pattern'], value):
+            raise _Invalid(f"must match the pattern {spec['pattern']}")
+    return value
+
+
+# --------------------------------------------------------------------------------------
+# Definitions
+# --------------------------------------------------------------------------------------
+
+def _parse_strict(spec):
+    if isinstance(spec, str):
+        spec = {'type': spec}
+    if not isinstance(spec, dict):
+        raise _Invalid("must be a type name such as 'int' or an object of rules")
+    unknown = sorted(set(spec) - _RULE_KEYS)
+    if unknown:
+        raise _Invalid(f"has unknown rule(s): {', '.join(unknown)} (allowed: {', '.join(sorted(_RULE_KEYS))})")
+
+    type_name = None
+    if spec.get('type') is not None:
+        type_name = _TYPES.get(str(spec['type']).lower())
+        if type_name is None:
+            raise _Invalid(f"has unknown type '{spec['type']}' (use int, float, str or bool)")
+    parsed = _plain({'type': type_name})
+
+    if 'required' in spec:
+        if not isinstance(spec['required'], bool):
+            raise _Invalid("'required' must be true or false")
+        parsed['required'] = spec['required']
+    elif 'default' in spec:
+        parsed['required'] = False
+    if 'description' in spec:
+        if not isinstance(spec['description'], str):
+            raise _Invalid("'description' must be text")
+        parsed['description'] = spec['description']
+
+    numeric = type_name in (None, 'integer', 'number')
+    textual = type_name in (None, 'string')
+    for key in ('min', 'max'):
+        if key in spec:
+            if not numeric:
+                raise _Invalid(f"'{key}' only applies to numbers (use min_length/max_length for text)")
+            if not _is_number(spec[key]):
+                raise _Invalid(f"'{key}' must be a number")
+            parsed[key] = spec[key]
+    if parsed['min'] is not None and parsed['max'] is not None and parsed['min'] > parsed['max']:
+        raise _Invalid("'min' must not be greater than 'max'")
+    for key in ('min_length', 'max_length'):
+        if key in spec:
+            if not textual:
+                raise _Invalid(f"'{key}' only applies to text")
+            if not isinstance(spec[key], int) or isinstance(spec[key], bool) or spec[key] < 0:
+                raise _Invalid(f"'{key}' must be a non-negative integer")
+            parsed[key] = spec[key]
+    if (parsed['min_length'] is not None and parsed['max_length'] is not None
+            and parsed['min_length'] > parsed['max_length']):
+        raise _Invalid("'min_length' must not be greater than 'max_length'")
+    if 'pattern' in spec:
+        if not textual:
+            raise _Invalid("'pattern' only applies to text")
+        if not isinstance(spec['pattern'], str) or len(spec['pattern']) > _MAX_PATTERN_LENGTH:
+            raise _Invalid(f"'pattern' must be text of at most {_MAX_PATTERN_LENGTH} characters")
+        try:
+            re.compile(spec['pattern'])
+        except re.error as error:
+            raise _Invalid(f"'pattern' is not a valid regular expression ({error})") from None
+        parsed['pattern'] = spec['pattern']
+
+    if 'enum' in spec:
+        if not isinstance(spec['enum'], list) or not spec['enum']:
+            raise _Invalid("'enum' must be a non-empty list")
+        try:
+            parsed['enum'] = [_coerce(type_name, v) for v in spec['enum']]
+        except _Invalid as error:
+            raise _Invalid(f"has an 'enum' value that {error}") from None
+    if 'default' in spec:
+        if spec.get('required') is True:
+            raise _Invalid("cannot be both 'required' and have a 'default'")
+        try:
+            parsed['default'] = _check(parsed, spec['default'])
+        except _Invalid as error:
+            raise _Invalid(f"has a 'default' that {error}") from None
+        parsed['has_default'] = True
+    return parsed
+
+
+def parse_definitions(query_parameters):
+    """Validate the ``query_parameters`` of a query about to be saved; raises ApiError(400) listing every problem."""
+    if not isinstance(query_parameters, dict):
+        raise ApiError('query_parameters must be a JSON object')
+    errors = {}
+    for name, spec in query_parameters.items():
+        if not _NAME_RE.match(name):
+            errors[name] = 'is not a valid parameter name (letters, digits and underscores; not starting with a digit)'
+            continue
+        try:
+            _parse_strict(spec)
+        except _Invalid as error:
+            errors[name] = str(error)
+    if errors:
+        raise ApiError('Invalid query_parameters: ' + '; '.join(f'{n} {m}' for n, m in errors.items()), 400,
+                       errors=errors)
+
+
+def read_definition(spec):
+    """A definition as the normalised rule dict; anything unusable degrades to an untyped, required parameter."""
+    try:
+        return _parse_strict(spec)
+    except _Invalid:
+        if isinstance(spec, str):
+            return _plain({'type': _TYPES.get(spec.lower())})
+        return _plain({})
+
+
+def read_definitions(query_parameters):
+    if not isinstance(query_parameters, dict):
+        return {}
+    return {name: read_definition(spec) for name, spec in query_parameters.items()}
+
+
+# --------------------------------------------------------------------------------------
+# Values
+# --------------------------------------------------------------------------------------
+
+def resolve(declared, supplied, used=None):
+    """Apply a saved query's parameter definitions to the values a request supplied.
+
+    Returns every value to bind: supplied ones converted and checked, defaults filled in, optional ones without
+    a value set to None. Values for undeclared parameters pass through untouched. Declared parameters that the
+    SQL does not use (``used`` is the set of names it does use) are not enforced. Raises ApiError(400) listing
+    every violation.
+    """
+    values = dict(supplied)
+    errors = {}
+    for name, spec in read_definitions(declared).items():
+        if used is not None and name not in used:
+            continue
+        if name not in values:
+            if spec['has_default']:
+                values[name] = spec['default']
+            elif spec['required']:
+                errors[name] = 'is required'
+            else:
+                values[name] = None
+            continue
+        if values[name] is None:
+            if spec['required']:
+                errors[name] = 'must not be null'
+            continue
+        try:
+            values[name] = _check(spec, values[name])
+        except _Invalid as error:
+            errors[name] = str(error)
+    if errors:
+        raise ApiError('Invalid parameters: ' + '; '.join(f'{n} {m}' for n, m in errors.items()), 400,
+                       errors=errors)
+    return values
+
+
+def json_schema(spec):
+    """The OpenAPI/JSON-schema description of a parameter's type and rules."""
+    schema = {'type': spec['type'] or 'string'}
+    for rule, key in (('enum', 'enum'), ('min', 'minimum'), ('max', 'maximum'), ('min_length', 'minLength'),
+                      ('max_length', 'maxLength'), ('pattern', 'pattern')):
+        if spec[rule] is not None:
+            schema[key] = spec[rule]
+    if spec['has_default']:
+        schema['default'] = spec['default']
+    return schema

queryapigate/pool.py

@@ -0,0 +1,146 @@
+"""A small thread-safe pool of live database connections.
+
+Only *idle* connections are pooled: a connection is checked out exclusively by one request, and on the way
+back it is reset (its transaction ended) and kept for reuse - unless the request failed, in which case its
+state is unknown and it is closed instead. The number of concurrent connections is not limited here; the
+pool only bounds how many idle ones are kept around.
+"""
+import atexit
+import hashlib
+import json
+import logging
+import threading
+import time
+from collections import deque
+from contextlib import contextmanager
+
+from . import config
+
+log = logging.getLogger('queryapigate')
+
+# A connection that has been idle for less than this is trusted without a round trip to check it.
+VALIDATE_AFTER = 5.0  # seconds
+
+
+class Session:
+    """A connection plus what the driver wants to remember about it (e.g. limits already applied)."""
+
+    def __init__(self, conn, driver):
+        self.conn = conn
+        self.driver = driver
+        self.state = {}
+        self.idle_since = time.monotonic()
+
+    def close(self):
+        try:
+            self.driver.close(self)
+        except Exception:  # a connection that cannot be closed cleanly is gone anyway
+            log.debug('Error while closing a connection', exc_info=True)
+
+
+def fingerprint(details, read_only):
+    """Pool key: connections are interchangeable when their settings and read-only mode are identical."""
+    blob = json.dumps(details, sort_keys=True, default=str)
+    return hashlib.sha256(blob.encode('utf-8')).hexdigest(), bool(read_only)
+
+
+class ConnectionPool:
+    def __init__(self):
+        self._idle = {}  # key -> deque of Sessions, least recently used on the left
+        self._lock = threading.Lock()
+
+    @contextmanager
+    def checkout(self, driver, details, read_only):
+        key = fingerprint(details, read_only)
+        session = self._acquire(key)
+        if session is None:
+            session = Session(driver.connect(details, read_only), driver)
+        try:
+            yield session
+        except BaseException:
+            session.close()  # after an error its state is unknown - never reuse it
+            raise
+        self._release(key, session)
+
+    # ---- internals ------------------------------------------------------------------
+
+    def _acquire(self, key):
+        while True:
+            with self._lock:
+                queue = self._idle.get(key)
+                if not queue:
+                    return None
+                session = queue.pop()  # most recently used first, so spare connections age out
+                if not queue:
+                    del self._idle[key]
+            idle_for = time.monotonic() - session.idle_since
+            if idle_for > config.pool_idle_timeout():
+                session.close()
+                continue
+            if idle_for > VALIDATE_AFTER:
+                try:
+                    alive = session.driver.is_alive(session)
+                except Exception:
+                    alive = False
+                if not alive:
+                    log.info('Discarding a dead pooled connection')
+                    session.close()
+                    continue
+            return session
+
+    def _release(self, key, session):
+        try:
+            session.driver.reset(session)  # ends the transaction so the next user sees fresh data
+        except Exception:
+            log.debug('Could not reset a connection; closing it', exc_info=True)
+            session.close()
+            return
+        session.idle_since = time.monotonic()
+        surplus = []
+        with self._lock:
+            queue = self._idle.setdefault(key, deque())
+            queue.append(session)
+            while len(queue) > config.pool_size():
+                surplus.append(queue.popleft())
+            surplus.extend(self._expired_locked())
+        for old in surplus:
+            old.close()
+
+    def _expired_locked(self):
+        """Remove and return idle sessions past the idle timeout, across every key (caller holds the lock)."""
+        cutoff = time.monotonic() - config.pool_idle_timeout()
+        expired = []
+        for key in list(self._idle):
+            queue = self._idle[key]
+            while queue and queue[0].idle_since < cutoff:
+                expired.append(queue.popleft())
+            if not queue:
+                del self._idle[key]
+        return expired
+
+    # ---- housekeeping ---------------------------------------------------------------
+
+    def idle_count(self):
+        with self._lock:
+            return sum(len(q) for q in self._idle.values())
+
+    def close_all(self):
+        with self._lock:
+            sessions = [s for queue in self._idle.values() for s in queue]
+            self._idle.clear()
+        for session in sessions:
+            session.close()
+
+
+_default = ConnectionPool()
+atexit.register(_default.close_all)
+
+
+def close_pooled_connections():
+    """Close every idle connection in the shared pool (also runs at interpreter exit)."""
+    _default.close_all()
+
+
+def get_pool():
+    """The shared pool, or None when pooling is disabled (QUERYAPIGATE_POOL_SIZE=0)."""
+    return _default if config.pool_size() > 0 else None

queryapigate/ratelimit.py

@@ -0,0 +1,88 @@
+"""A small in-memory token-bucket rate limiter, off unless QUERYAPIGATE_RATE_LIMIT is set (e.g. ``60/minute``).
+
+Each client (by IP address) has a bucket holding up to the full quota, refilled continuously, so short bursts are
+allowed but the sustained rate is capped. State lives in this process: with several worker processes each has its
+own buckets, so the effective limit is multiplied by the number of workers.
+
+``RateLimiter`` handles the server-wide, IP-keyed limit above. ``KeyRateLimiters`` (below) is the same
+mechanism applied per API key instead, for a key's own optional ``rate_limit`` grant (see ``apikeys.py``) -
+both are checked, not one instead of the other, so a per-key limit can never be used to escape the
+server-wide IP-based one.
+"""
+import math
+import threading
+import time
+from collections import OrderedDict
+
+
+class RateLimiter:
+    MAX_CLIENTS = 100_000  # bounds memory; the least recently seen clients are forgotten first
+
+    def __init__(self, clock=time.monotonic):
+        self._clock = clock
+        self._lock = threading.Lock()
+        self._buckets = OrderedDict()  # client -> [tokens, last_seen]; least recently seen first
+        self._spec = None
+        self._last_sweep = clock()
+
+    def hit(self, client, count, period):
+        """Record a request. Returns (allowed, remaining, retry_after_seconds)."""
+        now = self._clock()
+        with self._lock:
+            if self._spec != (count, period):  # limit changed: start again rather than mix old and new rules
+                self._buckets.clear()
+                self._spec = (count, period)
+            self._sweep(now, period)
+            tokens, seen = self._buckets.pop(client, (float(count), now))
+            tokens = min(float(count), tokens + (now - seen) * count / period)
+            allowed = tokens >= 1
+            if allowed:
+                tokens -= 1
+            self._buckets[client] = (tokens, now)  # re-inserted last: most recently seen
+            if len(self._buckets) > self.MAX_CLIENTS:
+                self._buckets.popitem(last=False)
+            # round first: 20.000000000000004 seconds must not become a 21 second wait
+            retry_after = 0 if allowed else max(1, math.ceil(round((1 - tokens) * period / count, 6)))
+            return allowed, int(tokens), retry_after
+
+    def _sweep(self, now, period):
+        """Forget clients idle for a full period: their bucket has refilled, so they look brand new anyway."""
+        if now - self._last_sweep < period:
+            return
+        self._last_sweep = now
+        while self._buckets:
+            client, (_, seen) = next(iter(self._buckets.items()))
+            if now - seen < period:
+                break
+            del self._buckets[client]
+
+    def size(self):
+        with self._lock:
+            return len(self._buckets)
+
+
+class KeyRateLimiters:
+    """Per-API-key rate limiting needs a genuinely separate RateLimiter per distinct (count, period) spec,
+    not just a different `client` identifier fed into one shared instance: a single RateLimiter enforces
+    exactly one spec at a time and wipes every client's bucket the moment a *different* spec arrives (see
+    its own `hit()` - by design, so an admin changing QUERYAPIGATE_RATE_LIMIT doesn't mix old and new rules for
+    the IP-based limiter). Two keys configured with different `rate_limit` values are exactly that "two
+    different specs" case, so they need their own RateLimiter instances; two keys that happen to share the
+    same configured limit correctly share one instance instead (and so still get independent per-key
+    buckets within it, keyed by name) - there is no reason to multiply instances beyond one per distinct
+    spec actually in use.
+    """
+
+    def __init__(self, clock=time.monotonic):
+        self._clock = clock
+        self._lock = threading.Lock()
+        self._limiters = {}  # (count, period) -> RateLimiter
+
+    def hit(self, key_name, count, period):
+        spec = (count, period)
+        with self._lock:
+            limiter = self._limiters.get(spec)
+            if limiter is None:
+                limiter = RateLimiter(self._clock)
+                self._limiters[spec] = limiter
+        return limiter.hit(key_name, count, period)