accordsync-server 0.3.0__py3-none-any.whl

accordsync_server/__init__.py

@@ -0,0 +1,54 @@
+"""The Accord sync server on PostgreSQL, framework-agnostic: push, pull, scopes, compaction."""
+
+from accordsync_core import PROTOCOL_VERSION
+
+from .app import AccordServer, Response
+from .auth import AuthError, create_verifier
+from .compact import CompactionResult, compact
+from .db import create_pool
+from .define import (
+    Access,
+    Auth,
+    Claims,
+    Compaction,
+    Limits,
+    RateLimit,
+    RateLimits,
+    ScopedRecord,
+    ServerDefinition,
+    define_server,
+)
+from .migrations import migrate
+from .ratelimit import Limiter, RateLimiter
+from .sync import FEED_LOCK, BadRequestError, Caller, ForbiddenError, op_hash, pull, push
+
+__all__ = [
+    "FEED_LOCK",
+    "PROTOCOL_VERSION",
+    "Access",
+    "AccordServer",
+    "Auth",
+    "AuthError",
+    "BadRequestError",
+    "Caller",
+    "Claims",
+    "Compaction",
+    "CompactionResult",
+    "ForbiddenError",
+    "Limiter",
+    "Limits",
+    "RateLimit",
+    "RateLimiter",
+    "RateLimits",
+    "Response",
+    "ScopedRecord",
+    "ServerDefinition",
+    "compact",
+    "create_pool",
+    "create_verifier",
+    "define_server",
+    "migrate",
+    "op_hash",
+    "pull",
+    "push",
+]

accordsync_server/__main__.py

@@ -0,0 +1,50 @@
+"""`python -m accordsync_server migrate|compact`.
+
+`migrate` needs only ACCORD_DATABASE_URL. `compact` also needs the server definition:
+`--definition module:attribute` (or ACCORD_SERVER) naming a `ServerDefinition`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import os
+import sys
+
+from .compact import compact
+from .db import create_pool
+from .define import ServerDefinition
+from .migrations import migrate
+
+
+def load_definition(spec: str) -> ServerDefinition:
+    module, _, attr = spec.partition(":")
+    value = getattr(importlib.import_module(module), attr or "server")
+    if not isinstance(value, ServerDefinition):
+        raise SystemExit(f"{spec} is not a ServerDefinition (use define_server(...))")
+    return value
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m accordsync_server")
+    parser.add_argument("command", choices=["migrate", "compact"])
+    parser.add_argument("--database-url", default=os.environ.get("ACCORD_DATABASE_URL"))
+    parser.add_argument("--definition", default=os.environ.get("ACCORD_SERVER"))
+    args = parser.parse_args(argv)
+    if not args.database_url:
+        parser.error("ACCORD_DATABASE_URL (or --database-url) is required")
+    if args.command == "migrate":
+        ran = migrate(args.database_url)
+        print(f"applied {len(ran)} migration(s): {', '.join(ran)}" if ran else "up to date")
+        return 0
+    if not args.definition:
+        parser.error("compact needs --definition module:attribute (or ACCORD_SERVER)")
+    definition = load_definition(args.definition)
+    with create_pool(args.database_url, size=2) as pool:
+        print(json.dumps(compact(pool, definition)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

accordsync_server/app.py

@@ -0,0 +1,281 @@
+"""The framework-agnostic request layer (`app.ts`): one `handle()` call per HTTP request.
+
+Framework adapters (FastAPI, Django, WSGI) only translate their request into
+`handle(method, path, query, headers, body)` and the returned `Response` back.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+import time
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import parse_qs
+
+from accordsync_core import PROTOCOL_VERSION, AccordError, assert_node, canonical_json
+
+from .auth import AuthError, Verifier, create_verifier
+from .define import ServerDefinition
+from .jsnum import js_safe_integer
+from .ratelimit import Limiter, RateLimiter
+from .sync import (
+    BadRequestError,
+    Caller,
+    ForbiddenError,
+    Pool,
+    SyncContext,
+    device_ttl_ms,
+    pull,
+    push,
+    touch_device,
+)
+
+log = logging.getLogger("accordsync_server")
+
+Query = Mapping[str, Sequence[str]] | str
+
+
+@dataclass(slots=True)
+class Response:
+    status: int
+    headers: list[tuple[str, str]] = field(default_factory=list)
+    body: bytes = b""
+
+
+class TooManyRequestsError(Exception):
+    def __init__(self, retry_after_ms: int) -> None:
+        super().__init__("too many requests")
+        self.retry_after_ms = retry_after_ms
+
+
+def now_ms() -> int:
+    return time.time_ns() // 1_000_000
+
+
+def _json(status: int, body: object, headers: Iterable[tuple[str, str]] = ()) -> Response:
+    return Response(
+        status,
+        [("Content-Type", "application/json"), *headers],
+        canonical_json(body).encode("utf-8"),
+    )
+
+
+def _error(status: int, message: str) -> Response:
+    return _json(status, {"error": message})
+
+
+def _reject_constant(name: str) -> object:
+    raise ValueError(f"{name} is not JSON")
+
+
+class AccordServer:
+    """Push, pull and health over one connection pool, with auth, limits, CORS and rate limits."""
+
+    def __init__(
+        self,
+        definition: ServerDefinition,
+        pool: Pool,
+        *,
+        now: Callable[[], int] = now_ms,
+        verifier: Verifier | None = None,
+    ) -> None:
+        self.definition = definition
+        self.pool = pool
+        self.ctx = SyncContext(pool, definition, now)
+        self._verify = verifier or create_verifier(definition.auth)
+        self._limiters: tuple[Limiter, Limiter] | None = None
+        self.reset_rate_limits()
+
+    def reset_rate_limits(self) -> None:
+        """Starts every rate-limit bucket full again (the conformance control API's reset)."""
+        rl = self.definition.rate_limit
+        self._limiters = (
+            None
+            if rl is None
+            else (
+                RateLimiter(rl.per_device, self.ctx.now),
+                RateLimiter(rl.per_user, self.ctx.now),
+            )
+        )
+
+    @property
+    def max_body_bytes(self) -> int:
+        """Adapters may stop reading a body past this (it gets 413 anyway)."""
+        return self.definition.limits.max_body_bytes
+
+    # ------------------------------------------------------------------ dispatch
+
+    def handle(
+        self,
+        method: str,
+        path: str,
+        query: Query,
+        headers: Mapping[str, str],
+        body: bytes = b"",
+    ) -> Response:
+        h = {k.lower(): v for k, v in headers.items()}
+        q = parse_qs(query, keep_blank_values=True) if isinstance(query, str) else query
+        method = method.upper()
+        cors = self._cors(method, h)
+        if cors is not None and cors[0]:
+            res = Response(204, cors[1])
+        else:
+            try:
+                res = self._route(method, path, q, h, body)
+            except AuthError as e:
+                res = _error(401, str(e))
+            except ForbiddenError as e:
+                res = _error(403, str(e))
+            except BadRequestError as e:
+                res = _error(400, str(e))
+            except TooManyRequestsError as e:
+                res = _error(429, "too many requests")
+                res.headers.append(("Retry-After", str(math.ceil(e.retry_after_ms / 1000))))
+            except Exception:
+                log.exception("accord: internal error")
+                res = _error(500, "internal error")
+            if cors is not None:
+                res.headers += cors[1]
+        res.headers.append(("Accord-Protocol", str(PROTOCOL_VERSION)))
+        return res
+
+    def _cors(self, method: str, h: Mapping[str, str]) -> tuple[bool, list[tuple[str, str]]] | None:
+        """(is a preflight, headers to add), or None when CORS is off for this request."""
+        origins = self.definition.cors
+        if not origins:
+            return None
+        origin = h.get("origin")
+        allowed = origin if origin in origins else None
+        out: list[tuple[str, str]] = []
+        if allowed:
+            out.append(("Access-Control-Allow-Origin", allowed))
+        out.append(("Vary", "Origin"))
+        if method == "OPTIONS":
+            out += [
+                ("Access-Control-Max-Age", "600"),
+                ("Access-Control-Allow-Methods", "GET,HEAD,PUT,POST,DELETE,PATCH"),
+                ("Access-Control-Allow-Headers", "Authorization,Accord-Device,Content-Type"),
+            ]
+            return True, out
+        out.append(("Access-Control-Expose-Headers", "Accord-Protocol"))
+        return False, out
+
+    def _route(
+        self,
+        method: str,
+        path: str,
+        q: Mapping[str, Sequence[str]],
+        h: Mapping[str, str],
+        body: bytes,
+    ) -> Response:
+        if path == "/health" and method in ("GET", "HEAD"):
+            return self.health()
+        if path.startswith("/v1/"):
+            limit = self.definition.limits.max_body_bytes
+            length = js_safe_integer(h.get("content-length", "") or "0")
+            if len(body) > limit or (length is not None and length > limit):
+                return _error(413, "request body too large")
+            if path == "/v1/push" and method == "POST":
+                return self._push(h, body)
+            if path == "/v1/pull" and method in ("GET", "HEAD"):
+                return self._pull(h, q)
+        return _error(404, "not found")
+
+    # ------------------------------------------------------------------ routes
+
+    def health(self) -> Response:
+        try:
+            with self.pool.connection(timeout=5) as conn:
+                conn.execute("select 1")
+        except Exception:
+            return _json(503, {"status": "unavailable", "reason": "database unreachable"})
+        return _json(200, {"status": "ok", "protocolVersion": PROTOCOL_VERSION})
+
+    def _caller(self, h: Mapping[str, str]) -> Caller:
+        claims = self._verify(h.get("authorization"))
+        device_id = h.get("accord-device", "")
+        try:
+            assert_node(device_id)
+        except AccordError:
+            raise BadRequestError(
+                "Accord-Device header must be a device id ([A-Za-z0-9_-]{1,64})"
+            ) from None
+        if self._limiters is not None:
+            device, user = self._limiters
+            wait = max(device.take(device_id), user.take(claims["sub"]))
+            if wait > 0:
+                raise TooManyRequestsError(wait)
+        access = self.definition.access(claims)
+        who = Caller(claims["sub"], device_id, tuple(access.read), tuple(access.write))
+        touch_device(self.pool, who, device_ttl_ms(self.definition))
+        return who
+
+    def _push(self, h: Mapping[str, str], body: bytes) -> Response:
+        who = self._caller(h)
+        try:
+            data: Any = json.loads(body.decode("utf-8", "replace"), parse_constant=_reject_constant)
+        except (ValueError, RecursionError):
+            raise BadRequestError("body must be JSON") from None
+        if not (isinstance(data, dict) and set(data) == {"ops"} and isinstance(data["ops"], list)):
+            raise BadRequestError('body must be { "ops": [...] }')
+        return _json(200, push(self.ctx, who, data["ops"]))
+
+    def _pull(self, h: Mapping[str, str], q: Mapping[str, Sequence[str]]) -> Response:
+        who = self._caller(h)
+        cursor = js_safe_integer(_first(q, "cursor", "0"))
+        limit = js_safe_integer(_first(q, "limit", "500"))
+        if cursor is None or cursor < 0:
+            raise BadRequestError("cursor must be an integer ≥ 0")
+        if limit is None or limit < 1:
+            raise BadRequestError("limit must be an integer ≥ 1")
+        return _json(200, pull(self.ctx, who, cursor, limit))
+
+    # ------------------------------------------------------------------ WSGI
+
+    def wsgi(
+        self, environ: Mapping[str, Any], start_response: Callable[..., object]
+    ) -> list[bytes]:
+        """A WSGI application: `AccordServer(...).wsgi` can be served by any WSGI server."""
+        headers = {
+            k[5:].replace("_", "-"): str(v) for k, v in environ.items() if k.startswith("HTTP_")
+        }
+        if environ.get("CONTENT_TYPE"):
+            headers["content-type"] = str(environ["CONTENT_TYPE"])
+        if environ.get("CONTENT_LENGTH"):
+            headers["content-length"] = str(environ["CONTENT_LENGTH"])
+        length = js_safe_integer(headers.get("content-length", "0")) or 0
+        stream = environ["wsgi.input"]
+        # Read at most one byte past the limit: a larger body gets 413 without being buffered.
+        body = stream.read(min(max(length, 0), self.max_body_bytes + 1)) if length > 0 else b""
+        res = self.handle(
+            str(environ.get("REQUEST_METHOD", "GET")),
+            str(environ.get("PATH_INFO", "/")) or "/",
+            str(environ.get("QUERY_STRING", "")),
+            headers,
+            body,
+        )
+        reason = _REASONS.get(res.status, "Unknown")
+        start_response(f"{res.status} {reason}", res.headers)
+        return [res.body]
+
+
+def _first(q: Mapping[str, Sequence[str]], name: str, default: str) -> str:
+    v = q.get(name)
+    return v[0] if v else default
+
+
+_REASONS = {
+    200: "OK",
+    204: "No Content",
+    400: "Bad Request",
+    401: "Unauthorized",
+    403: "ForbiddenError",
+    404: "Not Found",
+    413: "Payload Too Large",
+    429: "Too Many Requests",
+    500: "Internal Server Error",
+    503: "Service Unavailable",
+}

accordsync_server/auth.py

@@ -0,0 +1,63 @@
+"""Verifies `Authorization: Bearer <jwt>` against the app's JWKS (or a dev secret)."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from typing import Any
+
+import jwt
+from jwt.types import Options
+
+from .define import Auth, Claims
+
+
+class AuthError(Exception):
+    """The request is not authenticated: answered 401."""
+
+
+Verifier = Callable[[str | None], Claims]
+
+_BEARER = re.compile(r"Bearer (.+)")
+ASYMMETRIC = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512", "ES256", "ES384", "ES512"]
+ASYMMETRIC.append("EdDSA")
+
+
+def create_verifier(auth: Auth) -> Verifier:
+    """Like the TypeScript server (jose): `exp` and `nbf` checked when present, `iss` and `aud`
+    only when configured, `sub` required. With a JWKS, keys are fetched once and cached."""
+    jwks = (
+        jwt.PyJWKClient(auth.jwks_url, cache_keys=True, lifespan=auth.jwks_cache_seconds)
+        if auth.jwks_url
+        else None
+    )
+    options: Options = {"verify_aud": auth.audience is not None, "verify_iat": False}
+
+    def verify(authorization: str | None) -> Claims:
+        m = _BEARER.fullmatch(authorization or "")
+        if not m:
+            raise AuthError("missing bearer token")
+        token = m[1]
+        try:
+            if jwks is not None:
+                key: Any = jwks.get_signing_key_from_jwt(token).key
+                algorithms = ASYMMETRIC
+            else:
+                key = auth.hs256_secret
+                algorithms = ["HS256"]
+            payload: dict[str, Any] = jwt.decode(
+                token,
+                key,
+                algorithms=algorithms,
+                issuer=auth.issuer,
+                audience=auth.audience,
+                options=options,
+            )
+        except (jwt.PyJWTError, ValueError, TypeError) as e:
+            raise AuthError(f"invalid token: {e}") from e
+        sub = payload.get("sub")
+        if not isinstance(sub, str) or sub == "":
+            raise AuthError('token has no "sub" claim')
+        return payload
+
+    return verify

accordsync_server/compact.py

@@ -0,0 +1,82 @@
+"""Log compaction, from `app/packages/server/src/compact.ts` (ADR-0005, ADR-0008, ADR-0010)."""
+
+from __future__ import annotations
+
+from typing import Any, TypedDict
+
+from accordsync_core import canonical_json, encode_op, parse_op_id
+
+from .define import ServerDefinition
+from .sync import FEED_LOCK, Pool, device_ttl_ms, load_record, op_hash
+
+
+class CompactionResult(TypedDict):
+    watermark: int
+    """Feed position every live device has applied: only ops at or below it were folded."""
+    records: int
+    opsFolded: int
+    tombstonesPruned: int
+
+
+def compact(pool: Pool, definition: ServerDefinition) -> CompactionResult:
+    """Folds the history of records every live device already has into one snapshot per record.
+
+    The snapshot takes the position of the last op it replaces, so live devices (past it) never
+    receive it; devices starting from zero and records entering a scope get it instead.
+    """
+    min_ops = definition.compaction.min_ops
+    ttl_secs = device_ttl_ms(definition) / 1000
+    with pool.connection() as conn, conn.transaction():
+        # Exclusive: waits for running pushes (they hold the lock shared) and holds new ones back.
+        conn.execute("select pg_advisory_xact_lock(%s)", (FEED_LOCK,))
+        conn.execute("select set_config('accord.compaction', 'on', true)")
+        w = conn.execute(
+            "select coalesce((select min(cursor) from devices"
+            " where last_seen > now() - make_interval(secs => %s)), accord_horizon() - 1) as w",
+            (ttl_secs,),
+        ).fetchone()
+        assert w is not None
+        watermark = int(w[0])
+        candidates: list[tuple[Any, ...]] = conn.execute(
+            "select record, max(pos) as last from feed where kind in ('op', 'snapshot')"
+            " group by record having max(pos) <= %s"
+            " and count(*) filter (where kind = 'op') >= %s",
+            (watermark, min_ops),
+        ).fetchall()
+
+        ops_folded = 0
+        for record, last in candidates:
+            replica = load_record(conn, definition, record)
+            ops = replica.ops()
+            row = conn.execute("select scopes from records where record = %s", (record,)).fetchone()
+            if row is None:
+                raise RuntimeError(f"no records row for {record}")
+            if ops:
+                values = ", ".join(["(%s, %s, %s, %s)"] * len(ops))
+                params: list[object] = []
+                for op in ops:
+                    p = parse_op_id(op.op_id)
+                    params += [op.op_id, p.device, p.seq, op_hash(encode_op(op))]
+                conn.execute(
+                    "insert into compacted_ops (op_id, device, op_seq, op_hash)"  # noqa: S608 (placeholders only)
+                    f" values {values} on conflict do nothing",
+                    params,
+                )
+            conn.execute("delete from feed where record = %s and pos <= %s", (record, last))
+            conn.execute(
+                "insert into feed (pos, kind, record, op, scopes)"
+                " values (%s, 'snapshot', %s, %s::jsonb, %s::text[])",
+                (last, record, canonical_json(replica.snapshot_record(record).to_json()), row[0]),
+            )
+            ops_folded += len(ops)
+        # Entries a device can no longer retry (it has pushed past them) are no longer needed.
+        pruned = conn.execute(
+            "delete from compacted_ops c using devices d"
+            " where c.device = d.device_id and c.op_seq < d.push_floor"
+        ).rowcount
+        return {
+            "watermark": watermark,
+            "records": len(candidates),
+            "opsFolded": ops_folded,
+            "tombstonesPruned": max(pruned, 0),
+        }

accordsync_server/db.py

@@ -0,0 +1,25 @@
+"""The connection pool: psycopg 3, autocommit connections, explicit transactions."""
+
+from __future__ import annotations
+
+from psycopg_pool import ConnectionPool
+
+from .sync import Conn, Pool
+
+
+def _configure(conn: Conn) -> None:
+    # Every statement outside `with conn.transaction()` commits on its own; transactions are
+    # always explicit (BEGIN ... COMMIT), so nothing is ever left open between requests.
+    conn.autocommit = True
+
+
+def create_pool(database_url: str, *, size: int = 20, open: bool = True) -> Pool:
+    """A pool of up to `size` connections (the TypeScript server's default is 20)."""
+    pool: Pool = ConnectionPool(
+        database_url,
+        min_size=1,
+        max_size=size,
+        configure=_configure,
+        open=open,
+    )
+    return pool