py-app-runner 0.5.49.dev0__py3-none-any.whl

py_app_runner/__init__.py

@@ -0,0 +1,11 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from py_app_runner.registry import AppRegistry
+
+__all__ = ["AppRegistry"]
+
+try:
+    __version__ = version("py_app_runner")
+except PackageNotFoundError:
+    # Running straight from a source tree that was never installed
+    __version__ = "0.0.dev0"

py_app_runner/audit/__init__.py

@@ -0,0 +1,29 @@
+"""An append-only trail of who changed what, written explicitly at the call site."""
+
+from py_app_runner.audit.audit import Audit
+from py_app_runner.audit.errors import AuditError
+from py_app_runner.audit.event import (
+    CREATED,
+    DELETED,
+    UPDATED,
+    Actor,
+    AuditEvent,
+    RequestContext,
+    current_context,
+    new_request_id,
+    request_context,
+)
+
+__all__ = [
+    "CREATED",
+    "DELETED",
+    "UPDATED",
+    "Actor",
+    "Audit",
+    "AuditError",
+    "AuditEvent",
+    "RequestContext",
+    "current_context",
+    "new_request_id",
+    "request_context",
+]

py_app_runner/audit/_service.py

@@ -0,0 +1,91 @@
+import asyncio
+import logging
+import pathlib
+from argparse import Namespace
+from datetime import UTC, datetime
+
+import psycopg
+
+from py_app_runner.audit.commands import Out, cmd_install, cmd_prune
+from py_app_runner.audit.errors import AuditError
+from py_app_runner.migrations._service import connect_kwargs, resolve_targets
+from py_app_runner.pybridge import PyBridge
+from py_app_runner.registry import AppRegistry
+
+_DEFAULT_DB = "main"
+_DEFAULT_TABLE = "audit_log"
+
+
+def _migrations_dir(args: Namespace, config: dict) -> pathlib.Path:
+    """Where `install` writes.
+
+    The file it produces is a migration and belongs wherever the rest of them are, so the
+    default comes from the migrations config rather than from an audit key of its own.
+    """
+
+    raw = getattr(args, "dir", None)
+    if raw:
+        directory = pathlib.Path(raw)
+        if directory.is_absolute():
+            return directory
+
+        return pathlib.Path(config.get("current_path") or ".") / directory
+
+    targets = resolve_targets(config)
+    return next(iter(targets.values())).directory
+
+
+async def init_service(args: Namespace, _pybridge: PyBridge, logger: logging.Logger) -> None:
+    out: Out = print
+
+    # Same reasoning as migrations: runner.py catches Exception around init_service and
+    # returns normally, which exits 0. A prune that could not reach the database must not
+    # report success to a retention job.
+    code = 1
+    try:
+        config = AppRegistry.config()
+        settings = config.get("audit") or {}
+        table = getattr(args, "table", None) or settings.get("table") or _DEFAULT_TABLE
+
+        if args.step == "install":
+            raise SystemExit(cmd_install(_migrations_dir(args, config), table, datetime.now(UTC), out))
+
+        db_name = getattr(args, "db", None) or settings.get("db") or _DEFAULT_DB
+        db_config = config.get("db") or {}
+        if db_name not in db_config:
+            out(
+                f'error: no database {db_name!r} in config["db"]; '
+                f"configured are: {', '.join(sorted(db_config)) or 'none'}."
+            )
+            raise SystemExit(2)
+
+        if args.step == "prune":
+            async with await psycopg.AsyncConnection.connect(**connect_kwargs(db_config[db_name])) as conn:
+                code = await cmd_prune(
+                    conn,
+                    table,
+                    args.before,
+                    args.batch,
+                    getattr(args, "dry_run", False),
+                    out,
+                )
+        else:
+            out(f"error: unknown audit command {args.step!r}")
+            code = 1
+
+    except AuditError as e:
+        # Configuration and refusal messages already say what to do; a stack trace above
+        # them would bury it.
+        out(f"error: {e}")
+        raise SystemExit(1) from None
+    except (KeyboardInterrupt, asyncio.CancelledError):
+        # Above `except Exception` because both derive from BaseException. An interrupted
+        # prune has deleted some batches and not others - safe to resume, but it must not
+        # be reported as complete.
+        logger.error("audit: interrupted; the prune is partially done")
+        raise SystemExit(1) from None
+    except Exception:
+        logger.exception("audit: unhandled failure")
+        raise SystemExit(1) from None
+
+    raise SystemExit(code)

py_app_runner/audit/_service_args.py

@@ -0,0 +1,44 @@
+"""CLI subparsers for the built-in audit service.
+
+    python3 src/app.py audit install [--dir PATH] [--table NAME]
+    python3 src/app.py audit prune --before YYYY-MM-DD [--batch N] [--dry-run] [--db NAME]
+
+`--dry-run` shadows a real top-level flag on runner.py's parser and needs
+`default=SUPPRESS`, the same collision migrations' `apply` has.
+"""
+
+import logging
+from argparse import SUPPRESS, ArgumentParser, _SubParsersAction  # type: ignore
+
+from py_app_runner.pybridge import PyBridge
+
+
+def reg_subparsers(
+    subparsers: "_SubParsersAction[ArgumentParser]",
+    _pybridge: PyBridge,
+    _base_logger: logging.Logger,
+) -> None:
+    """Command line subparsers"""
+
+    parser = subparsers.add_parser(
+        "audit",
+        description="Install the audit trail schema and prune old rows",
+        help="Audit trail",
+    )
+    group = parser.add_subparsers(title="command", dest="step", required=True)
+
+    install_parser = group.add_parser("install", help="Write the audit schema into the migrations directory")
+    install_parser.add_argument("--dir", default=None, help="Migrations directory to write into")
+    install_parser.add_argument("--table", default=None, help="Audit table (default: from config)")
+
+    prune_parser = group.add_parser("prune", help="Delete trail rows older than a date")
+    prune_parser.add_argument("--before", required=True, help="Delete rows older than this, YYYY-MM-DD")
+    prune_parser.add_argument("--batch", type=int, default=10000, help="Rows per statement (default: 10000)")
+    prune_parser.add_argument("--table", default=None, help="Audit table (default: from config)")
+    prune_parser.add_argument("--db", default=None, help='Entry of config["db"] to prune in (default: main)')
+    prune_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        default=SUPPRESS,
+        help="Count what would go, delete nothing",
+    )

py_app_runner/audit/audit.py

@@ -0,0 +1,319 @@
+"""The call-site API.
+
+Every audit row is the result of an explicit call. There is no ORM hook, no middleware and
+no database trigger, because a trail that writes itself is one nobody can reason about at
+the call site - and because a hook that does not fire produces a silent gap rather than a
+loud error.
+
+    await audit.insert(cur, "people", {"name": "Anna"}, module="hr")
+    await audit.update(cur, "people", {"status": "left"}, {"id": 42}, module="hr")
+
+`update` and `delete` read the affected rows before writing, so old values are recorded
+without hand-written before-fetches at every call site.
+"""
+
+import logging
+from typing import Any
+
+import psycopg
+from psycopg import sql
+
+from py_app_runner.audit import diff
+from py_app_runner.audit.errors import AuditError
+from py_app_runner.audit.event import CREATED, DELETED, UPDATED, AuditEvent, current_context
+from py_app_runner.audit.store import Store, assert_table_name, qualified
+
+_logger = logging.getLogger(__name__)
+
+_DEFAULTS: dict[str, Any] = {
+    "table": "audit_log",
+    "strict": True,
+    "max_rows": 1000,
+    "id_key": "id",
+    "exclude": {},
+}
+
+
+class Audit:
+    def __init__(
+        self,
+        table: str = "audit_log",
+        strict: bool = True,
+        max_rows: int = 1000,
+        id_key: str = "id",
+        exclude: dict[str, list[str]] | None = None,
+    ) -> None:
+        self.store = Store(table)
+        self.strict = strict
+        self.max_rows = max_rows
+        self.id_key = id_key
+        self.exclude = self._validated_exclude(exclude or {})
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> "Audit":
+        settings = {**_DEFAULTS, **(config.get("audit") or {})}
+
+        max_rows = settings["max_rows"]
+        if not isinstance(max_rows, int) or isinstance(max_rows, bool):
+            raise AuditError(
+                f'config["audit"]["max_rows"] must be an int; got {max_rows!r}. '
+                f"A string here would compare against a row count and silently never match."
+            )
+
+        return cls(
+            table=settings["table"],
+            strict=settings["strict"] is not False,
+            max_rows=max_rows,
+            id_key=settings["id_key"] or "id",
+            exclude=settings["exclude"] or {},
+        )
+
+    def _validated_exclude(self, exclude: dict[str, list[str]]) -> dict[str, list[str]]:
+        """Reject a malformed exclude block at construction.
+
+        Redaction fails open by its nature: a lookup that finds nothing to exclude simply
+        excludes nothing, so a mistake here leaks exactly the values it was meant to
+        withhold, silently and forever. Table names cannot be checked against the database
+        from here, but the shape can be, and that catches the common mistake of writing a
+        bare list where a mapping belongs.
+        """
+
+        if not isinstance(exclude, dict):
+            raise AuditError(
+                f'config["audit"]["exclude"] must be a mapping of table name -> list of '
+                f"columns; got {type(exclude).__name__}."
+            )
+
+        for table, columns in exclude.items():
+            if not isinstance(columns, (list, tuple)) or not all(isinstance(c, str) for c in columns):
+                raise AuditError(
+                    f'config["audit"]["exclude"][{table!r}] must be a list of column names; '
+                    f"got {columns!r}. Anything unreadable here is a redaction that will not happen."
+                )
+
+        return {table: list(columns) for table, columns in exclude.items()}
+
+    def excluded(self, table: str) -> list[str]:
+        return self.exclude.get(table, [])
+
+    ################
+    ### Recording ##
+    ################
+
+    async def record(self, cur: psycopg.AsyncCursor, event: AuditEvent) -> None:
+        """Write one event, on the caller's cursor and inside the caller's transaction."""
+
+        try:
+            await self.store.write(cur, event.with_resolved(current_context()))
+        except Exception as e:
+            self._fail(e)
+
+    def _fail(self, error: Exception) -> None:
+        if self.strict:
+            if isinstance(error, AuditError):
+                raise error
+
+            raise AuditError(f"Audit trail: {error}") from error
+
+        # Availability over completeness, which is rarely the trade an audit trail wants to
+        # make. It is a choice, not a silence: the line is always logged.
+        #
+        # Note what this cannot do on Postgres: a failed INSERT aborts the surrounding
+        # transaction, so swallowing the exception does not rescue the change - it converts
+        # a clear failure into an unattributable one at commit. Callers running inside a
+        # transaction should use a SAVEPOINT around the change if they set strict=False.
+        _logger.warning("Audit trail: %s", error)
+
+    ################
+    ### Wrappers ###
+    ################
+
+    async def insert(
+        self,
+        cur: psycopg.AsyncCursor,
+        table: str,
+        data: dict[str, Any],
+        module: str = "",
+        entity_id: str | None = None,
+        tags: list[str] | None = None,
+        context: dict[str, Any] | None = None,
+    ) -> Any:
+        """Insert a row and record it. Returns the inserted key.
+
+        Always uses RETURNING rather than reading the id back afterwards. Postgres cannot
+        answer "what id did I just insert" without being told which sequence to look at, so
+        every fallback for it either guesses or returns nothing - and returning nothing here
+        quietly empties the column `idx_audit_log_entity` exists to search.
+        """
+
+        assert_table_name(table)
+        columns = list(data.keys())
+
+        statement = sql.SQL("INSERT INTO {rel} ({cols}) VALUES ({vals}) RETURNING {id}").format(
+            rel=qualified(table),
+            cols=sql.SQL(", ").join(sql.Identifier(c) for c in columns),
+            vals=sql.SQL(", ").join(sql.Placeholder() for _ in columns),
+            id=sql.Identifier(self.id_key),
+        )
+
+        await cur.execute(statement, tuple(data[c] for c in columns))
+        returned = await cur.fetchone()
+        new_id = returned[0] if returned else None
+
+        _, new_values = diff.between(None, data, self.excluded(table))
+
+        await self.record(
+            cur,
+            AuditEvent(
+                event=CREATED,
+                entity_type=table,
+                entity_id=str(entity_id if entity_id is not None else (new_id if new_id is not None else "")),
+                module=module,
+                new_values=new_values,
+                tags=tags or [],
+                context=context,
+            ),
+        )
+
+        return new_id
+
+    async def update(
+        self,
+        cur: psycopg.AsyncCursor,
+        table: str,
+        data: dict[str, Any],
+        where: dict[str, Any],
+        module: str = "",
+        tags: list[str] | None = None,
+        context: dict[str, Any] | None = None,
+    ) -> int:
+        """Update rows and record one event per row that actually changed."""
+
+        assert_table_name(table)
+        rows = await self._rows(cur, table, where)
+        self._assert_within_limit(table, len(rows))
+
+        columns = list(data.keys())
+        statement = sql.SQL("UPDATE {rel} SET {sets} WHERE {cond}").format(
+            rel=qualified(table),
+            sets=sql.SQL(", ").join(sql.SQL("{} = {}").format(sql.Identifier(c), sql.Placeholder()) for c in columns),
+            cond=self._condition(where),
+        )
+
+        await cur.execute(statement, tuple(data[c] for c in columns) + tuple(where.values()))
+        affected = cur.rowcount
+
+        if not rows:
+            # Not an error, but it is what a mistyped condition looks like.
+            _logger.debug("Audit trail: update on %r matched no rows", table)
+
+        excluded = self.excluded(table)
+        for row in rows:
+            old_values, new_values = diff.between(row, data, excluded)
+            if new_values is None:
+                continue
+
+            await self.record(
+                cur,
+                AuditEvent(
+                    event=UPDATED,
+                    entity_type=table,
+                    entity_id=self._row_id(row),
+                    module=module,
+                    old_values=old_values,
+                    new_values=new_values,
+                    tags=tags or [],
+                    context=context,
+                ),
+            )
+
+        return affected
+
+    async def delete(
+        self,
+        cur: psycopg.AsyncCursor,
+        table: str,
+        where: dict[str, Any],
+        module: str = "",
+        tags: list[str] | None = None,
+        context: dict[str, Any] | None = None,
+    ) -> int:
+        """Delete rows and record each one, carrying the whole row as old values."""
+
+        assert_table_name(table)
+
+        if not where:
+            # An empty condition builds no WHERE at all, so it deletes the table and audits
+            # every row of it. Nothing below this call would refuse that, so it is refused
+            # here.
+            raise AuditError(
+                f"Refusing to delete every row of {table!r} with an empty condition. "
+                f"Pass an explicit condition, or run the DELETE yourself and record it with record()."
+            )
+
+        rows = await self._rows(cur, table, where)
+        self._assert_within_limit(table, len(rows))
+
+        statement = sql.SQL("DELETE FROM {rel} WHERE {cond}").format(rel=qualified(table), cond=self._condition(where))
+        await cur.execute(statement, tuple(where.values()))
+        affected = cur.rowcount
+
+        excluded = self.excluded(table)
+        for row in rows:
+            old_values, _ = diff.between(row, None, excluded)
+
+            await self.record(
+                cur,
+                AuditEvent(
+                    event=DELETED,
+                    entity_type=table,
+                    entity_id=self._row_id(row),
+                    module=module,
+                    old_values=old_values,
+                    tags=tags or [],
+                    context=context,
+                ),
+            )
+
+        return affected
+
+    ###############
+    ### Helpers ###
+    ###############
+
+    def _condition(self, where: dict[str, Any]) -> sql.Composed:
+        return sql.SQL(" AND ").join(sql.SQL("{} = {}").format(sql.Identifier(c), sql.Placeholder()) for c in where)
+
+    async def _rows(self, cur: psycopg.AsyncCursor, table: str, where: dict[str, Any]) -> list[dict[str, Any]]:
+        """Read the rows a change is about to affect, as dicts."""
+
+        statement = sql.SQL("SELECT * FROM {rel}{cond}").format(
+            rel=qualified(table),
+            cond=sql.SQL(" WHERE ") + self._condition(where) if where else sql.SQL(""),
+        )
+
+        await cur.execute(statement, tuple(where.values()))
+        if cur.description is None:
+            return []
+
+        names = [d.name for d in cur.description]
+        return [dict(zip(names, row, strict=True)) for row in await cur.fetchall()]
+
+    def _row_id(self, row: dict[str, Any]) -> str:
+        value = row.get(self.id_key)
+        return "" if value is None else str(value)
+
+    def _assert_within_limit(self, table: str, matched: int) -> None:
+        """Checked before the write, so a mistyped condition matching the whole table is
+        refused rather than becoming one write plus half a million audit rows."""
+
+        if self.max_rows < 1 or matched <= self.max_rows:
+            return
+
+        self._fail(
+            AuditError(
+                f"Refusing to audit {matched} rows of {table!r} in one call; "
+                f'config["audit"]["max_rows"] is {self.max_rows}. Narrow the condition, or raise '
+                f"the limit."
+            )
+        )

py_app_runner/audit/commands.py

@@ -0,0 +1,151 @@
+"""Command implementations, importable directly for programmatic use."""
+
+import datetime
+import pathlib
+import re
+from collections.abc import Callable
+
+import psycopg
+from psycopg import sql
+
+from py_app_runner.audit.errors import AuditError
+from py_app_runner.audit.store import assert_table_name, qualified
+from py_app_runner.migrations.discovery import MigrationError, new_filename
+
+Out = Callable[[str], None]
+
+_TEMPLATE = pathlib.Path(__file__).parent / "files" / "install.pgsql.sql"
+
+# No relative forms. "yesterday" in a retention job is a question about whose clock, and
+# the answer only ever surfaces once rows are gone.
+_BEFORE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}( \d{2}:\d{2}:\d{2})?$")
+
+
+def cmd_install(migrations_dir: pathlib.Path, table: str, now: datetime.datetime, out: Out) -> int:
+    """Write the schema into the project's own migrations directory.
+
+    The framework ships no migration of its own and never creates the table at runtime:
+    migrations are discovered by filename order and checksummed once applied, so a
+    framework-owned file would sort by dependency release date and turn every upgrade into
+    checksum drift on a file the application cannot edit.
+    """
+
+    try:
+        assert_table_name(table)
+    except AuditError as e:
+        out(f"error: {e}")
+        return 2
+
+    if not migrations_dir.is_dir():
+        out(f"error: no migrations directory at {migrations_dir}")
+        return 2
+
+    try:
+        schema = _TEMPLATE.read_text(encoding="utf-8")
+    except OSError as e:
+        out(f"error: could not read {_TEMPLATE}: {e}")
+        return 1
+
+    if table != "audit_log":
+        # Renames the indexes along with the table, so two trails can coexist in one schema
+        # without their index names colliding.
+        schema = schema.replace("audit_log", table)
+
+    try:
+        target = migrations_dir / new_filename(f"create {table}", now)
+    except MigrationError as e:
+        out(f"error: {e}")
+        return 2
+
+    if target.exists():
+        out(f"error: {target} already exists")
+        return 1
+
+    try:
+        target.write_text(schema, encoding="utf-8")
+    except OSError as e:
+        out(f"error: could not write {target}: {e}")
+        return 1
+
+    out(f"Wrote {target}")
+    out("Review it, then: python3 src/app.py migrations apply")
+
+    return 0
+
+
+async def cmd_prune(
+    conn: psycopg.AsyncConnection,
+    table: str,
+    before: str,
+    batch: int,
+    dry_run: bool,
+    out: Out,
+) -> int:
+    """Delete trail rows older than a date, in batches.
+
+    Batched because a single DELETE over a year of a busy trail takes a lock long enough to
+    be noticed, and because an interrupted run should leave the work it already did done.
+    """
+
+    try:
+        assert_table_name(table)
+    except AuditError as e:
+        out(f"error: {e}")
+        return 2
+
+    if _BEFORE_RE.match(before) is None:
+        out(f"error: --before={before!r} must be YYYY-MM-DD or 'YYYY-MM-DD HH:MM:SS'")
+        return 2
+
+    if batch < 1:
+        out("error: --batch must be at least 1")
+        return 2
+
+    relation = qualified(table)
+
+    try:
+        async with conn.cursor() as cur:
+            await cur.execute(
+                sql.SQL("SELECT count(*) FROM {rel} WHERE created_at < %s").format(rel=relation),
+                (before,),
+            )
+            row = await cur.fetchone()
+            total = row[0] if row else 0
+    except psycopg.Error as e:
+        out(f"error: cannot read {table}: {e}")
+        return 1
+
+    out(f"{total} rows in {table} older than {before}")
+
+    if total == 0:
+        return 0
+
+    if dry_run:
+        out("Nothing deleted (--dry-run).")
+        return 0
+
+    statement = sql.SQL(
+        "DELETE FROM {rel} WHERE id IN (SELECT id FROM {rel} WHERE created_at < %s ORDER BY id LIMIT %s)"
+    ).format(rel=relation)
+
+    deleted = 0
+    while deleted < total:
+        try:
+            async with conn.cursor() as cur:
+                await cur.execute(statement, (before, batch))
+                removed = cur.rowcount
+        except psycopg.Error as e:
+            out(f"error: cannot delete from {table}: {e}")
+            return 1
+
+        if removed <= 0:
+            # Nothing left to take. Breaking rather than looping keeps a miscounted total
+            # from spinning forever.
+            break
+
+        deleted += removed
+        out(f"Deleted {deleted}/{total}")
+
+    out(f"Done. {deleted} rows removed.")
+
+    return 0