py-app-runner 0.5.49.dev0__py3-none-any.whl

py_app_runner/db_pools.py

@@ -0,0 +1,20 @@
+from database_wrapper_pgsql import PgsqlWithPoolingAsync
+from database_wrapper_redis import RedisDbWithPoolAsync
+
+
+class DbPools:
+    """
+    Concrete class to hold database connection pools and wrappers for use
+    throughout the application
+    """
+
+    cache_db_pool: RedisDbWithPoolAsync
+    main_db_pool: PgsqlWithPoolingAsync
+
+    def __init__(
+        self,
+        cache_db_pool: RedisDbWithPoolAsync,
+        main_db_pool: PgsqlWithPoolingAsync,
+    ):
+        self.cache_db_pool = cache_db_pool
+        self.main_db_pool = main_db_pool

py_app_runner/http_exception.py

@@ -0,0 +1,31 @@
+from typing import Any
+
+
+class HTTPException(Exception):
+    message: str
+    code: int
+    description: str | None
+    http_status: int
+    logged: bool
+
+    def __init__(
+        self,
+        message: str,
+        code: int = 1,
+        description: str | None = None,
+        http_status: int = 400,
+        logged: bool = False,
+    ) -> None:
+        super().__init__(message)
+
+        self.message = message
+        self.code = code
+        self.description = description
+        self.http_status = http_status
+        self.logged = logged
+
+    def to_dict(self) -> dict[str, Any]:
+        result: dict[str, Any] = {"msg": self.message, "code": self.code}
+        if self.description:
+            result["description"] = self.description
+        return result

py_app_runner/logger_handlers.py

@@ -0,0 +1,167 @@
+import logging
+import socket
+import sys
+import time
+from argparse import Namespace
+from difflib import SequenceMatcher
+from typing import TypedDict
+
+import sentry_sdk
+from sentry_sdk.integrations.redis import RedisIntegration
+from sentry_sdk.integrations.tornado import TornadoIntegration
+from sentry_sdk.types import Event, Hint
+
+from py_app_runner.registry import AppRegistry
+
+from .colors import Colors
+from .http_exception import HTTPException
+
+# Constants
+ERROR_RATE = 60  # Minute
+
+
+# Types
+class LastError(TypedDict):
+    count: int
+    time: int
+    msg: str
+
+
+# Globals
+last_error: LastError = {"count": 0, "time": 0, "msg": ""}
+logger = logging.getLogger("meta")
+
+
+#################
+### Functions ###
+#################
+
+
+def RateControl(event: Event, hint: Hint) -> Event | None:
+    if "exc_info" in hint:
+        _exc_type, exc_value, _tb = hint["exc_info"]
+        if isinstance(exc_value, HTTPException):
+            return None
+
+    if "extra" in event and "dispatch" in event["extra"] and not event["extra"]["dispatch"]:
+        sys.stderr.write("#### Manually discarded error message meant to be sent via Sentry\n")
+        return None
+
+    now = int(time.time())
+    current_event = str(event)
+    test = SequenceMatcher(None, last_error["msg"], current_event).ratio()
+    if last_error["time"] == 0 or now - int(last_error["time"]) >= ERROR_RATE or test < 0.4:
+        sys.stderr.write("#### Allowing Sentry to send error message\n")
+        last_error["count"] = 0
+        last_error["time"] = now
+        last_error["msg"] = current_event
+        return event
+
+    sys.stderr.write("#### Discarded error message meant to be sent to Sentry\n")
+    last_error["count"] += 1
+    return None
+
+
+def InitSentry(
+    args: Namespace,
+    sentry_dns: str,
+    release: str,
+    environment: str,
+    server_name: str | None = None,
+) -> None:
+    config = AppRegistry.config()
+    if config.get("environment") != "prod":
+        logger.debug(f"Sentry logging disabled in {config['environment']}")
+        return
+
+    logger.debug("Enabling Sentry logging")
+    server_ip = socket.gethostbyname(socket.gethostname())
+    with sentry_sdk.configure_scope() as scope:
+        scope.set_tag("server_ip", server_ip)
+        scope.set_level(args.sv)
+
+    # Tracing/profiling rates come from config when a project sets them; 0 disables.
+    rates = (config.get("sentry") or {}).get("rate") or {}
+
+    sentry_sdk.init(
+        sentry_dns,
+        release=release,
+        environment=environment,
+        server_name=server_name,
+        attach_stacktrace=True,
+        before_send=RateControl,
+        integrations=[TornadoIntegration(), RedisIntegration()],
+        traces_sample_rate=float(rates.get("performance", 0) or 0),
+        profiles_sample_rate=float(rates.get("profiles", 0) or 0),
+        ignore_errors=[KeyboardInterrupt, HTTPException],
+        # debug=True,
+    )
+
+
+###############
+### Classes ###
+###############
+
+
+class ConsoleHandler(logging.Handler):
+    name_filter: str | None
+
+    should_buffer: bool
+    the_buffer: list[logging.LogRecord]
+
+    def __init__(self, filter: str | None = None, level: int = logging.NOTSET):
+        super().__init__(level)
+
+        self.name_filter = filter
+        self.should_buffer = False
+        self.the_buffer = []
+
+    def emit_buffer(self) -> None:
+        self.should_buffer = False
+
+        for record in self.the_buffer:
+            self.emit(record)
+        self.the_buffer = []
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if self.name_filter is not None and not record.name.startswith(self.name_filter):
+            return  # Skip logging for this module
+
+        if self.should_buffer:
+            self.the_buffer.append(record)
+            return
+
+        # Colors
+        prepend = ""
+        append = ""
+        if record.levelno == logging.INFO:
+            prepend = Colors.Yellow
+            append = Colors.ColorOff
+        elif record.levelno == logging.DEBUG:
+            prepend = Colors.Cyan
+            append = Colors.ColorOff
+        elif record.levelno == logging.WARNING:
+            prepend = Colors.Purple
+            append = Colors.ColorOff
+        elif record.levelno == logging.ERROR:
+            prepend = Colors.Red
+            append = Colors.ColorOff
+
+        # End char for printing to console
+        end: str = getattr(record, "end", "\n")
+        prefix: str = getattr(record, "prefix", "")
+
+        # Date
+        date = ""
+        clean = getattr(record, "clean", False)
+        skip_date = getattr(record, "skip_date", False)
+        if not clean:
+            date = f"[{record.levelname}] [{record.name}] "
+            if not skip_date:
+                date = f"[{time.strftime('%d.%m.%Y %H:%M:%S')}] [{record.levelname}] [{record.name}] "
+
+        # Message
+        error_msg = f"{prepend}{date}{self.format(record)}{append}"
+
+        # Print to console
+        sys.stderr.write(f"{prefix}{error_msg}{end}")

py_app_runner/migrations/__init__.py

@@ -0,0 +1,5 @@
+"""Tracked SQL migrations: apply only the files a database has not seen yet."""
+
+from py_app_runner.migrations.discovery import MigrationError
+
+__all__ = ["MigrationError"]

py_app_runner/migrations/_service.py

@@ -0,0 +1,296 @@
+import asyncio
+import logging
+import pathlib
+import socket
+from argparse import Namespace
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+import psycopg
+
+from py_app_runner.migrations.commands import (
+    Out,
+    cmd_apply,
+    cmd_baseline,
+    cmd_new,
+    cmd_repair,
+    cmd_status,
+)
+from py_app_runner.migrations.discovery import MigrationError
+from py_app_runner.pybridge import PyBridge
+from py_app_runner.registry import AppRegistry
+
+_DEFAULT_DIR = "data/migrations"
+_DEFAULT_TABLE = "migrations"
+_DEFAULT_TARGET_NAME = "main"
+
+
+def connect_kwargs(db_config: dict[str, Any]) -> dict[str, Any]:
+    """PgsqlConfig uses hostname/username/database; psycopg wants host/user/dbname.
+
+    autocommit is on because this service manages transactions per migration file
+    explicitly - an implicit outer transaction would swallow the per-file boundaries and
+    make the no-transaction directive impossible to honour.
+    """
+
+    return {
+        "host": db_config["hostname"],
+        "port": db_config.get("port") or 5432,
+        "user": db_config["username"],
+        "password": db_config["password"],
+        "dbname": db_config["database"],
+        "sslmode": db_config.get("ssl") or "prefer",
+        "autocommit": True,
+    }
+
+
+@dataclass(frozen=True)
+class Target:
+    name: str
+    db: str
+    directory: pathlib.Path
+    table: str
+
+
+def _resolve_directory(raw_dir: str | None, config: dict[str, Any]) -> pathlib.Path:
+    directory = pathlib.Path(raw_dir or _DEFAULT_DIR)
+    if directory.is_absolute():
+        return directory
+
+    current_path = config.get("current_path")
+    if current_path is None:
+        raise MigrationError(
+            f'migrations dir {str(directory)!r} is relative but config has no "current_path" to '
+            f"resolve it against; either set current_path or use an absolute dir."
+        )
+
+    return pathlib.Path(current_path) / directory
+
+
+def _database_identity(db_config: dict[str, Any]) -> tuple[Any, Any, Any]:
+    """The physical database a `config["db"]` entry points at.
+
+    Two different db keys can name the same host/port/dbname - a dev box collapsing "main"
+    and "gis" onto one postgres database is the documented case - so the tracking-table
+    collision check below has to compare on this, not on the db key. The port default
+    mirrors connect_kwargs, so both agree on what an absent port means.
+    """
+
+    return (db_config.get("hostname"), db_config.get("port") or 5432, db_config.get("database"))
+
+
+def _reject_shared_tracking_tables(targets: dict[str, Target], db_config: dict[str, Any]) -> None:
+    """Two targets sharing a database is legal and supported; sharing a database *and* a
+    tracking table is not. They would each read the other's rows, see no matching file and
+    report MISSING - and the remediation `apply` prints for MISSING is a DELETE of the
+    tracking row, which de-registers a genuinely applied migration and re-runs it on the
+    next apply. Migrations are explicitly allowed to be non-idempotent, so that is a
+    data-corruption path reached by following the tool's own advice."""
+
+    seen: dict[tuple[Any, Any, Any, str], str] = {}
+    for target in targets.values():
+        identity = (*_database_identity(db_config[target.db]), target.table)
+        clash = seen.get(identity)
+        if clash is not None:
+            raise MigrationError(
+                f"migrations targets {clash!r} and {target.name!r} resolve to the same physical "
+                f"database and both track in table {target.table!r}; each would report the other's "
+                f'migrations as MISSING. Give at least one of them its own "table" under '
+                f'config["migrations"]["targets"]. Sharing a database is fine - sharing a database '
+                f"and a tracking table is not."
+            )
+        seen[identity] = target.name
+
+
+def resolve_targets(config: dict[str, Any]) -> dict[str, Target]:
+    """Resolve `config["migrations"]` into an ordered mapping of target name -> Target.
+
+    Two shapes are supported and never mixed. The flat shape (today's shape, and what all
+    four consuming projects still ship) synthesises a single target named "main" against
+    db "main" - this is the backward-compatibility path and must keep working with zero
+    config changes. The "targets" shape opts a project into multiple named targets, each
+    with its own db/dir/table; presence of the "targets" key alone selects it.
+    """
+
+    settings = config.get("migrations") or {}
+
+    if "targets" in settings:
+        flat_keys = {"dir", "table"} & settings.keys()
+        if flat_keys:
+            raise MigrationError(
+                f'config["migrations"] mixes "targets" with the flat key(s) '
+                f"{', '.join(sorted(flat_keys))}; that is ambiguous, not merged. Move dir/table "
+                f'into each entry under config["migrations"]["targets"] instead.'
+            )
+
+        raw_targets = settings["targets"]
+        if not raw_targets:
+            raise MigrationError(
+                'config["migrations"]["targets"] is present but empty; declare at least one target.'
+            )
+
+        db_config = config.get("db") or {}
+        targets: dict[str, Target] = {}
+        for name, raw_entry in raw_targets.items():
+            entry = raw_entry or {}
+            db_name = entry.get("db") or name
+            if db_name not in db_config:
+                raise MigrationError(
+                    f"migrations target {name!r} names db {db_name!r}, which is not configured under "
+                    f'config["db"]; configured db keys are: {", ".join(sorted(db_config)) or "none"}.'
+                )
+            targets[name] = Target(
+                name=name,
+                db=db_name,
+                directory=_resolve_directory(entry.get("dir"), config),
+                table=entry.get("table") or _DEFAULT_TABLE,
+            )
+
+        _reject_shared_tracking_tables(targets, db_config)
+        return targets
+
+    directory = _resolve_directory(settings.get("dir"), config)
+    table = settings.get("table") or _DEFAULT_TABLE
+    target = Target(name=_DEFAULT_TARGET_NAME, db=_DEFAULT_TARGET_NAME, directory=directory, table=table)
+    return {_DEFAULT_TARGET_NAME: target}
+
+
+# status and apply run every configured target unless narrowed with --target: status is a
+# read-only report, and apply's own per-target failure handling (see below) is what keeps a
+# fan-out safe. new/baseline/repair are single-item: baseline in particular writes tracking
+# rows without ever running the SQL, so a mis-stamped baseline against the wrong target
+# produces a permanently green `status` over a database that never got its tables - the
+# worst failure this tool can produce, silently. Guessing which target that should be is not
+# acceptable, so those three refuse outright when more than one target is configured and
+# --target was not given.
+_FAN_OUT_STEPS = frozenset({"status", "apply"})
+
+
+def _out_for(name: str, multi: bool) -> Out:
+    """Single-target output must stay byte-identical to today, so `print` is used directly
+    below whenever only one target is in play. In the multi-target case, each line is
+    prefixed with its target's name - except blank lines, which commands use as visual
+    separators; a `[name] ` prefix on those would just be noise."""
+
+    if not multi:
+        return print
+
+    def prefixed(line: str) -> None:
+        print(f"[{name}] {line}" if line else "")
+
+    return prefixed
+
+
+def _select_targets(step: str, requested: str | None, targets: dict[str, Target]) -> list[Target]:
+    if requested is not None:
+        if requested not in targets:
+            print(
+                f"error: no migrations target {requested!r}; configured targets are: "
+                f"{', '.join(targets) or 'none'}."
+            )
+            raise SystemExit(1)
+
+        return [targets[requested]]
+
+    if step in _FAN_OUT_STEPS or len(targets) == 1:
+        return list(targets.values())
+
+    print(
+        f"error: migrations {step!r} needs --target since more than one target is configured; "
+        f"configured targets are: {', '.join(targets)}."
+    )
+    raise SystemExit(1)
+
+
+async def init_service(args: Namespace, _pybridge: PyBridge, logger: logging.Logger) -> None:
+    config = AppRegistry.config()
+
+    # runner.py catches Exception around init_service, logs it and returns normally - which
+    # exits 0. For a long-running service that is deliberate, but here it would tell an
+    # unattended playbook that migrations succeeded when the database was merely unreachable
+    # (or, for a multi-target config, mis-declared - an unknown `db` key under
+    # config["migrations"]["targets"] is exactly what resolve_targets raises on), and the
+    # playbook would go on to restart services against an unmigrated schema. Silent success
+    # is the one outcome this tool must never produce, so every non-SystemExit failure -
+    # config resolution included, not just the database connection - is converted into a
+    # non-zero exit here, inside the service, without touching runner.py. SystemExit derives
+    # from BaseException, so _select_targets's own refusals and the happy-path exit below are
+    # not re-wrapped.
+    code = 1
+    try:
+        targets = resolve_targets(config)
+        selected = _select_targets(args.step, getattr(args, "target", None), targets)
+        if not selected:
+            # resolve_targets always returns at least one target, and _select_targets never
+            # returns empty on a path that doesn't already SystemExit - this guards that
+            # invariant rather than trusting it. Falling through silently would exit 0 with
+            # nothing having run, which is exactly the failure mode this function exists to
+            # prevent.
+            raise RuntimeError("migrations: no targets selected; this should be unreachable")
+
+        multi = len(selected) > 1
+
+        if args.step == "new":
+            # Synchronous and needs no database - dispatched before any connection is
+            # opened. _select_targets already refused ambiguity above, so exactly one
+            # target here.
+            target = selected[0]
+            raise SystemExit(
+                cmd_new(target.directory, args.name, datetime.now(UTC), _out_for(target.name, multi))
+            )
+
+        applied_by = f"{config.get('app_version', 'unknown')} @ {socket.gethostname()}"
+
+        code = 0
+        # One connection per target, processed strictly in sequence: two targets can point at
+        # the same physical database (e.g. a dev box collapsing "main" and "gis"), and the
+        # advisory lock tracker.lock() takes would contend with itself under concurrency.
+        for target in selected:
+            logger.debug(f"migrations: target={target.name} dir={target.directory} table={target.table}")
+            out = _out_for(target.name, multi)
+
+            async with await psycopg.AsyncConnection.connect(**connect_kwargs(config["db"][target.db])) as conn:
+                if args.step == "status":
+                    target_code = await cmd_status(conn, target.directory, target.table, args.check, out)
+                elif args.step == "apply":
+                    dry_run = getattr(args, "dry_run", False)
+                    target_code = await cmd_apply(
+                        conn, target.directory, target.table, dry_run, args.to, applied_by, out
+                    )
+                elif args.step == "baseline":
+                    target_code = await cmd_baseline(
+                        conn, target.directory, target.table, args.to, args.yes, applied_by, input, out
+                    )
+                elif args.step == "repair":
+                    target_code = await cmd_repair(conn, target.directory, target.table, args.name, out)
+                else:
+                    out(f"error: unknown migrations command {args.step!r}")
+                    target_code = 1
+
+            if args.step == "apply":
+                # Stop at the first failing target: a broken `main` must never leave a
+                # deploy half-migrated across two databases by ploughing on to the next one.
+                code = target_code
+                if code != 0:
+                    break
+            elif args.step == "status":
+                # Every target is checked regardless of earlier results: `--check` must
+                # report on all of them, not just the first failure.
+                code = code or target_code
+            else:
+                code = target_code
+
+    except (KeyboardInterrupt, asyncio.CancelledError):
+        # Both derive from BaseException, so the `except Exception` below never sees them:
+        # they would sail past init_service into runner.py, which swallows them and returns -
+        # exit 0, with no log line at all. `apply` runs one file at a time, so an interrupt
+        # between two files leaves a partially migrated database while telling the playbook
+        # it succeeded. Must stay above the `except Exception` clause to take effect.
+        logger.error("migrations: interrupted; the database may be partially migrated")
+        raise SystemExit(1) from None
+    except Exception:
+        logger.exception("migrations: unhandled failure")
+        raise SystemExit(1) from None
+
+    raise SystemExit(code)

py_app_runner/migrations/_service_args.py

@@ -0,0 +1,91 @@
+"""CLI subparsers for the built-in migrations service.
+
+    python3 src/app.py migrations status   [--check] [--target NAME]
+    python3 src/app.py migrations apply    [--dry-run] [--to PREFIX] [--target NAME]
+    python3 src/app.py migrations baseline [--to PREFIX] [--yes] [--target NAME]
+    python3 src/app.py migrations new      <name> [--target NAME]
+    python3 src/app.py migrations repair   <filename> [--target NAME]
+
+`--target` has no top-level counterpart on `runner.py`'s parser (unlike `apply`'s
+`--dry-run`, which shadows a real top-level flag and needs `default=SUPPRESS` to avoid
+clobbering it - see the collision test in tests/test_migrations_service.py). A plain
+`default=None` is enough here, and `None` is what lets `init_service` tell "no --target
+given" apart from "given as main".
+"""
+
+import logging
+from argparse import SUPPRESS, ArgumentParser, _SubParsersAction  # type: ignore
+
+from py_app_runner.pybridge import PyBridge
+
+
+def _add_target(parser: ArgumentParser) -> None:
+    parser.add_argument(
+        "--target",
+        type=str,
+        default=None,
+        help='Named migrations target to operate on (see config["migrations"]["targets"])',
+    )
+
+
+def reg_subparsers(
+    subparsers: "_SubParsersAction[ArgumentParser]",
+    _pybridge: PyBridge,
+    _base_logger: logging.Logger,
+) -> None:
+    """Command line subparsers"""
+
+    parser = subparsers.add_parser(
+        "migrations",
+        description="Apply tracked SQL migrations to the configured database target(s)",
+        help="Database migrations",
+    )
+    group = parser.add_subparsers(title="command", dest="step", required=True)
+
+    status_parser = group.add_parser("status", help="Show which migrations are applied and which are pending")
+    status_parser.add_argument(
+        "--check",
+        action="store_true",
+        help="Exit 1 if anything is pending or drifted (for deploy assertions)",
+    )
+    _add_target(status_parser)
+
+    apply_parser = group.add_parser("apply", help="Apply every pending migration, in order")
+    apply_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        default=SUPPRESS,
+        help="List what would run; change nothing",
+    )
+    apply_parser.add_argument(
+        "--to",
+        type=str,
+        default=None,
+        help="Stop after the migration with this YYYY-MM-DD-HHMMSS prefix",
+    )
+    _add_target(apply_parser)
+
+    baseline_parser = group.add_parser(
+        "baseline",
+        help="Record migrations as applied WITHOUT running them (adopting an existing database)",
+    )
+    baseline_parser.add_argument(
+        "--to",
+        type=str,
+        default=None,
+        help="Only offer migrations up to and including this YYYY-MM-DD-HHMMSS prefix",
+    )
+    baseline_parser.add_argument(
+        "--yes",
+        action="store_true",
+        help="Do not ask; stamp every offered migration (for scripts)",
+    )
+    _add_target(baseline_parser)
+
+    new_parser = group.add_parser("new", help="Create an empty, timestamped migration file")
+    new_parser.add_argument("name", type=str, help="Short description, e.g. 'add widgets table'")
+    _add_target(new_parser)
+
+    repair_parser = group.add_parser("repair", help="Re-stamp one migration's checksum after a deliberate edit")
+    repair_parser.add_argument("name", type=str, help="Migration filename, e.g. 2026-08-04-091530-a.sql")
+    _add_target(repair_parser)