py-app-runner 0.5.49.dev0__py3-none-any.whl

py_app_runner/pybridge.py

@@ -0,0 +1,64 @@
+from importlib import import_module
+from typing import Any, Literal
+
+from py_app_runner.db_pools import DbPools
+
+
+class PyBridge:
+    services_placeholder: dict[str, Any]
+    db_pools: DbPools
+
+    def __init__(self) -> None:
+        self.services_placeholder = {}
+
+    def _import_or_none(self, module_name: str) -> Any | None:
+        """Imports `module_name`, returning None only when that module - or one of its own
+        ancestor packages - is the thing that is missing. An ImportError naming anything
+        else is a broken import *inside* the module and must surface - swallowing it would
+        let a typo in a project service silently fall through to the built-in of the same
+        name."""
+
+        try:
+            return import_module(module_name)
+        except ModuleNotFoundError as e:
+            if e.name is not None and (e.name == module_name or module_name.startswith(f"{e.name}.")):
+                return None
+
+            raise
+
+    def _resolve(self, service_name: str, module: str) -> Any | Literal[False]:
+        """Project services win over built-ins, so a project can override `bridge` or
+        `migrations` by shipping its own `services.<name>.<module>`."""
+
+        for package in (f"services.{service_name}", f"py_app_runner.{service_name}"):
+            found = self._import_or_none(f"{package}.{module}")
+            if found is not None:
+                return found
+
+        return False
+
+    def load_service(self, service_name: str) -> Any:
+        service = self._resolve(service_name, "_service")
+        if service is not False:
+            return service
+
+        return import_module(f"services.{service_name}")
+
+    def load_service_args(self, service_name: str) -> Any | Literal[False]:
+        return self._resolve(service_name, "_service_args")
+
+    def load_service_runner(self, service_name: str) -> Any | Literal[False]:
+        return self._resolve(service_name, "_service")
+
+    def load_service_pybridge(self, service_name: str) -> Any | Literal[False]:
+        return self._resolve(service_name, "_service_pybridge")
+
+    def cache_service(self, service_name: str, service: Any) -> None:
+        self.services_placeholder[service_name] = service
+
+    def register_alias(self, name: str, handler: Any) -> None:
+        """Expose an already-loaded handler under an additional service name."""
+        self.services_placeholder[name] = handler
+
+    def get_service(self, service_name: str) -> Any | Literal[False]:
+        return self.services_placeholder.get(service_name, False)

py_app_runner/queue/__init__.py

@@ -0,0 +1,25 @@
+"""Jobs on PostgreSQL, or on Redis streams behind the same interface.
+
+The database driver is the one to reach for first: a push there is an INSERT, so it joins
+the transaction that caused it. Redis cannot do that, whatever it is set to.
+"""
+
+from py_app_runner.queue.driver_pg import PgQueue
+from py_app_runner.queue.driver_redis import RedisQueue
+from py_app_runner.queue.handler import Handler, assert_resolvable, resolve
+from py_app_runner.queue.interface import QueueDriver
+from py_app_runner.queue.job import Job, QueueError
+from py_app_runner.queue.worker import Worker, backoff
+
+__all__ = [
+    "Handler",
+    "Job",
+    "PgQueue",
+    "QueueDriver",
+    "QueueError",
+    "RedisQueue",
+    "Worker",
+    "assert_resolvable",
+    "backoff",
+    "resolve",
+]

py_app_runner/queue/_service.py

@@ -0,0 +1,231 @@
+import asyncio
+import logging
+import os
+import pathlib
+import signal
+import socket
+from argparse import Namespace
+from datetime import UTC, datetime
+from typing import Any, cast
+
+import psycopg
+from database_wrapper_redis import RedisDbWithPoolAsync
+
+from py_app_runner.migrations._service import connect_kwargs, resolve_targets
+from py_app_runner.pybridge import PyBridge
+from py_app_runner.queue.commands import (
+    Out,
+    cmd_failed,
+    cmd_forget,
+    cmd_install,
+    cmd_retry,
+    cmd_status,
+    parse_queues,
+    resolve_handlers,
+)
+from py_app_runner.queue.driver_pg import PgQueue
+from py_app_runner.queue.driver_redis import RedisQueue
+from py_app_runner.queue.interface import QueueDriver
+from py_app_runner.queue.job import QueueError
+from py_app_runner.queue.worker import Worker
+from py_app_runner.registry import AppRegistry
+
+_DEFAULTS: dict[str, Any] = {
+    # "database" first because a push there joins the caller's transaction, which is the
+    # one thing redis cannot offer however fast it is.
+    "driver": "database",
+    "db": "main",
+    "table": "queue_jobs",
+    "failed_table": "queue_failed_jobs",
+    # Deliberately not the cache connection: a queue that shares a database with a cache is
+    # one FLUSHDB away from an empty backlog.
+    "redis": {},
+    "queue": "default",
+    "tries": 3,
+    "backoff": [10, 60, 300],
+    "timeout": 300,
+    "sleep": 1.0,
+    "handlers": {},
+}
+
+
+def _settings(config: dict[str, Any]) -> dict[str, Any]:
+    return {**_DEFAULTS, **(config.get("queue") or {})}
+
+
+def _migrations_dir(args: Namespace, config: dict[str, Any]) -> pathlib.Path:
+    raw = getattr(args, "dir", None)
+    if raw:
+        directory = pathlib.Path(raw)
+        if directory.is_absolute():
+            return directory
+
+        return pathlib.Path(config.get("current_path") or ".") / directory
+
+    return next(iter(resolve_targets(config).values())).directory
+
+
+def _install_shutdown_handlers(worker: Worker, logger: logging.Logger) -> None:
+    """SIGTERM and SIGINT ask the loop to stop; nothing interrupts the job in hand.
+
+    Installed on the running loop rather than through signal.signal, so the flag is set
+    between awaits rather than inside arbitrary C code. A second signal is left to the
+    default handler: if the first one is being ignored because a job is wedged, the
+    operator needs a way out that does not require finding the pid.
+    """
+
+    loop = asyncio.get_running_loop()
+    for name in ("SIGTERM", "SIGINT"):
+        try:
+            loop.add_signal_handler(getattr(signal, name), worker.stop)
+        except (NotImplementedError, AttributeError, RuntimeError):
+            # No signal support (Windows, or a non-main thread). The visibility timeout is
+            # still the backstop, so a killed worker's jobs come back on their own.
+            logger.debug("queue: could not install a %s handler; shutdown will not be graceful", name)
+
+
+async def init_service(args: Namespace, _pybridge: PyBridge, logger: logging.Logger) -> None:
+    out: Out = print
+
+    # Same reasoning as migrations: runner.py catches Exception around init_service and
+    # returns normally, which exits 0. A worker that could not reach the database must not
+    # look like a worker that ran cleanly.
+    code = 1
+    try:
+        config = AppRegistry.config()
+        settings = _settings(config)
+
+        driver = settings["driver"]
+        if driver not in ("database", "redis"):
+            out(f'error: config["queue"]["driver"] is {driver!r}; it must be "database" or "redis".')
+            raise SystemExit(2)
+
+        if args.step == "install":
+            if driver == "redis":
+                # Nothing to create: the keys appear on first push. Exiting 0 rather than
+                # refusing, so a deploy playbook can run `queue install` unconditionally
+                # whichever driver a project happens to be on.
+                out("Nothing to install: this queue is not on a database.")
+                raise SystemExit(0)
+
+            raise SystemExit(
+                cmd_install(
+                    _migrations_dir(args, config),
+                    settings["table"],
+                    settings["failed_table"],
+                    datetime.now(UTC),
+                    out,
+                )
+            )
+
+        if driver == "redis":
+            # No database connection is opened at all on this path, so a worker host running
+            # the redis driver needs no database credentials.
+            #
+            # `prefix` and `group` are this module's keys and are not part of the connector's
+            # config, so they are split out rather than handed to it.
+            redis_settings = dict(settings["redis"] or {})
+            prefix = redis_settings.pop("prefix", None) or "queue:"
+            group = redis_settings.pop("group", None) or "workers"
+
+            pool = RedisDbWithPoolAsync(cast(Any, redis_settings))
+            try:
+                async with pool as redis_con:
+                    queue = RedisQueue(redis_con, prefix, group)
+                    code = await _dispatch(args, settings, config, queue, logger, out)
+            finally:
+                await pool.close()
+
+            raise SystemExit(code)
+
+        db_name = settings["db"]
+        db_config = config.get("db") or {}
+        if db_name not in db_config:
+            out(
+                f'error: no database {db_name!r} in config["db"]; '
+                f"configured are: {', '.join(sorted(db_config)) or 'none'}."
+            )
+            raise SystemExit(2)
+
+        async with await psycopg.AsyncConnection.connect(**connect_kwargs(db_config[db_name])) as conn:
+            code = await _dispatch(
+                args, settings, config, PgQueue(conn, settings["table"], settings["failed_table"]), logger, out
+            )
+
+    except QueueError as e:
+        out(f"error: {e}")
+        raise SystemExit(1) from None
+    except (KeyboardInterrupt, asyncio.CancelledError):
+        # Above `except Exception` because both derive from BaseException. A worker
+        # interrupted mid-job leaves that job reserved until its deadline passes, which is
+        # correct - but it must not be reported as a clean run.
+        logger.error("queue: interrupted")
+        raise SystemExit(1) from None
+    except Exception:
+        logger.exception("queue: unhandled failure")
+        raise SystemExit(1) from None
+
+    raise SystemExit(code)
+
+
+async def _dispatch(
+    args: Namespace,
+    settings: dict[str, Any],
+    config: dict[str, Any],
+    queue: QueueDriver,
+    logger: logging.Logger,
+    out: Out,
+) -> int:
+    """Every command below is driver-agnostic, which is the point of the interface: switching
+    `driver` in config changes where the jobs live and nothing else about the CLI."""
+
+    if args.step == "work":
+        return await _work(args, settings, config, queue, logger, out)
+
+    if args.step == "status":
+        return await cmd_status(queue, out)
+
+    if args.step == "failed":
+        return await cmd_failed(queue, args.limit, out)
+
+    if args.step == "retry":
+        return await cmd_retry(queue, args.id, args.all, settings["tries"], out)
+
+    if args.step == "forget":
+        return await cmd_forget(queue, args.id, args.all, args.before, out)
+
+    out(f"error: unknown queue command {args.step!r}")
+    return 1
+
+
+async def _work(
+    args: Namespace,
+    settings: dict[str, Any],
+    config: dict[str, Any],
+    queue: QueueDriver,
+    logger: logging.Logger,
+    out: Out,
+) -> int:
+    worker = Worker(
+        queue=queue,
+        handlers=resolve_handlers(config),
+        backoff_steps=settings["backoff"],
+        out=out,
+        worker_id=f"{socket.gethostname()}:{os.getpid()}"[:64],
+    )
+
+    _install_shutdown_handlers(worker, logger)
+
+    # --once is not run_next(): it is a full run with the limits set, so it still installs
+    # signal handlers and still reports what it did.
+    max_jobs = 1 if args.once else args.max_jobs
+    stop_when_empty = args.stop_when_empty or args.once
+
+    return await worker.run(
+        queues=parse_queues(args.queue, settings["queue"]),
+        timeout=args.timeout if args.timeout is not None else settings["timeout"],
+        sleep=args.sleep if args.sleep is not None else settings["sleep"],
+        max_jobs=max_jobs,
+        max_time=args.max_time,
+        stop_when_empty=stop_when_empty,
+    )

py_app_runner/queue/_service_args.py

@@ -0,0 +1,67 @@
+"""CLI subparsers for the built-in queue service.
+
+    python3 src/app.py queue install [--dir PATH]
+    python3 src/app.py queue work    [--queue a,b] [--once] [--stop-when-empty]
+                                     [--max-jobs N] [--max-time N] [--timeout N] [--sleep N]
+    python3 src/app.py queue status
+    python3 src/app.py queue failed  [--limit N]
+    python3 src/app.py queue retry   (--id N | --all)
+    python3 src/app.py queue forget  (--id N | --all | --before DATE)
+
+There is no --driver flag by design: the driver comes from config, so a push and a worker
+cannot disagree about where the jobs are.
+"""
+
+import logging
+from argparse import ArgumentParser, _SubParsersAction  # type: ignore
+
+from py_app_runner.pybridge import PyBridge
+
+
+def reg_subparsers(
+    subparsers: "_SubParsersAction[ArgumentParser]",
+    _pybridge: PyBridge,
+    _base_logger: logging.Logger,
+) -> None:
+    """Command line subparsers"""
+
+    parser = subparsers.add_parser(
+        "queue",
+        description="Install the queue schema, run workers, and work through failed jobs",
+        help="Job queue",
+    )
+    group = parser.add_subparsers(title="command", dest="step", required=True)
+
+    install_parser = group.add_parser("install", help="Write the queue schema into the migrations directory")
+    install_parser.add_argument("--dir", default=None, help="Migrations directory to write into")
+
+    work_parser = group.add_parser("work", help="Reserve and run jobs until told to stop")
+    work_parser.add_argument("--queue", default=None, help="Comma separated, in precedence order")
+    work_parser.add_argument("--timeout", type=int, default=None, help="Visibility and per-job timeout")
+    work_parser.add_argument("--sleep", type=float, default=None, help="Idle poll interval in seconds")
+    work_parser.add_argument("--max-jobs", type=int, default=0, help="Stop after N jobs (0 = no limit)")
+    work_parser.add_argument("--max-time", type=int, default=0, help="Stop after N seconds (0 = no limit)")
+    work_parser.add_argument(
+        "--stop-when-empty",
+        action="store_true",
+        help="Exit once the queue has nothing due, for a cron-driven worker",
+    )
+    work_parser.add_argument(
+        "--once",
+        action="store_true",
+        help="Run at most one job and exit; implies --stop-when-empty",
+    )
+
+    group.add_parser("status", help="Show what is queued, delayed, held and failed")
+
+    failed_parser = group.add_parser("failed", help="List failed jobs")
+    failed_parser.add_argument("--limit", type=int, default=20, help="How many to show (default: 20)")
+
+    retry_parser = group.add_parser("retry", help="Put failed jobs back on the queue")
+    retry_parser.add_argument("--id", type=int, default=None, help="A single failed job id")
+    retry_parser.add_argument("--all", action="store_true", help="Every failed job")
+
+    forget_parser = group.add_parser("forget", help="Delete failed jobs without requeueing them")
+    forget_parser.add_argument("--id", type=int, default=None, help="A single failed job id")
+    forget_parser.add_argument("--all", action="store_true", help="Every failed job")
+    forget_parser.add_argument("--before", default=None, help="Delete rows failed before this date")

py_app_runner/queue/commands.py

@@ -0,0 +1,180 @@
+"""Command implementations, importable directly for programmatic use."""
+
+import datetime
+import pathlib
+import re
+from collections.abc import Callable
+from typing import Any
+
+import psycopg
+
+from py_app_runner.migrations.discovery import MigrationError, new_filename
+from py_app_runner.queue.interface import QueueDriver
+from py_app_runner.queue.job import QueueError
+
+Out = Callable[[str], None]
+
+_TEMPLATE = pathlib.Path(__file__).parent / "files" / "install.pgsql.sql"
+
+_BEFORE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}( \d{2}:\d{2}:\d{2})?$")
+
+
+def cmd_install(
+    migrations_dir: pathlib.Path,
+    table: str,
+    failed_table: str,
+    now: datetime.datetime,
+    out: Out,
+) -> int:
+    """Write the queue schema into the project's own migrations directory."""
+
+    if not migrations_dir.is_dir():
+        out(f"error: no migrations directory at {migrations_dir}")
+        return 2
+
+    try:
+        schema = _TEMPLATE.read_text(encoding="utf-8")
+    except OSError as e:
+        out(f"error: could not read {_TEMPLATE}: {e}")
+        return 1
+
+    # The failed table is substituted first. "queue_jobs" is a prefix of nothing, but
+    # "queue_failed_jobs" contains "queue_jobs" nowhere while its *index* names derive from
+    # it - do the longer name first and neither can be half-rewritten by the other.
+    if failed_table != "queue_failed_jobs":
+        schema = schema.replace("queue_failed_jobs", failed_table)
+    if table != "queue_jobs":
+        schema = schema.replace("queue_jobs", table)
+
+    try:
+        target = migrations_dir / new_filename("create queue tables", now)
+    except MigrationError as e:
+        out(f"error: {e}")
+        return 2
+
+    if target.exists():
+        out(f"error: {target} already exists")
+        return 1
+
+    try:
+        target.write_text(schema, encoding="utf-8")
+    except OSError as e:
+        out(f"error: could not write {target}: {e}")
+        return 1
+
+    out(f"Wrote {target}")
+    out("Review it, then: python3 src/app.py migrations apply")
+
+    return 0
+
+
+async def cmd_status(queue: QueueDriver, out: Out) -> int:
+    try:
+        rows = await queue.stats()
+        failed = await queue.failed_count()
+    except (psycopg.Error, QueueError) as e:
+        out(f"error: cannot read the queue: {e}")
+        return 1
+
+    if not rows:
+        out("No jobs queued.")
+    else:
+        out(f"{'queue':<24} {'pending':>9} {'delayed':>9} {'held':>9} {'total':>9}")
+        for row in rows:
+            out(f"{row['queue']:<24} {row['pending']:>9} {row['delayed']:>9} {row['reserved']:>9} {row['total']:>9}")
+
+    out("")
+    out(f"failed: {failed}")
+    if failed > 0:
+        out("Read them with: python3 src/app.py queue failed")
+
+    return 0
+
+
+async def cmd_failed(queue: QueueDriver, limit: int, out: Out) -> int:
+    if limit < 1:
+        out("error: --limit must be at least 1")
+        return 2
+
+    try:
+        rows = await queue.failed_rows(limit)
+    except (psycopg.Error, QueueError) as e:
+        out(f"error: cannot read the queue: {e}")
+        return 1
+
+    if not rows:
+        out("Nothing has failed.")
+        return 0
+
+    for row in rows:
+        out(
+            f"#{str(row['id']):<6} {str(row['failed_at']):<26} {row['queue']:<16} "
+            f"{row['name']} ({row['attempts']} attempts)"
+        )
+        # Only the first line: a traceback in a list view buries the next entry.
+        first_line = (row["error"] or "").split("\n", 1)[0]
+        if first_line:
+            out(f"    {first_line}")
+
+    out("")
+    out("Requeue one with: python3 src/app.py queue retry --id N")
+
+    return 0
+
+
+async def cmd_retry(queue: QueueDriver, job_id: int | None, all_jobs: bool, max_attempts: int, out: Out) -> int:
+    if job_id is None and not all_jobs:
+        out("error: retry needs --id N or --all")
+        return 2
+
+    try:
+        count = await queue.retry_failed(job_id, max_attempts)
+    except (psycopg.Error, QueueError) as e:
+        out(f"error: could not requeue: {e}")
+        return 1
+
+    if count == 0:
+        out("Nothing to requeue.")
+        # Naming a specific id that is not there is a failed instruction; --all finding
+        # nothing is a legitimately empty table.
+        return 0 if job_id is None else 1
+
+    out(f"Requeued {count} job(s).")
+    return 0
+
+
+async def cmd_forget(
+    queue: QueueDriver,
+    job_id: int | None,
+    all_jobs: bool,
+    before: str | None,
+    out: Out,
+) -> int:
+    if job_id is None and not all_jobs and before is None:
+        out("error: forget needs --id N, --all or --before DATE")
+        return 2
+
+    if before is not None and _BEFORE_RE.match(before) is None:
+        out(f"error: --before={before!r} must be YYYY-MM-DD or 'YYYY-MM-DD HH:MM:SS'")
+        return 2
+
+    try:
+        count = await queue.forget_failed(job_id, before)
+    except (psycopg.Error, QueueError) as e:
+        out(f"error: could not forget: {e}")
+        return 1
+
+    out(f"Deleted {count} row(s).")
+    return 0
+
+
+def parse_queues(raw: str | None, default: str = "default") -> list[str]:
+    """Comma separated, in precedence order. Blanks dropped so a trailing comma is not a
+    queue named "" that never has work."""
+
+    names = [part.strip() for part in (raw or default).split(",")]
+    return [name for name in names if name] or [default]
+
+
+def resolve_handlers(config: dict[str, Any]) -> dict[str, Any]:
+    return (config.get("queue") or {}).get("handlers") or {}