supython 0.1.0__py3-none-any.whl

supython/backups/_backup_job.py

@@ -0,0 +1,170 @@
+"""Job handler for admin database backups.
+
+Registered with the jobs framework so backups inherit retries, backoff,
+visibility timeout, and observability instead of running as one-shot
+asyncio tasks tied to the FastAPI process lifetime.
+
+Two execution modes (selected via ``settings.backup_via``):
+
+* ``host``   — invoke ``pg_dump`` from the worker's PATH and use ``-f file``.
+               Suitable for production where ``postgresql-client`` is bundled
+               in the worker image.
+* ``docker`` — ``docker exec`` into the running postgres container and stream
+               pg_dump's stdout into a host-side file. Suitable for the
+               bundled docker-compose dev setup; no host install required and
+               the pg_dump version always matches the server.
+
+Payload shape:
+    {"backup_id": "<uuid>", "kind": "full" | "schema-only", "file_path": "<abs path>"}
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import os
+from uuid import UUID
+
+from ..jobs.decorators import job
+from ..settings import get_settings
+from .service import _parse_db_url
+
+JOB_NAME = "admin_backup_run"
+
+
+def _build_args(*, kind: str, file_path: str) -> tuple[list[str], dict[str, str], bool]:
+    """Compose the argv, env, and "stream stdout to file?" flag.
+
+    Returns (args, env, stream_stdout). When ``stream_stdout`` is True the
+    caller must capture stdout into ``file_path`` itself (used for the
+    docker-exec path, where ``-f`` would write inside the container).
+    """
+    settings = get_settings()
+    db_info = _parse_db_url(settings.database_url)
+    env = os.environ.copy()
+
+    if settings.backup_via == "docker":
+        container = settings.backup_docker_container
+        if not container:
+            raise RuntimeError(
+                "BACKUP_DOCKER_CONTAINER is required when BACKUP_VIA=docker. "
+                "Set it to the name of your postgres container (for scaffolded "
+                "projects this is typically '<project>-db', e.g. 'igo-db'), or "
+                "switch to BACKUP_VIA=host."
+            )
+        args = [
+            "docker",
+            "exec",
+            "-i",
+            "-e",
+            f"PGPASSWORD={db_info['password']}",
+            container,
+            "pg_dump",
+            "-U",
+            db_info["user"],
+            "-d",
+            db_info["dbname"],
+            "--no-owner",
+            "--no-acl",
+        ]
+        if kind == "schema-only":
+            args.append("--schema-only")
+        return args, env, True
+
+    args = [
+        "pg_dump",
+        "-h",
+        db_info["host"],
+        "-p",
+        db_info["port"],
+        "-U",
+        db_info["user"],
+        "-d",
+        db_info["dbname"],
+        "--no-owner",
+        "--no-acl",
+        "-f",
+        file_path,
+    ]
+    if kind == "schema-only":
+        args.append("--schema-only")
+    env["PGPASSWORD"] = db_info["password"]
+    return args, env, False
+
+
+@job(
+    JOB_NAME,
+    max_attempts=2,
+    backoff="exponential",
+    backoff_base_s=30.0,
+    backoff_max_s=300.0,
+)
+async def admin_backup_run(ctx, payload: dict) -> None:
+    backup_id = UUID(payload["backup_id"])
+    kind: str = payload["kind"]
+    file_path: str = payload["file_path"]
+
+    timeout_s = get_settings().backup_timeout_s
+
+    await ctx.db.execute(
+        """
+        update admin.backups
+        set status = 'running', error_message = null
+        where id = $1
+        """,
+        backup_id,
+    )
+
+    try:
+        args, env, stream_stdout = _build_args(kind=kind, file_path=file_path)
+
+        with contextlib.ExitStack() as stack:
+            out_file = stack.enter_context(open(file_path, "wb")) if stream_stdout else None
+            proc = await asyncio.create_subprocess_exec(
+                *args,
+                env=env,
+                stdout=out_file if out_file is not None else asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            try:
+                _stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s)
+            except TimeoutError as exc:
+                proc.kill()
+                with contextlib.suppress(Exception):
+                    await proc.wait()
+                raise RuntimeError(f"pg_dump timed out after {timeout_s}s") from exc
+
+        if proc.returncode != 0:
+            error_text = stderr.decode(errors="replace")[:2000] or "pg_dump failed"
+            raise RuntimeError(error_text)
+
+        file_size = os.path.getsize(file_path)
+        await ctx.db.execute(
+            """
+            update admin.backups
+            set status = 'completed',
+                size = $2,
+                file_path = $3,
+                finished_at = now()
+            where id = $1
+            """,
+            backup_id,
+            file_size,
+            file_path,
+        )
+
+    except Exception as exc:
+        with contextlib.suppress(OSError):
+            os.unlink(file_path)
+        await ctx.db.execute(
+            """
+            update admin.backups
+            set status = 'failed',
+                error_message = $2,
+                finished_at = now()
+            where id = $1
+            """,
+            backup_id,
+            str(exc)[:2000],
+        )
+        raise

supython/backups/schemas.py

@@ -0,0 +1,18 @@
+"""Pydantic v2 models for the backups module."""
+
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+class BackupRecord(BaseModel):
+    id: UUID
+    kind: str
+    status: str
+    size: int | None = None
+    file_path: str | None = None
+    error_message: str | None = None
+    started_at: datetime
+    finished_at: datetime | None = None
+    created_at: datetime

supython/backups/service.py

@@ -0,0 +1,217 @@
+"""Framework-agnostic async service functions for the backups module.
+
+All functions take an asyncpg.Connection and raise BackupError on failure.
+No FastAPI imports here — this module is testable without HTTP.
+
+Backups execute via the jobs framework (see ``_backup_job.py``); this
+module is responsible for the admin.backups bookkeeping and for
+enqueueing the work. The jobs worker is the durable executor.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import logging
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from urllib.parse import urlparse
+from uuid import UUID
+
+import asyncpg
+
+from ..settings import get_settings
+from .schemas import BackupRecord
+
+logger = logging.getLogger(__name__)
+
+_VALID_KINDS = ("full", "schema-only")
+
+
+class BackupError(Exception):
+    def __init__(self, code: str, message: str, status: int = 400) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.status = status
+
+
+def _get_backups_dir() -> Path:
+    s = get_settings()
+    return Path(s.backups_dir).resolve()
+
+
+def _parse_db_url(db_url: str) -> dict[str, str]:
+    parsed = urlparse(db_url)
+    return {
+        "host": parsed.hostname or "localhost",
+        "port": str(parsed.port or 5432),
+        "user": parsed.username or "",
+        "password": parsed.password or "",
+        "dbname": (parsed.path or "/").lstrip("/") or "supython",
+    }
+
+
+def _row_to_record(row: asyncpg.Record) -> BackupRecord:
+    return BackupRecord(
+        id=row["id"],
+        kind=row["kind"],
+        status=row["status"],
+        size=row.get("size"),
+        file_path=row.get("file_path"),
+        error_message=row.get("error_message"),
+        started_at=row["started_at"],
+        finished_at=row.get("finished_at"),
+        created_at=row["created_at"],
+    )
+
+
+async def start_backup(conn: asyncpg.Connection, *, kind: str) -> BackupRecord:
+    """Insert a backup row and enqueue a job to execute the dump.
+
+    Returns the initial record immediately. The jobs worker picks up the
+    queued job, runs pg_dump, and updates admin.backups status/size/file_path
+    when done. If a worker dies mid-job the visibility timeout reclaims it.
+    """
+    if kind not in _VALID_KINDS:
+        raise BackupError(
+            "invalid_kind",
+            f"kind must be one of {_VALID_KINDS}",
+            422,
+        )
+
+    row = await conn.fetchrow(
+        """
+        insert into admin.backups (kind)
+        values ($1)
+        returning id, kind, status, size, file_path, error_message,
+                  started_at, finished_at, created_at
+        """,
+        kind,
+    )
+    if row is None:
+        raise BackupError("insert_failed", "Failed to create backup row", 500)
+    record = _row_to_record(row)
+
+    backups_dir = _get_backups_dir()
+    backups_dir.mkdir(parents=True, exist_ok=True)
+
+    timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    suffix = ".ddl.sql" if kind == "schema-only" else ".sql"
+    filename = f"backup_{record.id}_{kind}_{timestamp}{suffix}"
+    file_path = str(backups_dir / filename)
+
+    # Avoid an import cycle: jobs.service is imported lazily because
+    # ``jobs/__init__`` re-exports the admin router which can transitively
+    # touch this module.
+    from ..jobs.service import enqueue
+    from ._backup_job import JOB_NAME
+
+    await enqueue(
+        conn,
+        name=JOB_NAME,
+        payload={
+            "backup_id": str(record.id),
+            "kind": kind,
+            "file_path": file_path,
+        },
+        idempotency_key=f"admin_backup:{record.id}",
+    )
+
+    return record
+
+
+async def list_backups(
+    conn: asyncpg.Connection,
+    *,
+    limit: int = 50,
+    offset: int = 0,
+) -> list[BackupRecord]:
+    rows = await conn.fetch(
+        """
+        select id, kind, status, size, file_path, error_message,
+               started_at, finished_at, created_at
+        from admin.backups
+        order by created_at desc
+        limit $1 offset $2
+        """,
+        limit,
+        offset,
+    )
+    return [_row_to_record(r) for r in rows]
+
+
+async def get_backup(conn: asyncpg.Connection, backup_id: UUID) -> BackupRecord | None:
+    row = await conn.fetchrow(
+        """
+        select id, kind, status, size, file_path, error_message,
+               started_at, finished_at, created_at
+        from admin.backups
+        where id = $1
+        """,
+        backup_id,
+    )
+    return _row_to_record(row) if row else None
+
+
+async def count_backups(conn: asyncpg.Connection) -> int:
+    val = await conn.fetchval("select count(*) from admin.backups")
+    return val or 0
+
+
+_SIGNED_URL_TTL_S = 600  # 10 minutes
+
+
+def _signing_secret() -> str:
+    """Derive a signing secret from the storage signed URL secret.
+
+    Falls back to a hard-coded dev value when the secret is not configured
+    (mirrors the storage module's dev-friendly posture).
+    """
+    s = get_settings()
+    secret = s.storage_signed_url_secret
+    if secret is None:
+        logger.warning(
+            "backups: STORAGE_SIGNED_URL_SECRET not set; "
+            "download tokens are trivially forgeable in dev mode"
+        )
+        return "backups-dev-secret-not-for-production"
+    return f"backups:{secret}"
+
+
+def generate_download_token(backup_id: UUID) -> str:
+    """Generate a time-limited HMAC token for downloading a backup file.
+
+    Returns an opaque string that embeds the backup_id, expiry, and HMAC
+    signature. Valid for _SIGNED_URL_TTL_S seconds.
+    """
+    expires_at = int(time.time()) + _SIGNED_URL_TTL_S
+    payload = f"{backup_id}:{expires_at}"
+    secret = _signing_secret()
+    sig = hmac.new(secret.encode(), payload.encode(), hashlib.sha256).hexdigest()
+    token = f"{backup_id}:{expires_at}:{sig}"
+    return token
+
+
+def verify_download_token(token: str) -> UUID | None:
+    """Verify a download token. Returns the backup_id if valid, None otherwise."""
+    parts = token.split(":")
+    if len(parts) != 3:
+        return None
+    backup_id_str, expires_str, sig = parts
+    try:
+        expires_at = int(expires_str)
+    except ValueError:
+        return None
+    if time.time() > expires_at:
+        return None
+    payload = f"{backup_id_str}:{expires_at}"
+    secret = _signing_secret()
+    expected = hmac.new(secret.encode(), payload.encode(), hashlib.sha256).hexdigest()
+    if not hmac.compare_digest(expected, sig):
+        return None
+    try:
+        return UUID(backup_id_str)
+    except ValueError:
+        return None

supython/body_size.py

@@ -0,0 +1,184 @@
+"""Reject oversized request bodies before they reach the app.
+
+The cap is the first line of defense for routes that accept JSON/form
+payloads — auth, jobs control plane, realtime control, etc. Anything that
+genuinely streams (storage uploads, functions) is exempted via path
+prefix and governed by its own per-feature setting.
+
+The motivation is concrete: argon2 hashes the entire submitted password,
+so a multi-megabyte password DoS-es a worker. Bound the body, the worry
+goes away.
+"""
+
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from .settings import Settings, get_settings
+
+logger = logging.getLogger(__name__)
+
+ERR_BODY_TOO_LARGE = "body_too_large"
+
+# Methods that may carry a body. We don't gate GET/HEAD/OPTIONS — even if
+# a curious client attaches one, FastAPI ignores it for those methods.
+_BODY_METHODS = frozenset({"POST", "PUT", "PATCH"})
+
+
+class _BodyTooLargeError(Exception):
+    """Raised by the wrapped ASGI receive() once the cap is exceeded.
+
+    Propagates out of FastAPI's body-parsing code (``request.body()`` /
+    ``request.json()``) and is caught by ``BodySizeLimitMiddleware``,
+    which converts it into a 413 response. Custom class so we don't
+    accidentally swallow legitimate exceptions from the inner app.
+    """
+
+
+def _path_matches(path: str, prefixes: tuple[str, ...]) -> bool:
+    return any(path == p or path.startswith(p + "/") for p in prefixes)
+
+
+class BodySizeLimitMiddleware:
+    """ASGI middleware that enforces ``security_max_body_bytes``.
+
+    Two-layer defense:
+
+    1. *Cheap path*: reject up-front when ``Content-Length`` declares a
+       body larger than the cap. The inner app is never invoked.
+    2. *Streaming path*: forward chunks to the inner app as they arrive
+       while counting bytes. As soon as the cap is exceeded, the wrapped
+       receive() raises :class:`_BodyTooLargeError`, which propagates out
+       of the route handler and is caught here. The middleware then sends
+       a 413 — provided the inner app hasn't already started a response.
+
+    Streaming (rather than buffering) keeps memory bounded and lets
+    routes that *do* legitimately stream (storage, functions, exempt by
+    path prefix) operate without a copy.
+    """
+
+    def __init__(self, app: Any, settings: Settings | None = None) -> None:
+        self.app = app
+        self._settings = settings or get_settings()
+        self._max_bytes = self._settings.security_max_body_bytes
+        self._exempt = tuple(
+            p.strip()
+            for p in self._settings.security_body_limit_exempt_paths.split(",")
+            if p.strip()
+        )
+
+    async def __call__(
+        self,
+        scope: dict[str, Any],
+        receive: Callable[[], Awaitable[dict[str, Any]]],
+        send: Callable[[dict[str, Any]], Awaitable[None]],
+    ) -> None:
+        if scope["type"] != "http" or self._max_bytes <= 0:
+            await self.app(scope, receive, send)
+            return
+
+        method = scope.get("method", "").upper()
+        if method not in _BODY_METHODS:
+            await self.app(scope, receive, send)
+            return
+
+        if _path_matches(scope.get("path", ""), self._exempt):
+            await self.app(scope, receive, send)
+            return
+
+        if not await self._content_length_ok(scope, send):
+            return
+
+        await self._enforce_streaming(scope, receive, send)
+
+    async def _content_length_ok(
+        self,
+        scope: dict[str, Any],
+        send: Callable[[dict[str, Any]], Awaitable[None]],
+    ) -> bool:
+        for name, value in scope.get("headers", []):
+            if name == b"content-length":
+                try:
+                    declared = int(value)
+                except ValueError:
+                    await self._send_413(send, "Malformed Content-Length")
+                    return False
+                if declared > self._max_bytes:
+                    await self._send_413(send)
+                    return False
+                break
+        return True
+
+    async def _enforce_streaming(
+        self,
+        scope: dict[str, Any],
+        receive: Callable[[], Awaitable[dict[str, Any]]],
+        send: Callable[[dict[str, Any]], Awaitable[None]],
+    ) -> None:
+        received = 0
+        response_started = False
+
+        async def _bounded_receive() -> dict[str, Any]:
+            nonlocal received
+            msg = await receive()
+            if msg.get("type") == "http.request":
+                received += len(msg.get("body", b""))
+                if received > self._max_bytes:
+                    raise _BodyTooLargeError()
+            return msg
+
+        async def _send_wrapper(msg: dict[str, Any]) -> None:
+            nonlocal response_started
+            if msg["type"] == "http.response.start":
+                response_started = True
+            await send(msg)
+
+        try:
+            await self.app(scope, _bounded_receive, _send_wrapper)
+        except _BodyTooLargeError:
+            if not response_started:
+                await self._send_413(send)
+            else:
+                # The app already committed to a response before we
+                # noticed the overflow. We can't change the status — the
+                # bytes have left the building — but we should make this
+                # visible: the request was incomplete from our side.
+                logger.warning(
+                    "body-size: cap exceeded after response started "
+                    "(path=%s); response delivered but body was truncated",
+                    scope.get("path", ""),
+                )
+
+    async def _send_413(
+        self,
+        send: Callable[[dict[str, Any]], Awaitable[None]],
+        message: str | None = None,
+    ) -> None:
+        msg = message or (
+            f"Request body exceeds maximum size of {self._max_bytes} bytes"
+        )
+        body = (
+            b'{"detail":{"code":"'
+            + ERR_BODY_TOO_LARGE.encode("ascii")
+            + b'","message":"'
+            + msg.encode("utf-8").replace(b'"', b'\\"')
+            + b'"}}'
+        )
+        await send(
+            {
+                "type": "http.response.start",
+                "status": 413,
+                "headers": [
+                    (b"content-type", b"application/json"),
+                    (b"content-length", str(len(body)).encode("ascii")),
+                    (b"connection", b"close"),
+                ],
+            }
+        )
+        await send(
+            {
+                "type": "http.response.body",
+                "body": body,
+                "more_body": False,
+            }
+        )