svc-infra 0.1.595__py3-none-any.whl → 1.1.0__py3-none-any.whl

svc_infra/db/ops.py

@@ -0,0 +1,380 @@
+"""Database operations utilities for one-off administrative tasks.
+
+This module provides synchronous database utilities for operations that
+don't fit the normal async SQLAlchemy workflow, such as:
+- Waiting for database readiness at startup
+- Executing maintenance SQL
+- Dropping tables with lock handling
+- Terminating blocking queries
+
+These utilities use psycopg2 directly for maximum reliability in
+edge cases where the ORM might not be available or appropriate.
+
+Example:
+    >>> from svc_infra.db.ops import wait_for_database, run_sync_sql
+    >>>
+    >>> # Wait for database before app starts
+    >>> wait_for_database(timeout=30)
+    >>>
+    >>> # Run maintenance query
+    >>> run_sync_sql("VACUUM ANALYZE my_table")
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+import time
+from collections.abc import Sequence
+from typing import Any, cast
+
+from .sql.utils import get_database_url_from_env
+
+logger = logging.getLogger(__name__)
+
+# Timeout for individual database operations (seconds)
+DEFAULT_STATEMENT_TIMEOUT = 30
+
+# Default wait-for-database settings
+DEFAULT_WAIT_TIMEOUT = 30
+DEFAULT_WAIT_INTERVAL = 1.0
+
+
+def _flush() -> None:
+    """Force flush stdout/stderr for containerized log visibility."""
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+
+def _get_connection(url: str | None = None, connect_timeout: int = 10) -> Any:
+    """
+    Get a psycopg2 connection.
+
+    Args:
+        url: Database URL. If None, resolved from environment.
+        connect_timeout: Connection timeout in seconds.
+
+    Returns:
+        psycopg2 connection object
+
+    Raises:
+        ImportError: If psycopg2 is not installed
+        RuntimeError: If no database URL is available
+    """
+    try:
+        import psycopg2
+    except ImportError as e:
+        raise ImportError(
+            "psycopg2 is required for db.ops utilities. Install with: pip install psycopg2-binary"
+        ) from e
+
+    if url is None:
+        url = get_database_url_from_env(required=True)
+
+    # Add connect_timeout to connection options
+    return psycopg2.connect(url, connect_timeout=connect_timeout)
+
+
+def wait_for_database(
+    url: str | None = None,
+    timeout: float = DEFAULT_WAIT_TIMEOUT,
+    interval: float = DEFAULT_WAIT_INTERVAL,
+    verbose: bool = True,
+) -> bool:
+    """
+    Wait for database to be ready, with retries.
+
+    Useful for container startup scripts where the database may not
+    be immediately available.
+
+    Args:
+        url: Database URL. If None, resolved from environment.
+        timeout: Maximum time to wait in seconds (default: 30)
+        interval: Time between retry attempts in seconds (default: 1.0)
+        verbose: If True, log progress messages
+
+    Returns:
+        True if database is ready, False if timeout reached
+
+    Example:
+        >>> # In container startup script
+        >>> if not wait_for_database(timeout=60):
+        ...     sys.exit(1)
+        >>> # Database is ready, continue with app startup
+    """
+    if url is None:
+        url = get_database_url_from_env(required=True)
+
+    start = time.monotonic()
+    attempt = 0
+
+    while True:
+        attempt += 1
+        elapsed = time.monotonic() - start
+
+        if elapsed >= timeout:
+            if verbose:
+                logger.error(f"Database not ready after {timeout}s ({attempt} attempts)")
+                _flush()
+            return False
+
+        try:
+            conn = _get_connection(url, connect_timeout=min(5, int(timeout - elapsed)))
+            conn.close()
+            if verbose:
+                logger.info(f"Database ready after {elapsed:.1f}s ({attempt} attempts)")
+                _flush()
+            return True
+        except Exception as e:
+            if verbose:
+                remaining = timeout - elapsed
+                logger.debug(f"Database not ready ({e}), retrying... ({remaining:.0f}s remaining)")
+                _flush()
+            time.sleep(interval)
+
+
+def run_sync_sql(
+    sql: str,
+    params: Sequence[Any] | None = None,
+    url: str | None = None,
+    timeout: int = DEFAULT_STATEMENT_TIMEOUT,
+    fetch: bool = False,
+) -> list[tuple[Any, ...]] | None:
+    """
+    Execute SQL synchronously with a statement timeout.
+
+    This is useful for one-off administrative queries that don't fit
+    the normal async SQLAlchemy workflow.
+
+    Args:
+        sql: SQL statement to execute
+        params: Optional parameters for parameterized queries
+        url: Database URL. If None, resolved from environment.
+        timeout: Statement timeout in seconds (default: 30)
+        fetch: If True, return fetched rows; if False, return None
+
+    Returns:
+        List of tuples if fetch=True, otherwise None
+
+    Raises:
+        psycopg2.Error: On database errors
+        TimeoutError: If statement exceeds timeout
+
+    Example:
+        >>> # Run a maintenance query
+        >>> run_sync_sql("VACUUM ANALYZE users")
+        >>>
+        >>> # Fetch data with timeout
+        >>> rows = run_sync_sql(
+        ...     "SELECT id, name FROM users WHERE active = %s",
+        ...     params=(True,),
+        ...     fetch=True,
+        ...     timeout=10
+        ... )
+    """
+    conn = _get_connection(url)
+    try:
+        with conn.cursor() as cur:
+            # Set statement timeout (PostgreSQL-specific)
+            cur.execute(f"SET statement_timeout = '{timeout}s'")
+
+            if params:
+                cur.execute(sql, params)
+            else:
+                cur.execute(sql)
+
+            if fetch:
+                return cast("list[tuple[Any, ...]]", cur.fetchall())
+
+            conn.commit()
+            return None
+    finally:
+        conn.close()
+
+
+def kill_blocking_queries(
+    table_name: str,
+    url: str | None = None,
+    timeout: int = DEFAULT_STATEMENT_TIMEOUT,
+    dry_run: bool = False,
+) -> list[dict[str, Any]]:
+    """
+    Terminate queries blocking operations on a specific table.
+
+    This is useful before DROP TABLE or ALTER TABLE operations that
+    might be blocked by long-running queries or idle transactions.
+
+    Args:
+        table_name: Name of the table (can include schema as 'schema.table')
+        url: Database URL. If None, resolved from environment.
+        timeout: Statement timeout in seconds (default: 30)
+        dry_run: If True, only report blocking queries without terminating
+
+    Returns:
+        List of dicts with info about terminated (or found) queries:
+        [{"pid": 123, "query": "SELECT...", "state": "active", "terminated": True}]
+
+    Example:
+        >>> # Check what would be terminated
+        >>> blocking = kill_blocking_queries("embeddings", dry_run=True)
+        >>> print(f"Found {len(blocking)} blocking queries")
+        >>>
+        >>> # Actually terminate them
+        >>> kill_blocking_queries("embeddings")
+    """
+    # Query to find blocking queries on a table
+    find_blocking_sql = """
+        SELECT pid, state, query, age(clock_timestamp(), query_start) as duration
+        FROM pg_stat_activity
+        WHERE pid != pg_backend_pid()
+          AND state != 'idle'
+          AND (
+              query ILIKE %s
+              OR query ILIKE %s
+              OR query ILIKE %s
+          )
+        ORDER BY query_start;
+    """
+
+    # Patterns to match queries involving the table
+    patterns = (
+        f"%{table_name}%",
+        f"%{table_name.split('.')[-1]}%",  # Just table name without schema
+        f"%{table_name.replace('.', '%')}%",  # Handle schema.table pattern
+    )
+
+    conn = _get_connection(url)
+    terminated: list[dict[str, Any]] = []
+
+    try:
+        with conn.cursor() as cur:
+            cur.execute(f"SET statement_timeout = '{timeout}s'")
+            cur.execute(find_blocking_sql, patterns)
+            rows = cur.fetchall()
+
+            for pid, state, query, duration in rows:
+                info = {
+                    "pid": pid,
+                    "state": state,
+                    "query": query[:200] + "..." if len(query) > 200 else query,
+                    "duration": str(duration),
+                    "terminated": False,
+                }
+
+                if not dry_run:
+                    try:
+                        cur.execute("SELECT pg_terminate_backend(%s)", (pid,))
+                        info["terminated"] = True
+                        logger.info(f"Terminated query PID {pid}: {query[:100]}...")
+                    except Exception as e:
+                        logger.warning(f"Failed to terminate PID {pid}: {e}")
+                        info["error"] = str(e)
+
+                terminated.append(info)
+
+            conn.commit()
+    finally:
+        conn.close()
+
+    _flush()
+    return terminated
+
+
+def drop_table_safe(
+    table_name: str,
+    url: str | None = None,
+    timeout: int = DEFAULT_STATEMENT_TIMEOUT,
+    kill_blocking: bool = True,
+    if_exists: bool = True,
+    cascade: bool = False,
+) -> bool:
+    """
+    Drop a table safely with lock handling.
+
+    Handles common issues with DROP TABLE:
+    - Terminates blocking queries first (optional)
+    - Uses statement timeout to avoid hanging
+    - Handles 'table does not exist' gracefully
+
+    Args:
+        table_name: Name of table to drop (can include schema)
+        url: Database URL. If None, resolved from environment.
+        timeout: Statement timeout in seconds (default: 30)
+        kill_blocking: If True, terminate blocking queries first (default: True)
+        if_exists: If True, don't error if table doesn't exist (default: True)
+        cascade: If True, drop dependent objects (default: False)
+
+    Returns:
+        True if table was dropped (or didn't exist), False on error
+
+    Example:
+        >>> # Drop table, killing any blocking queries first
+        >>> drop_table_safe("embeddings", cascade=True)
+        True
+        >>>
+        >>> # Safe to call even if table doesn't exist
+        >>> drop_table_safe("nonexistent_table")
+        True
+    """
+    if url is None:
+        url = get_database_url_from_env(required=True)
+
+    # Kill blocking queries first if requested
+    if kill_blocking:
+        blocked = kill_blocking_queries(table_name, url=url, timeout=timeout)
+        if blocked:
+            logger.info(f"Terminated {len(blocked)} blocking queries before DROP")
+            # Brief pause to let connections clean up
+            time.sleep(0.5)
+
+    # Build DROP statement
+    drop_sql = "DROP TABLE"
+    if if_exists:
+        drop_sql += " IF EXISTS"
+    drop_sql += f" {table_name}"
+    if cascade:
+        drop_sql += " CASCADE"
+
+    try:
+        run_sync_sql(drop_sql, url=url, timeout=timeout)
+        logger.info(f"Dropped table: {table_name}")
+        _flush()
+        return True
+    except Exception as e:
+        logger.error(f"Failed to drop table {table_name}: {e}")
+        _flush()
+        return False
+
+
+def get_database_url(
+    required: bool = True,
+    normalize: bool = True,
+) -> str | None:
+    """
+    Convenience wrapper for get_database_url_from_env().
+
+    This is the recommended way to get the database URL, as it
+    handles all common environment variable names and normalizations.
+
+    Args:
+        required: If True, raise RuntimeError when no URL is found
+        normalize: If True, convert postgres:// to postgresql://
+
+    Returns:
+        Database URL string, or None if not found and not required
+
+    Example:
+        >>> url = get_database_url()
+        >>> print(url)
+        'postgresql://user:pass@host:5432/db'
+    """
+    return get_database_url_from_env(required=required, normalize=normalize)
+
+
+__all__ = [
+    "wait_for_database",
+    "run_sync_sql",
+    "kill_blocking_queries",
+    "drop_table_safe",
+    "get_database_url",
+]

svc_infra/db/outbox.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any, Protocol
+
+
+@dataclass
+class OutboxMessage:
+    id: int
+    topic: str
+    payload: dict[str, Any]
+    created_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    attempts: int = 0
+    processed_at: datetime | None = None
+
+
+class OutboxStore(Protocol):
+    def enqueue(self, topic: str, payload: dict[str, Any]) -> OutboxMessage:
+        pass
+
+    def fetch_next(self, *, topics: Iterable[str] | None = None) -> OutboxMessage | None:
+        """Return the next undispatched, unprocessed message (FIFO per-topic), or None.
+
+        Notes:
+        - Messages with attempts > 0 are considered "dispatched" to the job queue and won't be re-enqueued.
+        - Delivery retries are handled by the job queue worker, not by re-reading the outbox.
+        """
+        pass
+
+    def mark_processed(self, msg_id: int) -> None:
+        pass
+
+    def mark_failed(self, msg_id: int) -> None:
+        pass
+
+
+class InMemoryOutboxStore:
+    """Simple in-memory outbox for tests and local runs."""
+
+    def __init__(self):
+        self._seq = 0
+        self._messages: list[OutboxMessage] = []
+
+    def enqueue(self, topic: str, payload: dict[str, Any]) -> OutboxMessage:
+        self._seq += 1
+        msg = OutboxMessage(id=self._seq, topic=topic, payload=dict(payload))
+        self._messages.append(msg)
+        return msg
+
+    def fetch_next(self, *, topics: Iterable[str] | None = None) -> OutboxMessage | None:
+        allowed = set(topics) if topics else None
+        for msg in self._messages:
+            if msg.processed_at is not None:
+                continue
+            # skip already dispatched messages (attempts>0)
+            if msg.attempts > 0:
+                continue
+            if allowed is not None and msg.topic not in allowed:
+                continue
+            return msg
+        return None
+
+    def mark_processed(self, msg_id: int) -> None:
+        for msg in self._messages:
+            if msg.id == msg_id:
+                msg.processed_at = datetime.now(UTC)
+                return
+
+    def mark_failed(self, msg_id: int) -> None:
+        for msg in self._messages:
+            if msg.id == msg_id:
+                msg.attempts += 1
+                return
+
+
+class SqlOutboxStore:
+    """Skeleton for a SQL-backed outbox store.
+
+    Implementations should:
+    - INSERT on enqueue
+    - SELECT FOR UPDATE SKIP LOCKED (or equivalent) to fetch next
+    - UPDATE processed_at (and attempts on failure)
+    """
+
+    def __init__(self, session_factory):
+        self._session_factory = session_factory
+
+    # Placeholders to outline the API; not implemented here.
+    def enqueue(
+        self, topic: str, payload: dict[str, Any]
+    ) -> OutboxMessage:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def fetch_next(
+        self, *, topics: Iterable[str] | None = None
+    ) -> OutboxMessage | None:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def mark_processed(self, msg_id: int) -> None:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def mark_failed(self, msg_id: int) -> None:  # pragma: no cover - skeleton
+        raise NotImplementedError

svc_infra/db/sql/apikey.py

@@ -4,30 +4,47 @@ import hashlib
 import hmac
 import os
 import uuid
-from datetime import datetime, timezone
-from typing import Optional, Type
-
-from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Index, String, UniqueConstraint, text
+from datetime import UTC, datetime
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    DateTime,
+    ForeignKey,
+    Index,
+    String,
+    UniqueConstraint,
+    text,
+)
 from sqlalchemy.ext.mutable import MutableDict, MutableList
 from sqlalchemy.orm import Mapped, declared_attr, mapped_column, relationship
 
+from svc_infra.app.env import require_secret
 from svc_infra.db.sql.base import ModelBase
 from svc_infra.db.sql.types import GUID
 
-_APIKEY_HMAC_SECRET = os.getenv("APIKEY_HASH_SECRET") or "change-me-low-entropy-dev"
+
+def _get_apikey_secret() -> str:
+    """Get APIKEY_HASH_SECRET, requiring it in production."""
+    return require_secret(
+        os.getenv("APIKEY_HASH_SECRET"),
+        "APIKEY_HASH_SECRET",
+        dev_default="dev-only-apikey-hmac-secret-not-for-production",
+    )
 
 
 def _hmac_sha256(s: str) -> str:
-    return hmac.new(_APIKEY_HMAC_SECRET.encode(), s.encode(), hashlib.sha256).hexdigest()
+    secret = _get_apikey_secret()
+    return hmac.new(secret.encode(), s.encode(), hashlib.sha256).hexdigest()
 
 
 def _now() -> datetime:
-    return datetime.now(timezone.utc)
+    return datetime.now(UTC)
 
 
 # -------------------- Factory & registry --------------------
 
-_ApiKeyModel: Optional[type] = None
+_ApiKeyModel: type | None = None
 
 
 def get_apikey_model() -> type:
@@ -37,19 +54,19 @@ def get_apikey_model() -> type:
     return _ApiKeyModel
 
 
-def bind_apikey_model(user_model: Type, *, table_name: str = "api_keys") -> type:
+def bind_apikey_model(user_model: type[ModelBase], *, table_name: str = "api_keys") -> type:
     """
     Create and register an ApiKey model bound to the provided user_model and table name.
     Call this once during app boot (e.g., inside add_auth_users when enable_api_keys=True).
     """
 
-    class ApiKey(ModelBase):  # type: ignore[misc, valid-type]
+    class ApiKey(ModelBase):
         __tablename__ = table_name
 
         id: Mapped[uuid.UUID] = mapped_column(GUID(), primary_key=True, default=uuid.uuid4)
 
         @declared_attr
-        def user_id(cls) -> Mapped[uuid.UUID | None]:  # noqa: N805
+        def user_id(cls) -> Mapped[uuid.UUID | None]:
             return mapped_column(
                 GUID(),
                 ForeignKey(f"{user_model.__tablename__}.id", ondelete="SET NULL"),
@@ -58,7 +75,7 @@ def bind_apikey_model(user_model: Type, *, table_name: str = "api_keys") -> type
             )
 
         @declared_attr
-        def user(cls):  # noqa: N805
+        def user(cls):
             return relationship(user_model.__name__, lazy="selectin")
 
         name: Mapped[str] = mapped_column(String(128), nullable=False)
@@ -74,7 +91,9 @@ def bind_apikey_model(user_model: Type, *, table_name: str = "api_keys") -> type
         meta: Mapped[dict] = mapped_column(MutableDict.as_mutable(JSON), default=dict)
 
         created_at = mapped_column(
-            DateTime(timezone=True), server_default=text("CURRENT_TIMESTAMP"), nullable=False
+            DateTime(timezone=True),
+            server_default=text("CURRENT_TIMESTAMP"),
+            nullable=False,
         )
         updated_at = mapped_column(
             DateTime(timezone=True),
@@ -115,7 +134,7 @@ def bind_apikey_model(user_model: Type, *, table_name: str = "api_keys") -> type
     return ApiKey
 
 
-def try_autobind_apikey_model(*, require_env: bool = False) -> Optional[type]:
+def try_autobind_apikey_model(*, require_env: bool = False) -> type | None:
     """
     If API keys aren’t bound yet, try to discover the User model and bind.
     - If require_env=True, only bind when AUTH_ENABLE_API_KEYS is truthy.
@@ -133,7 +152,7 @@ def try_autobind_apikey_model(*, require_env: bool = False) -> Optional[type]:
         from svc_infra.db.sql.base import ModelBase
 
         # SQLAlchemy 2.x: iterate registry mappers to get mapped classes
-        for mapper in list(getattr(ModelBase, "registry").mappers):
+        for mapper in list(ModelBase.registry.mappers):
             cls = mapper.class_
             if getattr(cls, "__svc_infra_auth_user__", False):
                 return bind_apikey_model(cls)  # binds and returns ApiKey

svc_infra/db/sql/authref.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-from typing import Optional, Tuple
-
 from sqlalchemy import ForeignKeyConstraint
 from sqlalchemy.sql.type_api import TypeEngine
 
@@ -9,7 +7,7 @@ from svc_infra.db.sql.base import ModelBase
 from svc_infra.db.sql.types import GUID
 
 
-def _find_auth_mapper() -> Optional[Tuple[str, TypeEngine, str]]:
+def _find_auth_mapper() -> tuple[str, TypeEngine, str] | None:
     """
     Returns (table_name, pk_sqlatype, pk_name) for the auth user model.
     Looks for any mapped class with __svc_infra_auth_user__ = True that
@@ -22,17 +20,21 @@ def _find_auth_mapper() -> Optional[Tuple[str, TypeEngine, str]]:
                 table = mapper.local_table or getattr(cls, "__table__", None)
                 if table is None:
                     continue
-                pk_cols = list(table.primary_key.columns)
+                table_name = getattr(table, "name", None)
+                if not isinstance(table_name, str) or not table_name:
+                    continue
+                # SQLAlchemy's primary_key is iterable; don't rely on .columns typing.
+                pk_cols = list(table.primary_key)
                 if len(pk_cols) != 1:
                     continue  # require single-column PK
                 pk_col = pk_cols[0]
-                return (table.name, pk_col.type, pk_col.name)
+                return (table_name, pk_col.type, pk_col.name)
     except Exception:
         pass
     return None
 
 
-def resolve_auth_table_pk() -> Tuple[str, TypeEngine, str]:
+def resolve_auth_table_pk() -> tuple[str, TypeEngine, str]:
     """
     Single source of truth for the auth table and PK.
     Falls back to ('users', GUID(), 'id') if nothing is marked.

svc_infra/db/sql/constants.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
 
 import re
-from typing import Sequence
+from collections.abc import Sequence
 
 # Environment variable names to look up for DB URL
+# Order matters: svc-infra canonical names first, then common PaaS names
 DEFAULT_DB_ENV_VARS: Sequence[str] = (
     "SQL_URL",
     "DB_URL",
+    "DATABASE_URL",  # Heroku, Railway (public)
+    "DATABASE_URL_PRIVATE",  # Railway (private networking)
+    "PRIVATE_SQL_URL",  # Legacy svc-infra naming
 )
 
 # Regex used to detect async drivers from URL drivername