svc-infra 0.1.595__py3-none-any.whl → 0.1.706__py3-none-any.whl

svc_infra/db/ops.py

@@ -0,0 +1,384 @@
+"""Database operations utilities for one-off administrative tasks.
+
+This module provides synchronous database utilities for operations that
+don't fit the normal async SQLAlchemy workflow, such as:
+- Waiting for database readiness at startup
+- Executing maintenance SQL
+- Dropping tables with lock handling
+- Terminating blocking queries
+
+These utilities use psycopg2 directly for maximum reliability in
+edge cases where the ORM might not be available or appropriate.
+
+Example:
+    >>> from svc_infra.db.ops import wait_for_database, run_sync_sql
+    >>>
+    >>> # Wait for database before app starts
+    >>> wait_for_database(timeout=30)
+    >>>
+    >>> # Run maintenance query
+    >>> run_sync_sql("VACUUM ANALYZE my_table")
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+import time
+from typing import Any, Optional, Sequence, cast
+
+from .sql.utils import get_database_url_from_env
+
+logger = logging.getLogger(__name__)
+
+# Timeout for individual database operations (seconds)
+DEFAULT_STATEMENT_TIMEOUT = 30
+
+# Default wait-for-database settings
+DEFAULT_WAIT_TIMEOUT = 30
+DEFAULT_WAIT_INTERVAL = 1.0
+
+
+def _flush() -> None:
+    """Force flush stdout/stderr for containerized log visibility."""
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+
+def _get_connection(url: Optional[str] = None, connect_timeout: int = 10) -> Any:
+    """
+    Get a psycopg2 connection.
+
+    Args:
+        url: Database URL. If None, resolved from environment.
+        connect_timeout: Connection timeout in seconds.
+
+    Returns:
+        psycopg2 connection object
+
+    Raises:
+        ImportError: If psycopg2 is not installed
+        RuntimeError: If no database URL is available
+    """
+    try:
+        import psycopg2
+    except ImportError as e:
+        raise ImportError(
+            "psycopg2 is required for db.ops utilities. "
+            "Install with: pip install psycopg2-binary"
+        ) from e
+
+    if url is None:
+        url = get_database_url_from_env(required=True)
+
+    # Add connect_timeout to connection options
+    return psycopg2.connect(url, connect_timeout=connect_timeout)
+
+
+def wait_for_database(
+    url: Optional[str] = None,
+    timeout: float = DEFAULT_WAIT_TIMEOUT,
+    interval: float = DEFAULT_WAIT_INTERVAL,
+    verbose: bool = True,
+) -> bool:
+    """
+    Wait for database to be ready, with retries.
+
+    Useful for container startup scripts where the database may not
+    be immediately available.
+
+    Args:
+        url: Database URL. If None, resolved from environment.
+        timeout: Maximum time to wait in seconds (default: 30)
+        interval: Time between retry attempts in seconds (default: 1.0)
+        verbose: If True, log progress messages
+
+    Returns:
+        True if database is ready, False if timeout reached
+
+    Example:
+        >>> # In container startup script
+        >>> if not wait_for_database(timeout=60):
+        ...     sys.exit(1)
+        >>> # Database is ready, continue with app startup
+    """
+    if url is None:
+        url = get_database_url_from_env(required=True)
+
+    start = time.monotonic()
+    attempt = 0
+
+    while True:
+        attempt += 1
+        elapsed = time.monotonic() - start
+
+        if elapsed >= timeout:
+            if verbose:
+                logger.error(
+                    f"Database not ready after {timeout}s ({attempt} attempts)"
+                )
+                _flush()
+            return False
+
+        try:
+            conn = _get_connection(url, connect_timeout=min(5, int(timeout - elapsed)))
+            conn.close()
+            if verbose:
+                logger.info(f"Database ready after {elapsed:.1f}s ({attempt} attempts)")
+                _flush()
+            return True
+        except Exception as e:
+            if verbose:
+                remaining = timeout - elapsed
+                logger.debug(
+                    f"Database not ready ({e}), retrying... ({remaining:.0f}s remaining)"
+                )
+                _flush()
+            time.sleep(interval)
+
+
+def run_sync_sql(
+    sql: str,
+    params: Optional[Sequence[Any]] = None,
+    url: Optional[str] = None,
+    timeout: int = DEFAULT_STATEMENT_TIMEOUT,
+    fetch: bool = False,
+) -> Optional[list[tuple[Any, ...]]]:
+    """
+    Execute SQL synchronously with a statement timeout.
+
+    This is useful for one-off administrative queries that don't fit
+    the normal async SQLAlchemy workflow.
+
+    Args:
+        sql: SQL statement to execute
+        params: Optional parameters for parameterized queries
+        url: Database URL. If None, resolved from environment.
+        timeout: Statement timeout in seconds (default: 30)
+        fetch: If True, return fetched rows; if False, return None
+
+    Returns:
+        List of tuples if fetch=True, otherwise None
+
+    Raises:
+        psycopg2.Error: On database errors
+        TimeoutError: If statement exceeds timeout
+
+    Example:
+        >>> # Run a maintenance query
+        >>> run_sync_sql("VACUUM ANALYZE users")
+        >>>
+        >>> # Fetch data with timeout
+        >>> rows = run_sync_sql(
+        ...     "SELECT id, name FROM users WHERE active = %s",
+        ...     params=(True,),
+        ...     fetch=True,
+        ...     timeout=10
+        ... )
+    """
+    conn = _get_connection(url)
+    try:
+        with conn.cursor() as cur:
+            # Set statement timeout (PostgreSQL-specific)
+            cur.execute(f"SET statement_timeout = '{timeout}s'")
+
+            if params:
+                cur.execute(sql, params)
+            else:
+                cur.execute(sql)
+
+            if fetch:
+                return cast(list[tuple[Any, ...]], cur.fetchall())
+
+            conn.commit()
+            return None
+    finally:
+        conn.close()
+
+
+def kill_blocking_queries(
+    table_name: str,
+    url: Optional[str] = None,
+    timeout: int = DEFAULT_STATEMENT_TIMEOUT,
+    dry_run: bool = False,
+) -> list[dict[str, Any]]:
+    """
+    Terminate queries blocking operations on a specific table.
+
+    This is useful before DROP TABLE or ALTER TABLE operations that
+    might be blocked by long-running queries or idle transactions.
+
+    Args:
+        table_name: Name of the table (can include schema as 'schema.table')
+        url: Database URL. If None, resolved from environment.
+        timeout: Statement timeout in seconds (default: 30)
+        dry_run: If True, only report blocking queries without terminating
+
+    Returns:
+        List of dicts with info about terminated (or found) queries:
+        [{"pid": 123, "query": "SELECT...", "state": "active", "terminated": True}]
+
+    Example:
+        >>> # Check what would be terminated
+        >>> blocking = kill_blocking_queries("embeddings", dry_run=True)
+        >>> print(f"Found {len(blocking)} blocking queries")
+        >>>
+        >>> # Actually terminate them
+        >>> kill_blocking_queries("embeddings")
+    """
+    # Query to find blocking queries on a table
+    find_blocking_sql = """
+        SELECT pid, state, query, age(clock_timestamp(), query_start) as duration
+        FROM pg_stat_activity
+        WHERE pid != pg_backend_pid()
+          AND state != 'idle'
+          AND (
+              query ILIKE %s
+              OR query ILIKE %s
+              OR query ILIKE %s
+          )
+        ORDER BY query_start;
+    """
+
+    # Patterns to match queries involving the table
+    patterns = (
+        f"%{table_name}%",
+        f"%{table_name.split('.')[-1]}%",  # Just table name without schema
+        f"%{table_name.replace('.', '%')}%",  # Handle schema.table pattern
+    )
+
+    conn = _get_connection(url)
+    terminated: list[dict[str, Any]] = []
+
+    try:
+        with conn.cursor() as cur:
+            cur.execute(f"SET statement_timeout = '{timeout}s'")
+            cur.execute(find_blocking_sql, patterns)
+            rows = cur.fetchall()
+
+            for pid, state, query, duration in rows:
+                info = {
+                    "pid": pid,
+                    "state": state,
+                    "query": query[:200] + "..." if len(query) > 200 else query,
+                    "duration": str(duration),
+                    "terminated": False,
+                }
+
+                if not dry_run:
+                    try:
+                        cur.execute("SELECT pg_terminate_backend(%s)", (pid,))
+                        info["terminated"] = True
+                        logger.info(f"Terminated query PID {pid}: {query[:100]}...")
+                    except Exception as e:
+                        logger.warning(f"Failed to terminate PID {pid}: {e}")
+                        info["error"] = str(e)
+
+                terminated.append(info)
+
+            conn.commit()
+    finally:
+        conn.close()
+
+    _flush()
+    return terminated
+
+
+def drop_table_safe(
+    table_name: str,
+    url: Optional[str] = None,
+    timeout: int = DEFAULT_STATEMENT_TIMEOUT,
+    kill_blocking: bool = True,
+    if_exists: bool = True,
+    cascade: bool = False,
+) -> bool:
+    """
+    Drop a table safely with lock handling.
+
+    Handles common issues with DROP TABLE:
+    - Terminates blocking queries first (optional)
+    - Uses statement timeout to avoid hanging
+    - Handles 'table does not exist' gracefully
+
+    Args:
+        table_name: Name of table to drop (can include schema)
+        url: Database URL. If None, resolved from environment.
+        timeout: Statement timeout in seconds (default: 30)
+        kill_blocking: If True, terminate blocking queries first (default: True)
+        if_exists: If True, don't error if table doesn't exist (default: True)
+        cascade: If True, drop dependent objects (default: False)
+
+    Returns:
+        True if table was dropped (or didn't exist), False on error
+
+    Example:
+        >>> # Drop table, killing any blocking queries first
+        >>> drop_table_safe("embeddings", cascade=True)
+        True
+        >>>
+        >>> # Safe to call even if table doesn't exist
+        >>> drop_table_safe("nonexistent_table")
+        True
+    """
+    if url is None:
+        url = get_database_url_from_env(required=True)
+
+    # Kill blocking queries first if requested
+    if kill_blocking:
+        blocked = kill_blocking_queries(table_name, url=url, timeout=timeout)
+        if blocked:
+            logger.info(f"Terminated {len(blocked)} blocking queries before DROP")
+            # Brief pause to let connections clean up
+            time.sleep(0.5)
+
+    # Build DROP statement
+    drop_sql = "DROP TABLE"
+    if if_exists:
+        drop_sql += " IF EXISTS"
+    drop_sql += f" {table_name}"
+    if cascade:
+        drop_sql += " CASCADE"
+
+    try:
+        run_sync_sql(drop_sql, url=url, timeout=timeout)
+        logger.info(f"Dropped table: {table_name}")
+        _flush()
+        return True
+    except Exception as e:
+        logger.error(f"Failed to drop table {table_name}: {e}")
+        _flush()
+        return False
+
+
+def get_database_url(
+    required: bool = True,
+    normalize: bool = True,
+) -> Optional[str]:
+    """
+    Convenience wrapper for get_database_url_from_env().
+
+    This is the recommended way to get the database URL, as it
+    handles all common environment variable names and normalizations.
+
+    Args:
+        required: If True, raise RuntimeError when no URL is found
+        normalize: If True, convert postgres:// to postgresql://
+
+    Returns:
+        Database URL string, or None if not found and not required
+
+    Example:
+        >>> url = get_database_url()
+        >>> print(url)
+        'postgresql://user:pass@host:5432/db'
+    """
+    return get_database_url_from_env(required=required, normalize=normalize)
+
+
+__all__ = [
+    "wait_for_database",
+    "run_sync_sql",
+    "kill_blocking_queries",
+    "drop_table_safe",
+    "get_database_url",
+]

svc_infra/db/outbox.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, Iterable, List, Optional, Protocol
+
+
+@dataclass
+class OutboxMessage:
+    id: int
+    topic: str
+    payload: Dict[str, Any]
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    attempts: int = 0
+    processed_at: Optional[datetime] = None
+
+
+class OutboxStore(Protocol):
+    def enqueue(self, topic: str, payload: Dict[str, Any]) -> OutboxMessage:
+        pass
+
+    def fetch_next(
+        self, *, topics: Optional[Iterable[str]] = None
+    ) -> Optional[OutboxMessage]:
+        """Return the next undispatched, unprocessed message (FIFO per-topic), or None.
+
+        Notes:
+        - Messages with attempts > 0 are considered "dispatched" to the job queue and won't be re-enqueued.
+        - Delivery retries are handled by the job queue worker, not by re-reading the outbox.
+        """
+        pass
+
+    def mark_processed(self, msg_id: int) -> None:
+        pass
+
+    def mark_failed(self, msg_id: int) -> None:
+        pass
+
+
+class InMemoryOutboxStore:
+    """Simple in-memory outbox for tests and local runs."""
+
+    def __init__(self):
+        self._seq = 0
+        self._messages: List[OutboxMessage] = []
+
+    def enqueue(self, topic: str, payload: Dict[str, Any]) -> OutboxMessage:
+        self._seq += 1
+        msg = OutboxMessage(id=self._seq, topic=topic, payload=dict(payload))
+        self._messages.append(msg)
+        return msg
+
+    def fetch_next(
+        self, *, topics: Optional[Iterable[str]] = None
+    ) -> Optional[OutboxMessage]:
+        allowed = set(topics) if topics else None
+        for msg in self._messages:
+            if msg.processed_at is not None:
+                continue
+            # skip already dispatched messages (attempts>0)
+            if msg.attempts > 0:
+                continue
+            if allowed is not None and msg.topic not in allowed:
+                continue
+            return msg
+        return None
+
+    def mark_processed(self, msg_id: int) -> None:
+        for msg in self._messages:
+            if msg.id == msg_id:
+                msg.processed_at = datetime.now(timezone.utc)
+                return
+
+    def mark_failed(self, msg_id: int) -> None:
+        for msg in self._messages:
+            if msg.id == msg_id:
+                msg.attempts += 1
+                return
+
+
+class SqlOutboxStore:
+    """Skeleton for a SQL-backed outbox store.
+
+    Implementations should:
+    - INSERT on enqueue
+    - SELECT FOR UPDATE SKIP LOCKED (or equivalent) to fetch next
+    - UPDATE processed_at (and attempts on failure)
+    """
+
+    def __init__(self, session_factory):
+        self._session_factory = session_factory
+
+    # Placeholders to outline the API; not implemented here.
+    def enqueue(
+        self, topic: str, payload: Dict[str, Any]
+    ) -> OutboxMessage:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def fetch_next(
+        self, *, topics: Optional[Iterable[str]] = None
+    ) -> Optional[OutboxMessage]:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def mark_processed(self, msg_id: int) -> None:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def mark_failed(self, msg_id: int) -> None:  # pragma: no cover - skeleton
+        raise NotImplementedError

svc_infra/db/sql/apikey.py

@@ -7,18 +7,36 @@ import uuid
 from datetime import datetime, timezone
 from typing import Optional, Type
 
-from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Index, String, UniqueConstraint, text
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    DateTime,
+    ForeignKey,
+    Index,
+    String,
+    UniqueConstraint,
+    text,
+)
 from sqlalchemy.ext.mutable import MutableDict, MutableList
 from sqlalchemy.orm import Mapped, declared_attr, mapped_column, relationship
 
+from svc_infra.app.env import require_secret
 from svc_infra.db.sql.base import ModelBase
 from svc_infra.db.sql.types import GUID
 
-_APIKEY_HMAC_SECRET = os.getenv("APIKEY_HASH_SECRET") or "change-me-low-entropy-dev"
+
+def _get_apikey_secret() -> str:
+    """Get APIKEY_HASH_SECRET, requiring it in production."""
+    return require_secret(
+        os.getenv("APIKEY_HASH_SECRET"),
+        "APIKEY_HASH_SECRET",
+        dev_default="dev-only-apikey-hmac-secret-not-for-production",
+    )
 
 
 def _hmac_sha256(s: str) -> str:
-    return hmac.new(_APIKEY_HMAC_SECRET.encode(), s.encode(), hashlib.sha256).hexdigest()
+    secret = _get_apikey_secret()
+    return hmac.new(secret.encode(), s.encode(), hashlib.sha256).hexdigest()
 
 
 def _now() -> datetime:
@@ -33,7 +51,9 @@ _ApiKeyModel: Optional[type] = None
 def get_apikey_model() -> type:
     """Return the bound ApiKey model (or raise if not enabled)."""
     if _ApiKeyModel is None:
-        raise RuntimeError("ApiKey model is not enabled. Call bind_apikey_model(...) first.")
+        raise RuntimeError(
+            "ApiKey model is not enabled. Call bind_apikey_model(...) first."
+        )
     return _ApiKeyModel
 
 
@@ -43,10 +63,12 @@ def bind_apikey_model(user_model: Type, *, table_name: str = "api_keys") -> type
     Call this once during app boot (e.g., inside add_auth_users when enable_api_keys=True).
     """
 
-    class ApiKey(ModelBase):  # type: ignore[misc, valid-type]
+    class ApiKey(ModelBase):
         __tablename__ = table_name
 
-        id: Mapped[uuid.UUID] = mapped_column(GUID(), primary_key=True, default=uuid.uuid4)
+        id: Mapped[uuid.UUID] = mapped_column(
+            GUID(), primary_key=True, default=uuid.uuid4
+        )
 
         @declared_attr
         def user_id(cls) -> Mapped[uuid.UUID | None]:  # noqa: N805
@@ -67,14 +89,18 @@ def bind_apikey_model(user_model: Type, *, table_name: str = "api_keys") -> type
         key_prefix: Mapped[str] = mapped_column(String(12), index=True, nullable=False)
         key_hash: Mapped[str] = mapped_column(String(64), nullable=False)  # hex sha256
 
-        scopes: Mapped[list[str]] = mapped_column(MutableList.as_mutable(JSON), default=list)
+        scopes: Mapped[list[str]] = mapped_column(
+            MutableList.as_mutable(JSON), default=list
+        )
         active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
         expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
         last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
         meta: Mapped[dict] = mapped_column(MutableDict.as_mutable(JSON), default=dict)
 
         created_at = mapped_column(
-            DateTime(timezone=True), server_default=text("CURRENT_TIMESTAMP"), nullable=False
+            DateTime(timezone=True),
+            server_default=text("CURRENT_TIMESTAMP"),
+            nullable=False,
         )
         updated_at = mapped_column(
             DateTime(timezone=True),
@@ -99,7 +125,9 @@ def bind_apikey_model(user_model: Type, *, table_name: str = "api_keys") -> type
             import secrets
 
             prefix = secrets.token_urlsafe(6).replace("-", "").replace("_", "")[:8]
-            rand = base64.urlsafe_b64encode(secrets.token_bytes(24)).decode().rstrip("=")
+            rand = (
+                base64.urlsafe_b64encode(secrets.token_bytes(24)).decode().rstrip("=")
+            )
             plaintext = f"ak_{prefix}_{rand}"
             return plaintext, prefix, _hmac_sha256(plaintext)
 

svc_infra/db/sql/authref.py

@@ -22,11 +22,15 @@ def _find_auth_mapper() -> Optional[Tuple[str, TypeEngine, str]]:
                 table = mapper.local_table or getattr(cls, "__table__", None)
                 if table is None:
                     continue
-                pk_cols = list(table.primary_key.columns)
+                table_name = getattr(table, "name", None)
+                if not isinstance(table_name, str) or not table_name:
+                    continue
+                # SQLAlchemy's primary_key is iterable; don't rely on .columns typing.
+                pk_cols = list(table.primary_key)
                 if len(pk_cols) != 1:
                     continue  # require single-column PK
                 pk_col = pk_cols[0]
-                return (table.name, pk_col.type, pk_col.name)
+                return (table_name, pk_col.type, pk_col.name)
     except Exception:
         pass
     return None
@@ -58,4 +62,6 @@ def user_fk_constraint(
     Returns a table-level ForeignKeyConstraint([...], [<auth_table>.<pk>]) for the given column.
     """
     table, _pk_type, pk_name = resolve_auth_table_pk()
-    return ForeignKeyConstraint([column_name], [f"{table}.{pk_name}"], ondelete=ondelete)
+    return ForeignKeyConstraint(
+        [column_name], [f"{table}.{pk_name}"], ondelete=ondelete
+    )

svc_infra/db/sql/constants.py

@@ -4,9 +4,13 @@ import re
 from typing import Sequence
 
 # Environment variable names to look up for DB URL
+# Order matters: svc-infra canonical names first, then common PaaS names
 DEFAULT_DB_ENV_VARS: Sequence[str] = (
     "SQL_URL",
     "DB_URL",
+    "DATABASE_URL",  # Heroku, Railway (public)
+    "DATABASE_URL_PRIVATE",  # Railway (private networking)
+    "PRIVATE_SQL_URL",  # Legacy svc-infra naming
 )
 
 # Regex used to detect async drivers from URL drivername
@@ -18,16 +22,16 @@ try:
     import importlib.resources as pkg
 
     _tmpl_pkg = pkg.files("svc_infra.db.sql.templates.setup")
-    ALEMBIC_INI_TEMPLATE = _tmpl_pkg.joinpath("alembic.ini.tmpl").read_text(encoding="utf-8")
-    ALEMBIC_SCRIPT_TEMPLATE = _tmpl_pkg.joinpath("script.py.mako.tmpl").read_text(encoding="utf-8")
-except Exception:
-    # Fallbacks (should not normally happen). Provide minimal safe defaults.
-    ALEMBIC_INI_TEMPLATE = (
-        """[alembic]\nscript_location = {script_location}\nsqlalchemy.url = {sqlalchemy_url}\n"""
+    ALEMBIC_INI_TEMPLATE = _tmpl_pkg.joinpath("alembic.ini.tmpl").read_text(
+        encoding="utf-8"
     )
-    ALEMBIC_INI_TEMPLATE = (
-        """[alembic]\nscript_location = {script_location}\nsqlalchemy.url = {sqlalchemy_url}\n"""
+    ALEMBIC_SCRIPT_TEMPLATE = _tmpl_pkg.joinpath("script.py.mako.tmpl").read_text(
+        encoding="utf-8"
     )
+except Exception:
+    # Fallbacks (should not normally happen). Provide minimal safe defaults.
+    ALEMBIC_INI_TEMPLATE = """[alembic]\nscript_location = {script_location}\nsqlalchemy.url = {sqlalchemy_url}\n"""
+    ALEMBIC_INI_TEMPLATE = """[alembic]\nscript_location = {script_location}\nsqlalchemy.url = {sqlalchemy_url}\n"""
     ALEMBIC_SCRIPT_TEMPLATE = '"""${message}"""\nfrom alembic import op\nimport sqlalchemy as sa\n\nrevision = ${repr(up_revision)}\ndown_revision = ${repr(down_revision)}\nbranch_labels = ${repr(branch_labels)}\ndepends_on = ${repr(depends_on)}\n\ndef upgrade():\n    ${upgrades if upgrades else "pass"}\n\n\ndef downgrade():\n    ${downgrades if downgrades else "pass"}\n'
 __all__ = [
     "DEFAULT_DB_ENV_VARS",

svc_infra/db/sql/core.py

@@ -140,7 +140,7 @@ def revision(
         cfg,
         message=message,
         autogenerate=autogenerate,
-        head=head,
+        head=head or "head",
         branch_label=branch_label,
         version_path=version_path,
         sql=sql,
@@ -319,7 +319,7 @@ def setup_and_migrate(
     """
     resolved_url = database_url or get_database_url_from_env(required=True)
     root = prepare_env()
-    if create_db_if_missing:
+    if create_db_if_missing and resolved_url:
         ensure_database_exists(resolved_url)
 
     mig_dir = init_alembic(