matrx-orm 3.1.152__tar.gz → 3.1.154__tar.gz

{matrx_orm-3.1.152 → matrx_orm-3.1.154}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: matrx-orm
-Version: 3.1.152
+Version: 3.1.154
 Summary: Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current
@@ -25,7 +25,7 @@ Requires-Python: >=3.13
 Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: cryptography>=43.0
 Requires-Dist: gitpython>=3.1.40
-Requires-Dist: matrx-utils>=2.0.41
+Requires-Dist: matrx-utils>=2.0.42
 Requires-Dist: psycopg-pool>=3.2.5
 Requires-Dist: psycopg[binary]>=3.2.5
 Requires-Dist: python-dotenv>=1.0

{matrx_orm-3.1.152 → matrx_orm-3.1.154}/matrx_orm/__init__.py

@@ -60,6 +60,7 @@ from .core.diagnostics import (
     collect_pool_diagnostics,
     describe_transaction,
     describe_transactions,
+    introspect_lock_waits,
     log_pool_diagnostics,
     long_transactions,
     long_transactions_sql,
@@ -447,6 +448,7 @@ __all__ = [
     "OneDatabaseReport",
     "PoolIdentity",
     "introspect_columns",
+    "introspect_lock_waits",
     "introspect_rls_policies",
     "schema_exists",
     # ── LISTEN/NOTIFY ──────────────────────────────────────────────────────────

{matrx_orm-3.1.152 → matrx_orm-3.1.154}/matrx_orm/core/FEATURE.md

@@ -214,6 +214,15 @@ commit; exceptions roll back; cancellation-shielded teardown finishes before
 the connection returns to the pool. No path can leave `authenticated` on a
 reused server connection.
 
+**A DDL block needs `lock_timeout_ms`.** `rls_session(..., statement_timeout_ms=…)`
+bounds execution, not the WAIT for a lock: a `CREATE TABLE` queued behind a dump or a
+sweep is not executing, so `statement_timeout` never fires and an unbounded waiter
+blocks everyone behind it. `lock_timeout_ms` (default `None` = leave the server
+default) is `SET LOCAL` like the role and the claims, so it cannot outlive the
+transaction. First consumer: aidream's table-provisioning lane B
+(`aidream/services/provisioning/`), whose new tables carry FKs into two of the busiest
+relations on the platform database.
+
 **Empty-to-nonempty is not contamination proof.** A READ COMMITTED row can be
 created between the original empty query and its reproof (`guest_executions`
 does this under concurrent first requests). `reprove_empty_read` captures the
@@ -270,6 +279,26 @@ everywhere: tight ceiling for application sessions, explicit exemption for the
 backup session, and the exemption VERIFIED before the dump starts rather than
 assumed (`db/mirror/refresh.py::resolve_dump_endpoint`).
 
+**A ceiling that must survive a pooler is `SET LOCAL`, and the transaction is
+where it goes.** The split above left a gap: behind Supavisor neither the
+`server_settings` default nor `PGOPTIONS` reaches the server, so any caller that
+must bound what IT does — rather than what the deployment allows — had nowhere to
+put a ceiling. `set_transaction_ceilings({...})` /`transaction_ceilings()`
+(`matrx_orm.core.transaction`) close it: the named GUCs are emitted with
+`set_config(name, value, true)` immediately after every top-level `BEGIN`, before
+the host's session context and before `rls_session` drops to the restricted role,
+and they die with the transaction, so nothing can leak onto a pooled connection.
+A block that needs more says `SET LOCAL` later and wins.
+
+Empty in production — these are deployment settings there. The one caller is
+`matrx_orm.pytest_lock_guard`, the pytest11 guard that bounds what a test run may
+do to a SHARED database after a test transaction held ACCESS EXCLUSIVE on
+`platform.reachability` for 77s and stopped production chat on 2026-09-15
+(root-`conftest.py`, the sixth cross-suite hazard). Verified against the live
+database 2026-09-16: `lock_timeout`, `idle_in_transaction_session_timeout` and a
+tagged `application_name` all land through the pooler this way, and the same
+connection asked for them as startup options gets `0` and `"Supavisor"`.
+
 **`long_transactions()` names the blocker.** `matrx_orm.long_transactions`
 (async) and `matrx_orm.long_transactions_sql` (a parameter-free SQL string, so
 sync psycopg tooling runs the identical query) return every transaction open

{matrx_orm-3.1.152 → matrx_orm-3.1.154}/matrx_orm/core/diagnostics.py

@@ -217,3 +217,125 @@ def describe_transaction(row: dict[str, Any]) -> str:
 def describe_transactions(rows: list[dict[str, Any]]) -> str:
     """Multi-line blocker report, newest last. Empty string when there are none."""
     return "\n".join(f"  - {describe_transaction(r)}" for r in rows)
+
+
+# ---------------------------------------------------------------------------
+# LOCK WAITS — who is BLOCKED, on what, and who is holding it.
+#
+# Sibling to `long_transactions` above and deliberately NOT a duplicate of it:
+# that one answers "who has a transaction open too long" (one row per session,
+# no graph); this one answers "is a CROWD queued behind somebody right now, and
+# which somebody" — the blocking graph, the contended relations and the lock
+# modes. A long transaction that blocks nobody is a vacuum problem; a short one
+# holding ACCESS EXCLUSIVE on a hot table is an outage. They live in the same
+# module because this is the ONE place matrx-orm reads `pg_stat_activity`.
+#
+# 2026-09-15 23:45–23:52 UTC, production: `platform.rebuild_reachability()`
+# does `TRUNCATE platform.reachability` (ACCESS EXCLUSIVE on the hottest read
+# table) and fires from an AFTER trigger on `platform.association_types`. An
+# integration-test transaction inserted an association type and then kept
+# working for 77 seconds while holding that lock; 12–18 PostgREST sessions
+# queued on relation locks and every production chat turn returned
+# QueryTimeoutError. NOBODY WAS TOLD — the storm was invisible until a human
+# opened `pg_stat_activity` by hand.
+#
+# "Who is blocked, on what, and who is holding it" is a fact about POSTGRES,
+# not a row in an application table: `pg_locks` / `pg_stat_activity` /
+# `pg_blocking_pids()` have no Model and never will. That is exactly the shape
+# this module exists for (see the module docstring), so the statement lives
+# HERE — one fixed, parametrized statement in the ORM — rather than as a SQL
+# literal in the aidream service that consumes it. aidream keeps ZERO raw SQL
+# outside db/migrations; a missing capability is a todo for the ORM, never a
+# licence to write SQL in application code.
+#
+# clock_timestamp(), not now(): `now()` is frozen at transaction start, so a
+# backend whose `state_change` lands after our snapshot reports a NEGATIVE age.
+# A sampler must never report a hold as "-0.1 seconds old".
+_LOCK_WAITS_SQL = """
+WITH waiters AS (
+    SELECT l.pid              AS waiter_pid,
+           l.mode             AS requested_mode,
+           l.relation         AS relation_oid,
+           a.application_name AS waiter_application_name,
+           a.state            AS waiter_state,
+           a.query            AS waiter_query,
+           a.query_start      AS waiter_query_start,
+           pg_blocking_pids(l.pid) AS blocking_pids
+      FROM pg_locks l
+      JOIN pg_stat_activity a ON a.pid = l.pid
+     WHERE NOT l.granted
+       AND l.locktype = 'relation'
+       AND l.pid <> pg_backend_pid()
+       AND a.query_start IS NOT NULL
+       AND clock_timestamp() - a.query_start >= make_interval(secs => $1::float8)
+)
+SELECT w.waiter_pid,
+       w.waiter_application_name,
+       w.waiter_state,
+       left(coalesce(w.waiter_query, ''), $2::int)       AS waiter_query,
+       w.requested_mode,
+       CASE WHEN w.relation_oid IS NOT NULL
+            THEN rn.nspname || '.' || rc.relname END      AS relation,
+       extract(epoch FROM (clock_timestamp() - w.waiter_query_start))::float8
+                                                         AS waiter_waiting_seconds,
+       b.blocker_pid,
+       ba.application_name                               AS blocker_application_name,
+       ba.state                                          AS blocker_state,
+       left(coalesce(ba.query, ''), $2::int)             AS blocker_query,
+       extract(epoch FROM (clock_timestamp() - ba.xact_start))::float8
+                                                         AS blocker_xact_age_seconds,
+       extract(epoch FROM (clock_timestamp() - ba.state_change))::float8
+                                                         AS blocker_state_age_seconds,
+       bm.held_modes                                     AS blocker_held_modes
+  FROM waiters w
+  LEFT JOIN LATERAL unnest(w.blocking_pids) AS b(blocker_pid) ON true
+  LEFT JOIN pg_class     rc ON rc.oid = w.relation_oid
+  LEFT JOIN pg_namespace rn ON rn.oid = rc.relnamespace
+  LEFT JOIN pg_stat_activity ba ON ba.pid = b.blocker_pid
+  LEFT JOIN LATERAL (
+      SELECT string_agg(DISTINCT bl.mode, ',' ORDER BY bl.mode) AS held_modes
+        FROM pg_locks bl
+       WHERE bl.pid = b.blocker_pid
+         AND bl.granted
+         AND bl.relation IS NOT DISTINCT FROM w.relation_oid
+  ) bm ON true
+ ORDER BY waiter_waiting_seconds DESC
+"""
+
+#: How much of a query text one sampled row carries. A cap, not a filter — the
+#: alarm names the blocker, it does not archive its statement.
+LOCK_WAIT_QUERY_CHARS = 400
+
+
+async def introspect_lock_waits(
+    database: str,
+    *,
+    waiting_longer_than_seconds: float = 0.0,
+    query_chars: int = LOCK_WAIT_QUERY_CHARS,
+) -> list[dict[str, Any]]:
+    """Every session WAITING on a relation lock, with who is blocking it.
+
+    One row per (waiter, blocker) edge — a waiter blocked by three holders
+    yields three rows, and a waiter whose blocker has already vanished still
+    yields one row with ``blocker_pid`` ``None`` (the ``LEFT JOIN LATERAL`` is
+    deliberate: a waiter must never disappear from the sample because the
+    catalog raced us).
+
+    Columns: ``waiter_pid``, ``waiter_application_name``, ``waiter_state``,
+    ``waiter_query``, ``requested_mode``, ``relation`` (``schema.table``),
+    ``waiter_waiting_seconds``, ``blocker_pid``, ``blocker_application_name``,
+    ``blocker_state``, ``blocker_query``, ``blocker_xact_age_seconds``,
+    ``blocker_state_age_seconds``, ``blocker_held_modes``.
+
+    Read-only, parameter-bound, and takes no lock of its own — safe to call
+    DURING the storm it is describing, which is the only moment it is useful.
+    """
+    from matrx_orm.core.async_db_manager import AsyncDatabaseManager
+
+    rows = await AsyncDatabaseManager.execute_query(
+        database,
+        _LOCK_WAITS_SQL,
+        float(waiting_longer_than_seconds),
+        int(query_chars),
+    )
+    return [dict(r) for r in rows]

{matrx_orm-3.1.152 → matrx_orm-3.1.154}/matrx_orm/core/rls_session.py

@@ -59,11 +59,19 @@ async def rls_session(
     role: str,
     database: str | None = None,
     statement_timeout_ms: int | None = DEFAULT_STATEMENT_TIMEOUT_MS,
+    lock_timeout_ms: int | None = None,
 ) -> AsyncIterator[None]:
     """Run the enclosed block as ``role`` with ``request.jwt.claims`` set to ``claims``.
 
     All ORM/raw queries inside the block are subject to Postgres RLS as that role.
     ``claims`` must at minimum carry ``sub`` (the user id ``auth.uid()`` returns).
+
+    ``lock_timeout_ms`` bounds how long a statement inside the block waits for a lock
+    (``None`` leaves the server default alone). A block that issues DDL needs it: the
+    wait for an ``ACCESS EXCLUSIVE`` on a busy relation is not statement execution, so
+    ``statement_timeout`` does not bound it, and an unbounded waiter queues behind the
+    dump or sweep that is already running while blocking everyone behind it. Like the
+    role and the claims it is ``SET LOCAL`` — it cannot outlive the transaction.
     """
     if not _ROLE_RE.match(role):
         raise DatabaseError(
@@ -95,5 +103,7 @@ async def rls_session(
         await conn.execute("select set_config('request.jwt.claims', $1, true)", claims_json)
         if statement_timeout_ms is not None:
             await conn.execute(f"SET LOCAL statement_timeout = {int(statement_timeout_ms)}")
+        if lock_timeout_ms is not None:
+            await conn.execute(f"SET LOCAL lock_timeout = {int(lock_timeout_ms)}")
         await conn.execute(f"SET LOCAL ROLE {role}")
         yield

{matrx_orm-3.1.152 → matrx_orm-3.1.154}/matrx_orm/core/transaction.py

@@ -287,6 +287,7 @@ class TransactionContext:
             from matrx_orm.core.session_context import apply_session_context
 
             try:
+                await _apply_transaction_ceilings(conn)
                 await apply_session_context(conn)
             except BaseException as e:
                 await self.__aexit__(type(e), e, e.__traceback__)
@@ -372,6 +373,41 @@ class TransactionContext:
         return False  # never suppress exceptions
 
 
+# ── Process-wide transaction ceilings ───────────────────────────────────────────
+# Emitted as ``SET LOCAL`` immediately after every top-level ``BEGIN``.
+#
+# WHY THIS EXISTS AND WHY IT IS *HERE*, not in `server_settings`. We reach Postgres
+# through Supavisor in TRANSACTION mode. Measured 2026-09-16 against production:
+# Supavisor DISCARDS libpq startup options and asyncpg `server_settings` outright
+# (`show lock_timeout` → 0, `show application_name` → "Supavisor"), and a plain
+# `SET` does not survive the COMMIT because the next transaction may land on a
+# different server connection. The ONLY unit of session state that is real on this
+# path is the transaction — so a ceiling that is not `SET LOCAL` is a ceiling that
+# is not there. (That is also why the ORM's declared 60s
+# `idle_in_transaction_session_timeout` was inert during the 2026-09-15 outage.)
+#
+# ONE caller today: `matrx_orm.pytest_lock_guard`, bounding what a test run may do
+# to a shared database. Empty in production, where these are deployment settings.
+_TRANSACTION_CEILINGS: dict[str, str] = {}
+
+
+def set_transaction_ceilings(ceilings: "dict[str, str] | None") -> None:
+    """Replace the GUCs emitted as ``SET LOCAL`` after every top-level BEGIN."""
+    _TRANSACTION_CEILINGS.clear()
+    if ceilings:
+        _TRANSACTION_CEILINGS.update({str(k): str(v) for k, v in ceilings.items()})
+
+
+def transaction_ceilings() -> dict[str, str]:
+    """The ceilings currently in force (a copy)."""
+    return dict(_TRANSACTION_CEILINGS)
+
+
+async def _apply_transaction_ceilings(conn: Any) -> None:
+    for name, value in _TRANSACTION_CEILINGS.items():
+        await conn.execute("SELECT set_config($1, $2, true)", name, value)
+
+
 def transaction(database: str | None = None) -> TransactionContext:
     """Return a ``TransactionContext`` async context manager.