matrx-orm 3.1.96__tar.gz → 3.1.97__tar.gz

{matrx_orm-3.1.96 → matrx_orm-3.1.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: matrx-orm
-Version: 3.1.96
+Version: 3.1.97
 Summary: Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current
@@ -25,7 +25,7 @@ Requires-Python: >=3.13
 Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: cryptography>=43.0
 Requires-Dist: gitpython>=3.1.40
-Requires-Dist: matrx-utils>=2.0.20
+Requires-Dist: matrx-utils>=2.0.21
 Requires-Dist: psycopg-pool>=3.2.5
 Requires-Dist: psycopg[binary]>=3.2.5
 Requires-Dist: python-dotenv>=1.0

{matrx_orm-3.1.96 → matrx_orm-3.1.97}/matrx_orm/__init__.py

@@ -44,7 +44,15 @@ from .platform_db import (
     register_platform_db,
 )
 from .local_sqlite import SQLiteValue, check_local_sqlite_integrity, read_local_sqlite_rows
-from .core.diagnostics import collect_pool_diagnostics, log_pool_diagnostics
+from .core.diagnostics import (
+    LONG_TRANSACTION_SECONDS,
+    collect_pool_diagnostics,
+    describe_transaction,
+    describe_transactions,
+    log_pool_diagnostics,
+    long_transactions,
+    long_transactions_sql,
+)
 from .core.introspection import introspect_columns, introspect_rls_policies, schema_exists
 from .core.one_database import OneDatabaseReport, PoolIdentity, audit_one_database
 from .core.listen import notify, ping
@@ -380,6 +388,11 @@ __all__ = [
     # ── Diagnostics ────────────────────────────────────────────────────────────
     "collect_pool_diagnostics",
     "log_pool_diagnostics",
+    "LONG_TRANSACTION_SECONDS",
+    "long_transactions",
+    "long_transactions_sql",
+    "describe_transaction",
+    "describe_transactions",
     "install_asyncpg_idle_close_filter",
     # ── DB-side function calls (RPC) ──────────────────────────────────────────
     "call_function",

{matrx_orm-3.1.96 → matrx_orm-3.1.97}/matrx_orm/catalog.py

@@ -452,8 +452,19 @@ async def org_backstop_coverage(database: str) -> list[dict[str, Any]]:
           EXISTS (SELECT 1 FROM pg_trigger t
             JOIN pg_class cl ON cl.oid=t.tgrelid
             JOIN pg_namespace n ON n.oid=cl.relnamespace
+            JOIN pg_proc bp ON bp.oid=t.tgfoid
             WHERE n.nspname=c.table_schema AND cl.relname=c.table_name
-              AND t.tgname='_stamp_org_default' AND NOT t.tgisinternal) AS has_backstop
+              AND NOT t.tgisinternal
+              -- Match the FUNCTION, not the trigger name: db-rules FEATURE.md
+              -- §2 names THREE legal org backstops, and only the first is
+              -- conventionally called `_stamp_org_default`. A child entity's
+              -- backstop is `platform.inherit_org_from_parent` (trigger
+              -- `_inherit_org`), and `plan` uses `_stamp_from_node`. Keying on
+              -- the trigger name reported correctly-backstopped children as
+              -- DRIFT (D232 §C, 2026-08-21).
+              AND bp.proname IN ('_stamp_org_default',
+                                 'inherit_org_from_parent',
+                                 '_stamp_from_node')) AS has_backstop
         FROM pg_catalog_columns c
         JOIN pg_catalog_tables tb
           ON tb.table_schema=c.table_schema AND tb.table_name=c.table_name

{matrx_orm-3.1.96 → matrx_orm-3.1.97}/matrx_orm/core/async_db_manager.py

@@ -113,6 +113,30 @@ POOL_RELEASE_TIMEOUT_SECONDS = 10.0
 READ_TIMEOUT_RETRIES = 1  # extra attempts for a timed-out read; total = 1 + this
 READ_TIMEOUT_RETRY_BACKOFF = 0.25  # seconds before the read retry
 
+# ── idle-in-transaction ceiling (the platform-wide DDL blocker) ───────────────
+# A session that BEGINs and then goes quiet is the single most expensive thing a
+# client can do to a shared Postgres: it holds every lock it took (so DDL —
+# CREATE POLICY, ALTER TABLE — queues behind it platform-wide) AND it pins the
+# cluster xmin horizon, which stops autovacuum from reclaiming dead tuples
+# ANYWHERE in the database, not just on the tables it touched.
+#
+# The incident (2026-08-21, feedback c72a32ac): one session sat
+# `idle in transaction` for 18+ minutes mid-bulk-read of the ops log tables.
+# Every `CREATE POLICY` against the live DB blocked to lock_timeout while plain
+# DDL on the same brand-new table succeeded in <0.2s, and each agent applying a
+# migration was told by the applier to "fix the SQL" — which was fine.
+#
+# Postgres will end such a session for us if we ask. This is the ceiling for
+# every pooled ORM connection: nothing the ORM does legitimately sits idle
+# mid-transaction for a minute. Note what this does NOT kill — a long-RUNNING
+# statement is `active`, and this timeout only counts IDLE time inside an open
+# transaction, so a 10-minute bulk INSERT is untouched. A caller with a genuine
+# need overrides it per-database via `server_settings`.
+#
+# A threshold → a CAPS constant, never an env var (an invisible prod toggle is
+# the failure class this repo bans).
+IDLE_IN_TRANSACTION_TIMEOUT_MS = 60_000  # 60s
+
 # One idle holder is probed per tick. The interval adapts so every currently
 # connected holder is touched within this sweep window, below the default 60s
 # transaction-pool idle lifetime. A degraded pool still attempts at most one
@@ -497,6 +521,19 @@ class AsyncDatabaseManager:
                         # Disable globally — it's the standard recommendation
                         # for OLTP-style workloads.
                         "jit": "off",
+                        # A session that opens a transaction and then stops
+                        # talking holds its locks and pins the cluster-wide
+                        # xmin horizon until it comes back — blocking DDL for
+                        # everyone and stopping autovacuum EVERYWHERE, not just
+                        # on the tables it touched. Postgres kills such a
+                        # session for us if we ask it to, and nothing this ORM
+                        # does legitimately sits idle mid-transaction for
+                        # minutes (a long-RUNNING statement is `active`, which
+                        # this timeout does not count). See IDLE_IN_TRANSACTION
+                        # _TIMEOUT_MS for the incident that earned it.
+                        "idle_in_transaction_session_timeout": str(
+                            IDLE_IN_TRANSACTION_TIMEOUT_MS
+                        ),
                     }
                     server_settings.update(dict(config.get("server_settings") or {}))
 
@@ -528,7 +565,7 @@ class AsyncDatabaseManager:
                         logger.info(
                             "[matrx_orm] Pool created for '%s' "
                             "(mode=%s, port=%s, min=%d, max=%d, idle_lifetime=%.1fs, "
-                            "jit=%s)",
+                            "jit=%s, idle_in_transaction_timeout=%sms)",
                             config_name,
                             effective_mode,
                             config.get("port"),
@@ -536,6 +573,9 @@ class AsyncDatabaseManager:
                             config.get("pool_max", 30),
                             max_inactive_connection_lifetime,
                             server_settings.get("jit", "default"),
+                            server_settings.get(
+                                "idle_in_transaction_session_timeout", "default"
+                            ),
                         )
 
                         if config.get("write_queue_enabled", True):

matrx_orm-3.1.97/matrx_orm/core/diagnostics.py

@@ -0,0 +1,219 @@
+"""Pool diagnostics — surface the pool config and live server-side limits.
+
+Call ``log_pool_diagnostics(config_name)`` once at app startup (after the
+first query) to log:
+
+- The resolved pool mode (transaction vs session) and effective port.
+- The pool ceiling (min/max) the app is configured for.
+- The Postgres server's ``max_connections`` setting.
+- The current per-pool connection counts (``pool.get_size()``,
+  ``pool.get_idle_size()``).
+
+Surfacing these at startup makes Supavisor saturation issues debuggable
+*before* they happen — a misconfigured ``pool_max`` is visible in the
+boot log instead of as a 502 in production.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from matrx_orm.core.config import get_database_config, resolve_pool_mode
+
+logger = logging.getLogger("matrx_orm.diagnostics")
+
+
+async def collect_pool_diagnostics(config_name: str) -> dict[str, Any]:
+    """Return a dict of diagnostic info for ``config_name``.
+
+    Creates the pool if it doesn't exist yet (so the diagnostics also serve
+    as a cheap "warm up" probe). Safe to call from any async context.
+    """
+    from matrx_orm.core.async_db_manager import AsyncDatabaseManager
+
+    config = get_database_config(config_name)
+    pool = await AsyncDatabaseManager.get_pool(config_name)
+
+    server_max_connections: str | None = None
+    server_version: str | None = None
+    inet_port: str | None = None
+    application_name: str | None = None
+
+    try:
+        async with AsyncDatabaseManager.get_connection(config_name, timeout=5.0) as conn:
+            server_max_connections = await conn.fetchval(
+                "SELECT current_setting('max_connections')"
+            )
+            server_version = await conn.fetchval("SELECT current_setting('server_version')")
+            inet_port = await conn.fetchval("SELECT current_setting('port', true)")
+            application_name = await conn.fetchval(
+                "SELECT current_setting('application_name', true)"
+            )
+    except Exception as exc:
+        logger.warning(
+            "[matrx_orm] Diagnostics probe failed for '%s': %s",
+            config_name,
+            exc,
+        )
+
+    effective_mode = resolve_pool_mode(config)
+    configured_idle_lifetime = config.get("max_inactive_connection_lifetime")
+    if configured_idle_lifetime is None:
+        idle_lifetime_effective = 60.0 if effective_mode == "transaction" else 300.0
+        idle_lifetime_source = "default"
+    else:
+        idle_lifetime_effective = float(configured_idle_lifetime)
+        idle_lifetime_source = "configured"
+
+    return {
+        "config_name": config_name,
+        "host": config.get("host"),
+        "port": str(config.get("port")),
+        "database": config.get("database_name"),
+        "user": config.get("user"),
+        "pool_mode_configured": config.get("pool_mode"),
+        "pool_mode_effective": effective_mode,
+        "pool_min": config.get("pool_min"),
+        "pool_max": config.get("pool_max"),
+        "max_inactive_connection_lifetime": idle_lifetime_effective,
+        "max_inactive_connection_lifetime_source": idle_lifetime_source,
+        "current_pool_size": pool.get_size(),
+        "current_pool_idle": pool.get_idle_size(),
+        "server_version": server_version,
+        "server_max_connections": server_max_connections,
+        "server_inet_port": inet_port,
+        "application_name": application_name,
+    }
+
+
+async def log_pool_diagnostics(config_name: str) -> None:
+    """Emit a single human-readable INFO log line summarizing the pool state."""
+    info = await collect_pool_diagnostics(config_name)
+    # INFO, not WARNING — a healthy pool is not a warning. Surfaces only when
+    # the app raises its log level; saturation problems fail loudly elsewhere.
+    logger.info(
+        "[matrx_orm] Pool diagnostics '%s' — host=%s:%s db=%s user=%s | "
+        "mode=%s (configured=%s) min=%s max=%s idle_lifetime=%.0fs(%s) "
+        "now=%s/%s | server pg=%s max_connections=%s app=%s",
+        info["config_name"],
+        info["host"],
+        info["port"],
+        info["database"],
+        info["user"],
+        info["pool_mode_effective"],
+        info["pool_mode_configured"],
+        info["pool_min"],
+        info["pool_max"],
+        info["max_inactive_connection_lifetime"],
+        info["max_inactive_connection_lifetime_source"],
+        info["current_pool_idle"],
+        info["current_pool_size"],
+        info["server_version"],
+        info["server_max_connections"],
+        info["application_name"],
+    )
+
+
+# ---------------------------------------------------------------------------
+# Long-lived transactions — the platform-wide DDL blocker, made nameable.
+#
+# A session that holds a transaction open holds its locks (DDL platform-wide
+# queues behind it) and pins the cluster xmin horizon (autovacuum stops
+# reclaiming dead tuples EVERYWHERE, not just on the tables it touched). When
+# it is also `idle in transaction`, no work is even being done for the price.
+#
+# The incident (2026-08-21, feedback c72a32ac): one session sat idle in
+# transaction for 18+ minutes; every `CREATE POLICY` against the live database
+# blocked to lock_timeout, and the migration applier told each agent to "fix
+# the SQL" — which was fine. Nothing named the blocker, so the diagnosis cost
+# an hour of forensics that this primitive answers in one call.
+#
+# Reusable on purpose: the async form serves in-process guards, and
+# ``long_transactions_sql`` is exported so SYNC ops tooling (psycopg migration
+# appliers, watches) runs the exact same query with no driver coupling — it
+# takes no bind parameters, so every paramstyle can execute it verbatim.
+# ---------------------------------------------------------------------------
+
+#: A transaction open at least this long is worth naming in an error message.
+LONG_TRANSACTION_SECONDS = 120.0
+
+
+def long_transactions_sql(min_seconds: float = LONG_TRANSACTION_SECONDS) -> str:
+    """SQL naming every transaction open longer than ``min_seconds``.
+
+    Returns one row per session with a ``category`` of ``idle_in_transaction``
+    (open and doing nothing — always pathological) or ``long_active`` (open and
+    working — legitimate for a bulk job, still a blocker for DDL). Takes no
+    bind parameters: ``min_seconds`` is a float this code controls and is
+    embedded as a literal, so the same string runs under asyncpg, psycopg, or
+    psql unchanged.
+    """
+    seconds = float(min_seconds)
+    return f"""
+        SELECT pid,
+               CASE WHEN state LIKE 'idle in transaction%'
+                    THEN 'idle_in_transaction' ELSE 'long_active' END AS category,
+               usename,
+               application_name,
+               client_addr::text AS client_addr,
+               state,
+               wait_event_type,
+               wait_event,
+               round(EXTRACT(EPOCH FROM (now() - xact_start))::numeric, 1) AS xact_seconds,
+               round(EXTRACT(EPOCH FROM (now() - state_change))::numeric, 1) AS state_seconds,
+               to_char(xact_start, 'YYYY-MM-DD HH24:MI:SS TZ') AS xact_start,
+               left(regexp_replace(query, '\\s+', ' ', 'g'), 300) AS query
+          FROM pg_stat_activity
+         WHERE datname = current_database()
+           AND pid <> pg_backend_pid()
+           AND xact_start IS NOT NULL
+           AND backend_type = 'client backend'
+           AND now() - xact_start > interval '{seconds} seconds'
+         ORDER BY xact_start
+         LIMIT 50
+    """
+
+
+async def long_transactions(
+    config_name: str,
+    *,
+    min_seconds: float = LONG_TRANSACTION_SECONDS,
+    connection: Any | None = None,
+) -> list[dict[str, Any]]:
+    """Every transaction on this database open longer than ``min_seconds``.
+
+    Read-only. Never raises for the caller: a probe that cannot run returns an
+    empty list and logs, because this is diagnostics — it must never be the
+    reason a guard or a migration applier crashes.
+    """
+    sql = long_transactions_sql(min_seconds)
+    try:
+        if connection is not None:
+            rows = await connection.fetch(sql)
+        else:
+            from matrx_orm.core.async_db_manager import AsyncDatabaseManager
+
+            async with AsyncDatabaseManager.get_connection(config_name, timeout=10.0) as conn:
+                rows = await conn.fetch(sql)
+    except Exception as exc:
+        logger.warning("[matrx_orm] long_transactions probe failed: %s", exc)
+        return []
+    return [dict(row) for row in rows]
+
+
+def describe_transaction(row: dict[str, Any]) -> str:
+    """One dense line naming a blocker: who it is, how long, and its last query."""
+    return (
+        f"pid={row.get('pid')} {row.get('category')} "
+        f"xact_age={row.get('xact_seconds')}s state={row.get('state')!r} "
+        f"user={row.get('usename')!r} app={row.get('application_name')!r} "
+        f"client={row.get('client_addr')} "
+        f"wait={row.get('wait_event_type')}/{row.get('wait_event')} "
+        f"last_query={row.get('query')!r}"
+    )
+
+
+def describe_transactions(rows: list[dict[str, Any]]) -> str:
+    """Multi-line blocker report, newest last. Empty string when there are none."""
+    return "\n".join(f"  - {describe_transaction(r)}" for r in rows)

{matrx_orm-3.1.96 → matrx_orm-3.1.97}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matrx-orm"
-version = "3.1.96"
+version = "3.1.97"
 description = "Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching"
 readme = "README.md"
 license = { text = "MIT" }
@@ -41,7 +41,7 @@ dependencies = [
     "gitpython>=3.1.40",
     "psycopg[binary]>=3.2.5",
     "psycopg-pool>=3.2.5",
-    "matrx-utils>=2.0.20",
+    "matrx-utils>=2.0.21",
     "pyyaml>=6.0.3",
     "python-dotenv>=1.0",
     "cryptography>=43.0",

matrx_orm-3.1.96/matrx_orm/core/diagnostics.py

@@ -1,115 +0,0 @@
-"""Pool diagnostics — surface the pool config and live server-side limits.
-
-Call ``log_pool_diagnostics(config_name)`` once at app startup (after the
-first query) to log:
-
-- The resolved pool mode (transaction vs session) and effective port.
-- The pool ceiling (min/max) the app is configured for.
-- The Postgres server's ``max_connections`` setting.
-- The current per-pool connection counts (``pool.get_size()``,
-  ``pool.get_idle_size()``).
-
-Surfacing these at startup makes Supavisor saturation issues debuggable
-*before* they happen — a misconfigured ``pool_max`` is visible in the
-boot log instead of as a 502 in production.
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import Any
-
-from matrx_orm.core.config import get_database_config, resolve_pool_mode
-
-logger = logging.getLogger("matrx_orm.diagnostics")
-
-
-async def collect_pool_diagnostics(config_name: str) -> dict[str, Any]:
-    """Return a dict of diagnostic info for ``config_name``.
-
-    Creates the pool if it doesn't exist yet (so the diagnostics also serve
-    as a cheap "warm up" probe). Safe to call from any async context.
-    """
-    from matrx_orm.core.async_db_manager import AsyncDatabaseManager
-
-    config = get_database_config(config_name)
-    pool = await AsyncDatabaseManager.get_pool(config_name)
-
-    server_max_connections: str | None = None
-    server_version: str | None = None
-    inet_port: str | None = None
-    application_name: str | None = None
-
-    try:
-        async with AsyncDatabaseManager.get_connection(config_name, timeout=5.0) as conn:
-            server_max_connections = await conn.fetchval(
-                "SELECT current_setting('max_connections')"
-            )
-            server_version = await conn.fetchval("SELECT current_setting('server_version')")
-            inet_port = await conn.fetchval("SELECT current_setting('port', true)")
-            application_name = await conn.fetchval(
-                "SELECT current_setting('application_name', true)"
-            )
-    except Exception as exc:
-        logger.warning(
-            "[matrx_orm] Diagnostics probe failed for '%s': %s",
-            config_name,
-            exc,
-        )
-
-    effective_mode = resolve_pool_mode(config)
-    configured_idle_lifetime = config.get("max_inactive_connection_lifetime")
-    if configured_idle_lifetime is None:
-        idle_lifetime_effective = 60.0 if effective_mode == "transaction" else 300.0
-        idle_lifetime_source = "default"
-    else:
-        idle_lifetime_effective = float(configured_idle_lifetime)
-        idle_lifetime_source = "configured"
-
-    return {
-        "config_name": config_name,
-        "host": config.get("host"),
-        "port": str(config.get("port")),
-        "database": config.get("database_name"),
-        "user": config.get("user"),
-        "pool_mode_configured": config.get("pool_mode"),
-        "pool_mode_effective": effective_mode,
-        "pool_min": config.get("pool_min"),
-        "pool_max": config.get("pool_max"),
-        "max_inactive_connection_lifetime": idle_lifetime_effective,
-        "max_inactive_connection_lifetime_source": idle_lifetime_source,
-        "current_pool_size": pool.get_size(),
-        "current_pool_idle": pool.get_idle_size(),
-        "server_version": server_version,
-        "server_max_connections": server_max_connections,
-        "server_inet_port": inet_port,
-        "application_name": application_name,
-    }
-
-
-async def log_pool_diagnostics(config_name: str) -> None:
-    """Emit a single human-readable INFO log line summarizing the pool state."""
-    info = await collect_pool_diagnostics(config_name)
-    # INFO, not WARNING — a healthy pool is not a warning. Surfaces only when
-    # the app raises its log level; saturation problems fail loudly elsewhere.
-    logger.info(
-        "[matrx_orm] Pool diagnostics '%s' — host=%s:%s db=%s user=%s | "
-        "mode=%s (configured=%s) min=%s max=%s idle_lifetime=%.0fs(%s) "
-        "now=%s/%s | server pg=%s max_connections=%s app=%s",
-        info["config_name"],
-        info["host"],
-        info["port"],
-        info["database"],
-        info["user"],
-        info["pool_mode_effective"],
-        info["pool_mode_configured"],
-        info["pool_min"],
-        info["pool_max"],
-        info["max_inactive_connection_lifetime"],
-        info["max_inactive_connection_lifetime_source"],
-        info["current_pool_idle"],
-        info["current_pool_size"],
-        info["server_version"],
-        info["server_max_connections"],
-        info["application_name"],
-    )