matrx-orm 3.1.113__tar.gz → 3.1.115__tar.gz

{matrx_orm-3.1.113 → matrx_orm-3.1.115}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: matrx-orm
-Version: 3.1.113
+Version: 3.1.115
 Summary: Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current

{matrx_orm-3.1.113 → matrx_orm-3.1.115}/matrx_orm/FEATURE.md

@@ -34,10 +34,33 @@
 
 - Runtime hierarchy and formatting: `exceptions.py`
 - Failure capture + org stamping: `session/fallback.py`
-- Focused unit coverage: `tests/level1/test_exceptions.py`
+- Watched-lifecycle recovery: `session/lifecycle.py`
+- Focused unit coverage: `tests/level1/test_exceptions.py`,
+  `tests/level1/test_watched_lifecycle.py`
+
+## Watchdog reads use the resilient SELECT boundary
+
+- **Lifecycle SELECTs call `adapter.execute_query`.** They inherit the ORM's
+  bounded retry for one transient command timeout on a fresh connection.
+- **Raw borrowed-connection reads are forbidden here.** A direct `conn.fetch`
+  bypasses read recovery and turns a healthy table's transient timeout into a
+  `lifecycle_watchdog_query_failure`.
+
+## Watchdog transitions are the alert claim
+
+- **Only the replica that changes a timed-out row emits its stuck-row alert.**
+  `_transition_stuck_row()` returns the guarded UPDATE result; losing replicas
+  suppress duplicate `app_log` and `system_error` records for the same row.
+- **Alert-only watches still report the SELECT result.** With no timeout
+  transition there is no database claim, so their in-process reminder cadence
+  remains authoritative.
 
 ## Change Log
 
+- 2026-08-26 — Made the guarded timeout transition the multi-replica alert
+  claim so one stuck row produces one structured watchdog occurrence.
+- 2026-08-26 — Routed watched-lifecycle SELECTs through the resilient adapter
+  read boundary so one transient command timeout retries on a fresh connection.
 - 2026-08-24 — Lifecycle watchdog ERROR boundaries now call `record_error` so
   stuck rows plus SELECT/transition failures reach `ops.system_error` instead
   of existing only in `app_log`.

{matrx_orm-3.1.113 → matrx_orm-3.1.115}/matrx_orm/session/lifecycle.py

@@ -415,15 +415,18 @@ async def _fetch_stuck_rows(
 
     adapter = AdapterRegistry.get(database)
     sql, params = _build_select_query(cfg, limit)
-    async with adapter.get_connection() as conn:
-        return await conn.fetch(sql, *params)
+    # Route watchdog reads through the adapter's normal SELECT boundary. That
+    # path retries one transient command timeout on a fresh connection; using
+    # a raw borrowed connection here bypassed that recovery and emitted a
+    # lifecycle_watchdog_query_failure for an otherwise healthy table.
+    return await adapter.execute_query(sql, *params)
 
 
 async def _transition_stuck_row(
     cfg: WatchedLifecycleConfig,
     pk_value: Any,
     database: str | None,
-) -> None:
+) -> bool:
     """UPDATE one stuck row at timeout. Fully parameterized.
 
     Builds the SET list dynamically so a single UPDATE atomically applies, in
@@ -442,7 +445,7 @@ async def _transition_stuck_row(
         and not cfg.extra_fields_on_timeout
         and not cfg.extra_expression_fields_on_timeout
     ):
-        return
+        return False
 
     from matrx_orm.adapters import AdapterRegistry
     from matrx_orm.core.config import get_all_database_project_names
@@ -482,7 +485,7 @@ async def _transition_stuck_row(
         )
 
     if not set_clauses:
-        return
+        return False
 
     predicate_sql = cfg.predicate.as_sql(params)
     params.append(pk_value)
@@ -500,7 +503,10 @@ async def _transition_stuck_row(
         f"AND {cfg.age_column} < now() - make_interval(secs => ${age_param})"
     )
     async with adapter.get_connection() as conn:
-        await conn.execute(sql, *params)
+        result = await conn.execute(sql, *params)
+    # Multiple app replicas run the sweep. The guarded UPDATE is the claim:
+    # only the replica that changed the row may report it as newly stuck.
+    return result == "UPDATE 1"
 
 
 def _format_watchdog_db_failure(
@@ -625,9 +631,34 @@ async def sweep_once(
             _should_emit_stuck_rows_alert(cfg, stuck_count=0)
             continue
 
+        claimed_rows = rows
+        if cfg.transition_to_on_timeout is not None:
+            claimed_rows = []
+            for row in rows:
+                try:
+                    if await _transition_stuck_row(cfg, row["pk"], database):
+                        claimed_rows.append(row)
+                except Exception as exc:
+                    banner, structured = _format_watchdog_db_failure(
+                        cfg, "UPDATE", exc, pk=row["pk"]
+                    )
+                    vcprint(banner, color="red")
+                    logger.error("watchdog_update_failed", extra=structured)
+                    await _capture_watchdog_error(
+                        exc,
+                        kind="lifecycle_watchdog_transition_failure",
+                        cfg=cfg,
+                        phase="update",
+                        database=database,
+                    )
+
+        claimed_count = len(claimed_rows)
+        if claimed_count == 0:
+            continue
+
         color = _alert_color(cfg.alert_channel)
         ages_seconds = []
-        for row in rows:
+        for row in claimed_rows:
             try:
                 aged: datetime = row["aged"]
                 if aged.tzinfo is None:
@@ -637,10 +668,10 @@ async def sweep_once(
                 pass
         max_age_seen = max(ages_seconds) if ages_seconds else 0
 
-        if _should_emit_stuck_rows_alert(cfg, stuck_count=stuck_count):
+        if _should_emit_stuck_rows_alert(cfg, stuck_count=claimed_count):
             summary = _format_stuck_rows_summary(
                 cfg,
-                stuck_count=stuck_count,
+                stuck_count=claimed_count,
                 max_age_seen=max_age_seen,
             )
             _emit_stuck_rows_summary(summary, color=color)
@@ -650,7 +681,7 @@ async def sweep_once(
                     "matrx_watchdog": {
                         "table": cfg.table,
                         "schema": cfg.schema,
-                        "stuck_count": stuck_count,
+                        "stuck_count": claimed_count,
                         "predicate": repr(cfg.predicate),
                         "max_age_secs": cfg.max_age_secs,
                         "alert_channel": cfg.alert_channel,
@@ -664,25 +695,10 @@ async def sweep_once(
                 kind="lifecycle_watchdog_stuck_rows",
                 cfg=cfg,
                 phase="stuck_rows",
-                stuck_count=stuck_count,
+                stuck_count=claimed_count,
                 database=database,
             )
 
-        for row in rows:
-            try:
-                await _transition_stuck_row(cfg, row["pk"], database)
-            except Exception as exc:
-                banner, structured = _format_watchdog_db_failure(cfg, "UPDATE", exc, pk=row["pk"])
-                vcprint(banner, color="red")
-                logger.error("watchdog_update_failed", extra=structured)
-                await _capture_watchdog_error(
-                    exc,
-                    kind="lifecycle_watchdog_transition_failure",
-                    cfg=cfg,
-                    phase="update",
-                    database=database,
-                )
-
     return out
 
 

{matrx_orm-3.1.113 → matrx_orm-3.1.115}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matrx-orm"
-version = "3.1.113"
+version = "3.1.115"
 description = "Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching"
 readme = "README.md"
 license = { text = "MIT" }

{matrx_orm-3.1.113 → matrx_orm-3.1.115}/tests/level1/test_watched_lifecycle.py

@@ -80,6 +80,7 @@ def captured_update(monkeypatch):
         async def execute(self, sql, *params):
             captured["sql"] = sql
             captured["params"] = params
+            return "UPDATE 1"
 
     class _Adapter:
         @asynccontextmanager
@@ -138,6 +139,37 @@ async def test_timeout_update_stamps_extra_fields(captured_update):
     )
 
 
+@pytest.mark.asyncio
+async def test_stuck_row_select_uses_resilient_adapter_read(monkeypatch):
+    captured: dict[str, object] = {}
+
+    class _Adapter:
+        async def execute_query(self, sql, *params):
+            captured["sql"] = sql
+            captured["params"] = params
+            return [{"pk": "call-1", "aged": lc.datetime.now(lc.UTC)}]
+
+        def get_connection(self):
+            raise AssertionError("watchdog SELECT bypassed the resilient read boundary")
+
+    fake_adapters = types.ModuleType("matrx_orm.adapters")
+    fake_adapters.AdapterRegistry = types.SimpleNamespace(get=lambda db: _Adapter())
+    monkeypatch.setitem(sys.modules, "matrx_orm.adapters", fake_adapters)
+
+    cfg = lc.WatchedLifecycleConfig(
+        schema="chat",
+        table="tool_call",
+        predicate=ArrayAny(F("status"), ["pending", "running"]),
+        max_age_secs=1800,
+    )
+
+    rows = await lc._fetch_stuck_rows(cfg, "matrx", 500)
+
+    assert rows[0]["pk"] == "call-1"
+    assert "FROM chat.tool_call" in str(captured["sql"])
+    assert captured["params"] == (["pending", "running"], 1800)
+
+
 @pytest.mark.asyncio
 async def test_extra_fields_only_without_status_transition(captured_update):
     cfg = lc.WatchedLifecycleConfig(
@@ -300,7 +332,7 @@ async def test_sweep_forces_stuck_row_into_system_error(monkeypatch):
         return [{"pk": "job-1", "aged": lc.datetime.now(lc.UTC)}]
 
     async def fake_transition(*_args, **_kwargs):
-        return None
+        return True
 
     async def fake_record_error(_exc: BaseException, **fields):
         captured.append(fields)
@@ -316,6 +348,37 @@ async def test_sweep_forces_stuck_row_into_system_error(monkeypatch):
     assert [row["kind"] for row in captured] == ["lifecycle_watchdog_stuck_rows"]
 
 
+@pytest.mark.asyncio
+async def test_sweep_only_alerts_replica_that_wins_timeout_transition(monkeypatch):
+    captured: list[dict] = []
+    cfg = lc.WatchedLifecycleConfig(
+        table="tool_call",
+        schema="chat",
+        predicate=Compare(F("status"), "=", "running"),
+        max_age_secs=1800,
+        transition_to_on_timeout="error",
+    )
+
+    async def fake_fetch(*_args, **_kwargs):
+        return [{"pk": "call-1", "aged": lc.datetime.now(lc.UTC)}]
+
+    async def lost_transition_race(*_args, **_kwargs):
+        return False
+
+    async def fake_record_error(_exc: BaseException, **fields):
+        captured.append(fields)
+
+    monkeypatch.setattr(lc.WatchedLifecycleRegistry, "all", lambda: [cfg])
+    monkeypatch.setattr(lc, "_fetch_stuck_rows", fake_fetch)
+    monkeypatch.setattr(lc, "_transition_stuck_row", lost_transition_race)
+    monkeypatch.setattr(lc, "_emit_stuck_rows_summary", lambda *_args, **_kwargs: None)
+    monkeypatch.setattr(lc, "record_error", fake_record_error)
+
+    await lc.sweep_once(database="matrx")
+
+    assert captured == []
+
+
 def test_alert_only_watch_alerts_on_change_and_hourly_reminder():
     cfg = lc.WatchedLifecycleConfig(
         table="system_write_failure",