matrx-runtime 0.0.2__tar.gz → 0.0.4__tar.gz

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/CLAUDE.md

@@ -51,7 +51,11 @@ A test (`test_proof.py`) proves it: one engine drives `utility` + `conversation`
   unleased actives are structurally invisible to the reaper, **`find_stuck(now=, older_than=)`**
   is the independent second layer: it RETURNS (never mutates) the reaper-blind stuck rows
   (unleased/ never-started actives past a deadline) so the host watchdog screams — no work is
-  ever silently lost. Both host-scheduled.
+  ever silently lost. `waiting_input` is excluded from BOTH layers (a legitimate unbounded
+  wait), so **`reap_stale_waiting(older_than_seconds=)`** is the host's far-future
+  abandonment backstop: any execution still waiting past the host threshold (staleness from
+  the last status change, never `created_at`) → FAILED `matrx_execution_wait_abandoned`,
+  cost preserved, CAS-safe. All host-scheduled.
 - **`MeterEvent`** (`models.py`) — the metering primitive. A consumer subclasses it
   with numeric fields; EVERY numeric is a tracked quantity summed per execution +
   tree; `money_field` feeds the budgeted `cost`. Covers money, tokens, rate-limits,

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrx-runtime
-Version: 0.0.2
+Version: 0.0.4
 Summary: The durable-execution substrate: one global_execution spine, one lifecycle state machine, one cost ledger, that every matrx-* executor (agent loop, workflow superstep, utilities) plugs into.
 Author-email: Matrx <admin@aimatrx.com>
 License: MIT

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/matrx_runtime/db/bootstrap/020_runtime.sql

@@ -104,6 +104,12 @@ create index if not exists ix_exec_type_status on runtime.global_execution (type
 -- an index range scan, not a growing seq scan over every execution ever created.
 create index if not exists ix_exec_stuck_scan on runtime.global_execution (created_at)
   where status in ('pending','running','paused');
+-- waiting_input abandonment backstop (reap_stale_waiting): stale-suspend scan by the
+-- suspend instant (updated_at). PARTIAL for the same reason as the stuck scan — a
+-- healthy spine holds few parked rows, so the per-minute sweep stays an index range
+-- scan instead of a growing seq scan.
+create index if not exists ix_exec_waiting_scan on runtime.global_execution (updated_at)
+  where status = 'waiting_input';
 create index if not exists ix_exec_event on runtime.global_execution_event (execution_id, created_at);
 create index if not exists ix_exec_ckpt on runtime.global_execution_checkpoint (execution_id, created_at desc);
 create index if not exists ix_meter_exec on runtime.global_meter_entry (execution_id);

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/matrx_runtime/engine.py

@@ -283,6 +283,56 @@ class ExecutionEngine:
             reaped.append(ex_id)
         return reaped
 
+    async def reap_stale_waiting(
+        self, *, older_than_seconds: float, now: datetime | None = None
+    ) -> list[str]:
+        """Far-future abandonment backstop for `waiting_input`. A legitimate unbounded
+        wait is excluded from BOTH crash-recovery layers (the reaper sweeps leased
+        RUNNING only; `find_stuck` excludes waiting_input) — so an input that never
+        arrives would otherwise stay open FOREVER. Any execution still waiting
+        `older_than_seconds` after its suspend (measured from the last status change,
+        never `created_at`) is settled FAILED with a structured
+        `matrx_execution_wait_abandoned` error, cost preserved. The threshold is the
+        HOST's (runtime doesn't know what the input is or how long is reasonable) and
+        must be far-future — a backstop, never an answer deadline. Idempotent + CAS-safe
+        exactly like `reap_expired`: an input answered between the sweep and the write
+        loses the CAS and is skipped."""
+        at = now or _now()
+        reaped: list[str] = []
+        for ex in await self._store.find_stale_waiting(
+            now=at, older_than_seconds=older_than_seconds
+        ):
+            error = ExecutionError(
+                error_type="matrx_execution_wait_abandoned",
+                message=(
+                    f"execution {ex.id} abandoned: waiting_input for more than "
+                    f"{older_than_seconds:.0f}s with no input; closed by the host backstop."
+                ),
+            )
+            outcome = ExecutionOutcome(
+                status=ExecutionStatus.FAILED, cost=ex.cost, error=error
+            )
+            # The CAS is pinned to WAITING_INPUT — NOT the generic fail() path, whose
+            # _settle CAS's on whatever status it re-reads. With fail(), a resume that
+            # flipped WAITING_INPUT→RUNNING between the finder and the settle would be
+            # read as RUNNING and RUNNING→FAILED is a LEGAL transition — false-failing a
+            # live, freshly-resumed run. Pinning expected=WAITING_INPUT makes any racing
+            # resume win the CAS: the sweep then simply skips the row.
+            result = transition(ExecutionStatus.WAITING_INPUT, ExecutionStatus.FAILED, outcome=outcome)
+            won = await self._store.compare_and_set_status(
+                ex.id,
+                expected=ExecutionStatus.WAITING_INPUT,
+                new_status=ExecutionStatus.FAILED,
+                ended_at=_now(),
+                cost=ex.cost,
+                error=error,
+            )
+            if not won:
+                continue  # resumed/settled out from under us — the live outcome wins
+            await self._emit(ex.id, result.event, detail=_outcome_detail(outcome))
+            reaped.append(ex.id)
+        return reaped
+
     async def find_stuck(
         self,
         *,
@@ -686,6 +736,33 @@ class ExecutionScope:
             raise RuntimeError("ExecutionScope used before it was entered (`async with`).")
         return self._execution
 
+    @classmethod
+    def for_existing(
+        cls, engine: ExecutionEngine, execution: GlobalExecution
+    ) -> ExecutionScope:
+        """Bind a scope to an ALREADY-STARTED execution — the RESUME path, where ONE
+        execution spans many requests (design §`request` ⊂ `execution`). The caller has
+        re-`start()`ed a PAUSED/WAITING_INPUT execution back to RUNNING (with a fresh
+        lease); this wraps it so the SAME `record`/`note`/`complete`/`fail`/`request_input`
+        machinery drives the resumed segment — no create, no second execution.
+
+        `scope._cost` is SEEDED from the execution's prior `cost`, because the terminal
+        settle SETs the row's `cost` to `scope._cost` (absolute, not additive). Without the
+        seed a resumed segment's `complete(cost=segment)` would OVERWRITE the accumulated
+        cost with just this segment's spend, dropping every prior segment. Seeding makes the
+        terminal write the CUMULATIVE total (prior + this segment). Metered quantities
+        (tokens, etc.) are always additive via `record`, so they need no seeding."""
+        scope = cls(
+            engine,
+            type=execution.type,
+            execution_id=execution.id,
+            link_kind=execution.link_kind,
+            link_id=execution.link_id,
+        )
+        scope._execution = execution
+        scope._cost = execution.cost
+        return scope
+
     async def begin(self) -> ExecutionScope:
         """Create + start the execution (the `__aenter__` body, exposed). Use this with
         the explicit `complete`/`fail` settle when managing the lifecycle by hand

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/matrx_runtime/orm_store.py

@@ -328,6 +328,19 @@ class OrmExecutionStore:
             if getattr(r, "lease_expires_at", None) is None or r.lease_expires_at < now
         ]
 
+    async def find_stale_waiting(
+        self, *, now: datetime, older_than_seconds: float
+    ) -> list[GlobalExecution]:
+        cutoff = now - timedelta(seconds=older_than_seconds)
+        # `updated_at` (row touch trigger) IS the suspend instant: a WAITING_INPUT row
+        # receives no writes while it waits, so staleness is measured from the last
+        # status change — a month-old execution that suspended yesterday is NOT stale.
+        rows = await _ExecRow.filter(
+            status=ExecutionStatus.WAITING_INPUT.value,
+            updated_at__lt=cutoff,
+        ).all()
+        return [_to_execution(r) for r in rows]
+
     # --- events / checkpoints / meters --------------------------------------
     async def append_event(self, event: ExecutionEvent) -> None:
         await _EventRow.create(

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/matrx_runtime/store.py

@@ -15,6 +15,7 @@ from typing import Protocol, runtime_checkable
 
 from matrx_runtime.models import (
     STUCK_CANDIDATE_STATUSES,
+    TERMINAL_STATUSES,
     ExecutionCheckpoint,
     ExecutionControl,
     ExecutionError,
@@ -132,6 +133,19 @@ class ExecutionStore(Protocol):
         rows so the host can classify + scream."""
         ...
 
+    async def find_stale_waiting(
+        self, *, now: datetime, older_than_seconds: float
+    ) -> list[GlobalExecution]:
+        """`waiting_input` executions whose status last changed before
+        `now - older_than_seconds`. WAITING_INPUT is a legitimate unbounded wait, so it
+        is excluded from BOTH crash-recovery layers (the reaper sweeps leased RUNNING;
+        `find_stuck_executions` excludes it) — which means an input that never arrives
+        has no upper bound at all. This is the candidate set for the host's far-future
+        abandonment backstop (`ExecutionEngine.reap_stale_waiting`). Staleness is
+        measured from the LAST status change (a month-old execution that suspended
+        yesterday is NOT stale), never from `created_at`."""
+        ...
+
     # --- events / checkpoints / meters --------------------------------------
     async def append_event(self, event: ExecutionEvent) -> None: ...
     async def list_events(self, execution_id: str) -> list[ExecutionEvent]:
@@ -156,6 +170,9 @@ class InMemoryExecutionStore:
         self._meter_entries: list[MeterEntry] = []
         self._controls: dict[str, ExecutionControl] = {}
         self._checkpoints: dict[str, list[ExecutionCheckpoint]] = {}
+        # Last successful status-change instant per execution — the in-memory stand-in
+        # for the DB row's touch-trigger `updated_at`; feeds find_stale_waiting.
+        self._status_changed_at: dict[str, datetime] = {}
         self._lock = asyncio.Lock()
 
     @staticmethod
@@ -228,6 +245,12 @@ class InMemoryExecutionStore:
                     "error": error if error is not None else ex.error,
                 }
             )
+            if new_status in TERMINAL_STATUSES:
+                # Terminal rows can never become stale-waiting candidates — drop the
+                # entry so the map doesn't grow unboundedly in a long-lived store.
+                self._status_changed_at.pop(execution_id, None)
+            else:
+                self._status_changed_at[execution_id] = datetime.now(UTC)
             return True
 
     async def add_meters(
@@ -325,6 +348,19 @@ class InMemoryExecutionStore:
             and (ex.lease_expires_at is None or ex.lease_expires_at < now)
         ]
 
+    async def find_stale_waiting(
+        self, *, now: datetime, older_than_seconds: float
+    ) -> list[GlobalExecution]:
+        cutoff = now - timedelta(seconds=older_than_seconds)
+        out: list[GlobalExecution] = []
+        for ex in self._executions.values():
+            if ex.status is not ExecutionStatus.WAITING_INPUT:
+                continue
+            changed = self._status_changed_at.get(ex.id) or ex.created_at
+            if changed is not None and changed < cutoff:
+                out.append(ex)
+        return out
+
     # --- events / checkpoints / meters --------------------------------------
     async def append_event(self, event: ExecutionEvent) -> None:
         async with self._lock:

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matrx-runtime"
-version = "0.0.2"
+version = "0.0.4"
 description = "The durable-execution substrate: one global_execution spine, one lifecycle state machine, one cost ledger, that every matrx-* executor (agent loop, workflow superstep, utilities) plugs into."
 readme = "README.md"
 license = { text = "MIT" }

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/tests/test_lease_reaper.py

@@ -269,3 +269,65 @@ async def test_find_stuck_flags_stuck_paused_with_dead_resumer():
     later = ex.created_at + timedelta(seconds=2000)
     stuck = await engine.find_stuck(now=later, older_than_seconds=1800)
     assert [s.id for s in stuck] == [ex.id]
+
+
+# --- waiting_input abandonment backstop (reap_stale_waiting) ------------------
+# WAITING_INPUT is excluded from both crash-recovery layers by design; this is the
+# host's far-future backstop so an input that never arrives eventually terminates.
+
+
+async def test_stale_waiting_input_is_reaped_to_failed_wait_abandoned():
+    engine, store = _engine()
+    ex = await engine.create_root(type="conversation")
+    await engine.start(ex.id, holder="worker-1")
+    await engine.record(ex.id, Spend(usd=Decimal("0.40")))
+    await engine.request_input(ex.id)
+    store._status_changed_at[ex.id] = _at(0)  # suspended at t0 (deterministic age)
+
+    reaped = await engine.reap_stale_waiting(older_than_seconds=3600, now=_at(7200))
+    assert reaped == [ex.id]
+    row = await engine.get_execution(ex.id)
+    assert row.status is ExecutionStatus.FAILED
+    assert row.error.error_type == "matrx_execution_wait_abandoned"
+    assert row.cost == Decimal("0.40")  # spend preserved through the close
+
+
+async def test_fresh_waiting_input_is_left_alone():
+    engine, store = _engine()
+    ex = await engine.create_root(type="conversation")
+    await engine.start(ex.id, holder="worker-1")
+    await engine.request_input(ex.id)
+    store._status_changed_at[ex.id] = _at(0)
+
+    assert await engine.reap_stale_waiting(older_than_seconds=3600, now=_at(1800)) == []
+    assert (await engine.get_execution(ex.id)).status is ExecutionStatus.WAITING_INPUT
+
+
+async def test_staleness_measured_from_suspend_not_created_at():
+    """A month-old execution that suspended RECENTLY is NOT stale — staleness is
+    measured from the last status change, never created_at."""
+    engine, store = _engine()
+    ex = await engine.create_root(type="conversation")
+    await engine.start(ex.id, holder="worker-1")
+    await engine.request_input(ex.id)
+    # created long ago; suspend happened just now
+    store._executions[ex.id] = store._executions[ex.id].model_copy(
+        update={"created_at": _at(-30 * 24 * 3600)}
+    )
+    store._status_changed_at[ex.id] = _at(7000)
+
+    assert await engine.reap_stale_waiting(older_than_seconds=3600, now=_at(7200)) == []
+    assert (await engine.get_execution(ex.id)).status is ExecutionStatus.WAITING_INPUT
+
+
+async def test_reap_stale_waiting_ignores_running_and_terminal():
+    engine, store = _engine()
+    running = await engine.create_root(type="utility")
+    await engine.start(running.id, holder="w")
+    done = await engine.create_root(type="utility")
+    await engine.start(done.id)
+    await engine.complete(done.id)
+    for ex_id in (running.id, done.id):
+        store._status_changed_at[ex_id] = _at(0)
+
+    assert await engine.reap_stale_waiting(older_than_seconds=60, now=_at(10_000)) == []

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/tests/test_orm_store.py

@@ -5,7 +5,7 @@ the mappings) for the payload-blind tables."""
 
 from __future__ import annotations
 
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from decimal import Decimal
 from types import SimpleNamespace
 
@@ -204,3 +204,18 @@ async def test_origin_upsert_conflict_on_app_surface(monkeypatch):
     await _store().upsert_origin(Origin(id="o1", app="frontend", surface="chat", allow_unknown=True))
     assert captured["conflict"] == ["app", "surface"]
     assert captured["data"]["allow_unknown"] is True
+
+
+async def test_find_stale_waiting_filters_status_and_updated_at(monkeypatch):
+    seen = {}
+
+    def filter_(**kwargs):
+        seen.update(kwargs)
+        return _FakeQuery(rows=[])
+
+    _patch(monkeypatch, "_ExecRow", filter=filter_)
+    now = datetime(2026, 6, 25, 12, 0, tzinfo=UTC)
+    rows = await _store().find_stale_waiting(now=now, older_than_seconds=3600)
+    assert rows == []
+    assert seen["status"] == "waiting_input"
+    assert seen["updated_at__lt"] == now - timedelta(seconds=3600)

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/tests/test_payload_blind.py

@@ -45,6 +45,18 @@ FORBIDDEN: frozenset[str] = frozenset(
     }
 )
 
+# Exact identifiers that contain a forbidden WORD but are not a payload CONCEPT.
+# Kept as whole identifiers (never a substring rule) so the guard keeps all its
+# teeth: `token`, `access_token`, `token_count`, `tool_call` still trip.
+#
+#   _entity_token — matrx-orm's MatrxEntity registry attribute, stamped into the
+#     GENERATED db/models_runtime.py by the schema generator (it is the ENTITY
+#     TYPE's name, e.g. `_entity_token = "global_execution"`). It is an ORM
+#     framework field, not an auth/LLM token, and its values are themselves
+#     payload-blind. The generated file cannot be hand-edited, so the exemption
+#     has to live here.
+ALLOWED_IDENTIFIERS: frozenset[str] = frozenset({"_entity_token"})
+
 _PKG_ROOT = Path(__file__).resolve().parent.parent / "matrx_runtime"
 
 
@@ -54,6 +66,8 @@ def _forbidden_hits(path: Path) -> list[tuple[int, str]]:
         for tok in tokenize.tokenize(fh.readline):
             if tok.type != tokenize.NAME:
                 continue
+            if tok.string in ALLOWED_IDENTIFIERS:
+                continue
             for part in tok.string.lower().split("_"):
                 if part in FORBIDDEN:
                     hits.append((tok.start[0], tok.string))
@@ -97,3 +111,22 @@ def test_the_scanner_actually_catches_a_forbidden_word(tmp_path: Path):
         "    return label\n"
     )
     assert _forbidden_hits(clean) == []
+
+
+def test_entity_token_exemption_is_narrow(tmp_path: Path):
+    """The ORM's `_entity_token` framework attribute is exempt — but the exemption
+    is an EXACT identifier match, so any real token concept still SCREAMS."""
+    ok = tmp_path / "ok.py"
+    ok.write_text('class M:\n    _entity_token = "global_execution"\n')
+    assert _forbidden_hits(ok) == []
+
+    # Anything that is actually about tokens must still be caught.
+    for leak_src in (
+        "def f():\n    token = 1\n",
+        "def f():\n    access_token = 1\n",
+        "def f():\n    token_count = 1\n",
+        "def f():\n    entity_token = 1\n",  # no leading underscore -> not the ORM attr
+    ):
+        leak = tmp_path / "leak2.py"
+        leak.write_text(leak_src)
+        assert _forbidden_hits(leak), f"guard went blind on: {leak_src!r}"

{matrx_runtime-0.0.2 → matrx_runtime-0.0.4}/tests/test_scope.py

@@ -55,6 +55,61 @@ async def test_clean_exit_auto_completes_with_accrued_cost():
     assert settled.meters["input_tokens"] == Decimal("5")
 
 
+async def test_for_existing_resume_accumulates_cost_across_segments():
+    """The RESUME path: ONE execution spans suspend→resume. A first segment parks the
+    execution WAITING_INPUT with cost recorded; a re-attached scope (ExecutionScope.for_existing
+    on the re-started execution) records more and completes — the terminal cost must be the
+    CUMULATIVE total (prior + this segment), NOT just the resumed segment (the row's cost is SET,
+    not added, at terminal, so the scope seeds _cost from the prior cost)."""
+    from matrx_runtime import ExecutionScope
+
+    engine = _engine()
+
+    # Segment 1: run, spend 0.10, then suspend awaiting input.
+    async with engine.execution(type="conversation") as exe:
+        await exe.record(Spend(usd=Decimal("0.10"), input_tokens=100))
+        ex_id = exe.id
+        await exe.request_input()  # marks handled → clean exit does NOT auto-complete
+    parked = await engine.get_execution(ex_id)
+    assert parked.status is ExecutionStatus.WAITING_INPUT
+    assert parked.cost == Decimal("0.10")
+
+    # Segment 2 (resume): re-start → RUNNING, bind a scope to it, spend 0.05 more, complete.
+    started = await engine.start(ex_id, holder="resumer", lease_seconds=300)
+    assert started.status is ExecutionStatus.RUNNING
+    scope = ExecutionScope.for_existing(engine, started)
+    assert scope.cost == Decimal("0.10")  # seeded from the prior cost
+    await scope.record(Spend(usd=Decimal("0.05"), input_tokens=50))
+    await scope.complete()
+
+    settled = await engine.get_execution(ex_id)
+    assert settled.status is ExecutionStatus.COMPLETED
+    assert settled.cost == Decimal("0.15")            # cumulative, not just the 0.05 segment
+    assert settled.meters["input_tokens"] == Decimal("150")  # tokens additive across segments
+
+
+async def test_for_existing_resume_can_re_suspend():
+    """A resumed segment that suspends again re-parks the SAME execution WAITING_INPUT, ready
+    for the next resume — the execution cycles RUNNING↔WAITING_INPUT until it finally settles."""
+    from matrx_runtime import ExecutionScope
+
+    engine = _engine()
+    async with engine.execution(type="conversation") as exe:
+        await exe.record(Spend(usd=Decimal("0.10")))
+        ex_id = exe.id
+        await exe.request_input()
+
+    started = await engine.start(ex_id, holder="resumer", lease_seconds=300)
+    scope = ExecutionScope.for_existing(engine, started)
+    await scope.record(Spend(usd=Decimal("0.05")))
+    await scope.request_input()  # re-suspend
+
+    reparked = await engine.get_execution(ex_id)
+    assert reparked.status is ExecutionStatus.WAITING_INPUT
+    assert reparked.cost == Decimal("0.15")           # cumulative across both segments
+    assert reparked.lease_holder is None              # lease released on re-suspend
+
+
 async def test_exception_auto_fails_and_reraises():
     engine = _engine()
     captured_id = None