matrx-graph 0.2.330__py3-none-any.whl → 0.2.332__py3-none-any.whl

matrx_graph/compile.py

@@ -18,7 +18,7 @@ from matrx_graph.errors import CompilationError
 from matrx_graph.executor.schema_validation import schema_validity_errors
 from matrx_graph.executor.registry import NodeRegistry, default_registry
 from matrx_graph.types.channel import ChannelSpec
-from matrx_graph.types.compiled import CompiledEdge, CompiledGraph, CompiledNode
+from matrx_graph.types.compiled import CompiledEdge, CompiledGraph, CompiledNode, StepClock
 from matrx_graph.types.input_surface import InputSpec, hint_values
 from matrx_graph.types.definition import Definition
 from matrx_graph.types.primitives import EdgeKind, RetryPolicy
@@ -285,6 +285,10 @@ def compile(  # noqa: A001 — mirrors matplotlib-style top-level compile()
             except Exception as e:  # noqa: BLE001
                 raise CompilationError(f"node '{node.id}' has an invalid retry_policy: {e}") from e
 
+        # Per-node wall clock. Authored as data.step_clock = {seconds, knob?:
+        # {feature, key}}; a bad value fails compilation (loud), never a run.
+        step_clock = _compile_step_clock(node.id, node.data.get("step_clock"))
+
         # Per-node output-channel binding. Authored as
         # data.output_channel = "<channel name>": the scheduler writes the
         # node's full output dump to that channel after success/mock, so
@@ -393,6 +397,7 @@ def compile(  # noqa: A001 — mirrors matplotlib-style top-level compile()
             writes_channels=writes,
             mock_response=mock_response,
             retry_policy=retry_policy,
+            step_clock=step_clock,
             effective_output_schema=effective_output_schema,
             output_channel=output_channel,
             input_kind=input_kind,
@@ -562,3 +567,39 @@ def _build_input_surface(
         declared_by[var.name] = "the workflow's declared variables"
 
     return surface
+
+
+def _compile_step_clock(node_id: str, raw: Any) -> StepClock | None:
+    """``data.step_clock`` → :class:`StepClock`, or a CompilationError naming the node."""
+    if raw is None:
+        return None
+    from matrx_graph.knobs import MAX_STEP_SECONDS, MIN_STEP_SECONDS
+
+    if not isinstance(raw, dict):
+        raise CompilationError(f"node '{node_id}' has an invalid step_clock: expected an object, got {raw!r}")
+    seconds = raw.get("seconds")
+    if isinstance(seconds, bool) or not isinstance(seconds, int) or not (
+        MIN_STEP_SECONDS <= seconds <= MAX_STEP_SECONDS
+    ):
+        raise CompilationError(
+            f"node '{node_id}' has an invalid step_clock.seconds {seconds!r}: a whole number "
+            f"from {MIN_STEP_SECONDS} to {MAX_STEP_SECONDS}"
+        )
+    knob = raw.get("knob")
+    unknown = set(raw) - {"seconds", "knob"}
+    if unknown:
+        raise CompilationError(f"node '{node_id}' step_clock has unknown keys {sorted(unknown)}")
+    if knob is None:
+        return StepClock(seconds=seconds)
+    if not (
+        isinstance(knob, dict)
+        and isinstance(knob.get("feature"), str)
+        and knob["feature"].strip()
+        and isinstance(knob.get("key"), str)
+        and knob["key"].strip()
+        and set(knob) <= {"feature", "key"}
+    ):
+        raise CompilationError(
+            f"node '{node_id}' has an invalid step_clock.knob {knob!r}: expected {{feature, key}}"
+        )
+    return StepClock(seconds=seconds, knob_feature=knob["feature"], knob_key=knob["key"])

matrx_graph/executor/scheduler.py

@@ -2181,6 +2181,11 @@ class Scheduler:
         try:
             while open_members:
                 task, result, result_step = await done_queue.get()
+                if task.cancelled() and _job_lease().lost():
+                    # A member stopped because this copy lost its job lease (``JobLeaseLost`` is a cancellation):
+                    # the whole copy stops here, never routing or checkpointing on (lane K1b, B2).
+                    lease = _job_lease().current()
+                    raise _job_lease().JobLeaseLost(f"job lease lost: {lease.why if lease else ''}")
                 if id(result) in member_order and result_step == step:
                     # Kept in frontier order by insertion — O(log n) search per finished member, never a re-sort.
                     bisect.insort(finished_members, result, key=lambda r: member_order[id(r)])
@@ -3261,7 +3266,9 @@ class Scheduler:
                 # ``matrx_graph.knobs`` § THE STEP WALL CLOCK.
                 from matrx_graph.knobs import step_timeout_seconds
 
-                clock_seconds, clock_source = await step_timeout_seconds(spec.timeout_ms)
+                clock_seconds, clock_source = await step_timeout_seconds(
+                    spec.timeout_ms, node_clock=node.step_clock
+                )
                 try:
                     try:
                         output = await asyncio.wait_for(
@@ -6488,3 +6495,10 @@ def _settle_default_on_decision_gate(
     logger.info("decision gate settled by its default %r → %s", raw, shaped["answer"])
     value["answer"] = shaped["answer"]
     return value
+
+
+def _job_lease():
+    """``matrx_graph.workers.job_lease``, imported on use (the workers package imports the executor)."""
+    from matrx_graph.workers import job_lease
+
+    return job_lease

matrx_graph/knobs.py

@@ -432,12 +432,34 @@ async def default_step_seconds() -> int:
     )
 
 
-async def step_timeout_seconds(declared_timeout_ms: int | None) -> tuple[float, str]:
+async def step_timeout_seconds(
+    declared_timeout_ms: int | None, *, node_clock: Any = None
+) -> tuple[float, str]:
     """``(seconds, source)`` for one node attempt.
 
-    ``source`` is the word the failure message uses, so a person reading a timed-out
-    run can tell a ceiling the workflow's own author set from the platform's.
+    Precedence: the node's own ``data.step_clock`` (its knob row when it
+    resolves, else its declared seconds) → the NodeSpec's ``timeout_ms`` → the
+    platform ceiling. ``source`` is the word the failure message uses, so a
+    person reading a timed-out run can tell a ceiling the workflow's own author
+    set from the platform's.
     """
+    if node_clock is not None:
+        seconds = int(node_clock.seconds)
+        if node_clock.knob_feature and node_clock.knob_key:
+            values = await _feature_values(node_clock.knob_feature)
+            if values is not None and node_clock.knob_key in values:
+                seconds = _coerce_int(
+                    values[node_clock.knob_key], seconds, MIN_STEP_SECONDS, MAX_STEP_SECONDS
+                )
+            else:
+                logger.error(
+                    "[matrx_graph] step clock knob %s.%s unresolved — the step keeps its "
+                    "declared %ss. REMEDY: seed that platform.feature_knob row.",
+                    node_clock.knob_feature,
+                    node_clock.knob_key,
+                    seconds,
+                )
+        return float(max(MIN_STEP_SECONDS, min(MAX_STEP_SECONDS, seconds))), "declared"
     if declared_timeout_ms is not None and declared_timeout_ms > 0:
         return declared_timeout_ms / 1000.0, "declared"
     return float(await default_step_seconds()), "platform"

matrx_graph/types/compiled.py

@@ -11,6 +11,22 @@ from matrx_graph.types.review import ReviewMark
 from matrx_graph.types.variables import VariableSpec
 
 
+class StepClock(BaseModel):
+    """One node's OWN wall-clock limit (``NodeDef.data.step_clock``).
+
+    ``seconds`` is the declared starting limit; ``knob`` (``{feature, key}``)
+    names the ``platform.feature_knob`` row an admin turns. The row wins when it
+    resolves; an unreadable row falls back to ``seconds`` — never to the
+    platform ceiling and never to "no limit".
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    seconds: int = Field(ge=1)
+    knob_feature: str | None = None
+    knob_key: str | None = None
+
+
 class CompiledNode(BaseModel):
     """Scheduler-ready view of a node.
 
@@ -48,6 +64,11 @@ class CompiledNode(BaseModel):
     # inspector without editing the action's code. None = inherit the
     # spec default.
     retry_policy: RetryPolicy | None = None
+    # Per-node wall clock (``data.step_clock``). When set it wins over the
+    # NodeSpec's ``timeout_ms`` and the platform ceiling for THIS node only —
+    # a long step (a podcast produced end to end) declares its own limit
+    # instead of the platform default being raised for every step.
+    step_clock: StepClock | None = None
     # Set when the node's NodeSpec has ``dynamic_output=True`` (e.g.
     # io.user_input): the JSON Schema computed from this node's config at
     # compile time. Used to generate the run form and for downstream type

matrx_graph/workers/FEATURE.md

@@ -17,6 +17,7 @@ and `workflow.job`'s post-reorg baseline schema on 2026-08-09.
 - **Jobs run in parallel, bounded, each in its own task (2026-09-17).** `WorkerRunner` holds an `asyncio.Semaphore(max_concurrent)`: it keeps claiming while slots are free, waits on the *clock* (`poll_interval`) only when the queue is empty, and waits on a *slot* when every one is busy. Each claimed job runs as its own `asyncio.Task` with its own lease renewer, and `complete`/`fail`/cancel semantics are per job and unchanged. **`max_concurrent` defaults to 1** — a host that has not thought about the bound keeps exactly the behavior it had; aidream passes `workflow_worker.max_concurrent_jobs` (a `platform.feature_knob` row, currently 8, read once at worker start). Why: the loop was strictly serial, so ONE slow job held every other workflow on the platform. Measured in production 2026-09-17, five simultaneous Cloud Browser activations (each a workflow job that waits on a container coming up) came up as a 5.5-second-per-run ladder, and for those 22 seconds nothing else in the queue moved; at the 50-run fleet ceiling that is a ~4.5-minute tail. **Per-job isolation is structural, not conventional**: a job task is created from the claim loop's context, so `contextvars` it sets (an ambient `AppContext`, an ORM connection) belong to that task alone — they cannot reach a sibling job and cannot outlive the job in the loop. Under the old serial `await` they did: aidream's `_run_one` installs an ambient AppContext at the tail of every job, and that context stayed installed in the claim loop afterwards. Proof: `tests/test_workers.py` — the wall-clock test, the peak-concurrency test, the neighbour-isolation test, the two drain tests, and the context-isolation test (each demonstrated failing against the serial loop before the change).
 - **Stopping is a bounded drain.** `stop()` takes no new claims; `run_forever()` then waits out the jobs already running for `stop_grace_seconds` (aidream passes its share of THE DEPLOY WINDOW) and cancels whatever is left. A canceled job fails retryable and its run resumes from its last checkpoint — the same outcome a serial worker's single job always had when the container was replaced.
 - **A worker only claims what it can run (2026-10-12, `capability.py`).** Every server — deployed fleet and developer laptops — shares one database, so a job may belong to a graph whose node types this build has not shipped (live: a local build's `brand.setup.land_logo` run was resumed by production and errored "Unknown node type"). `enqueue()` / `enqueue_claimed()` stamp `workflow.job.metadata.required_node_types` (canonical spellings, removed types excluded) from the run's stored definition; `claim_one(capabilities=...)` adds `COALESCE(j.metadata->'required_node_types','[]') <@ caps` (`matrx_orm.JsonbContainedBy`), so an unstamped legacy job stays claimable by anyone. `WorkerRunner` and `run_single_job` pass `worker_capabilities()` (the default registry) on every claim. Belt two: a host job runner that finds a gap at run time raises `JobNeedsCapability`; the runner calls `defer_for_capability()` (un-claim, attempt refunded, requirement stamped, immediately visible) — never `fail()`, never `complete()`, never a run-status write. The run says what it waits for in `workflow.run.metadata._awaiting_capability` (`missing`, `message`, `since`, `checked_at`, `checked_by`) plus one `workflow.run_log` line (`run_awaiting_capability`); every incapable worker refreshes it via `stamp_runs_awaiting_capability()` and `checked_at` is a lifecycle-watchdog liveness signal, so a waiting run is not force-failed as stranded while a fleet is watching it. The first capable worker clears the marker. Proof: `tests/test_capability_aware_claim.py` (incl. the predicate evaluated in live Postgres; fails against the pre-change queue/runner).
+- **A copy that loses its lease stops itself (lane K1, 2026-10-21).** A renewal that answers "not ours" (reaped — including a FALSE "dead" after a database outage longer than the lease, live job c1d89868 — or settled elsewhere) marks the copy's `job_lease.JobLease` lost and cancels its task; the host skips its terminal write (the copy that holds the job owns the run); many-item work calls `job_lease.check()` before each item (it also reads the job row, at most every tenth of a renewal beat). The reaper does not try to tell an outage from a death: a false "dead" is harmless by construction. Proof: aidream `aidream/services/tests/test_a_reaped_run_never_waits_on_itself.py`. **Lane K1b:** whoever notices the loss first (the renewal or the work's own `check`) cancels the copy's owner task, and run-ahead re-raises a member that ended cancelled on a lost lease (it used to swallow it and route on). A job the reaper only DEAD-LETTERED (a false "dead" on its last attempt) is taken back by the live copy (`queue.take_back_false_dead_letter`, compare-and-set) and finished — stopping would strand the run, since nothing claims a dead-lettered job; a requeued or re-claimed job still stops the old copy. A renewal landing after `complete()` is not a loss (`JobLease.settling`). Each copy's `JobLease.generation` (the job's attempt) fences what it holds (import slots). A platform-wide outage longer than the lease plus grace cancels EVERY in-flight job at once and they all restart from their checkpoints together: the same spend per job as before, but as one burst. Proof: `tests/test_a_lost_lease_never_strands_or_lingers.py`.
 - **Recovery starts before work.** `WorkerRunner.run_forever()` calls `reap_expired_leases()` before its first claim. The host's recurring reaper remains a backstop.
 - **A transient claim outage never kills the worker.** A database timeout or
   connection failure while claiming `workflow.job` is captured durably and

matrx_graph/workers/job_lease.py

@@ -0,0 +1,155 @@
+"""The job lease a running execution holds, as seen from inside the work (lane K1, 2026-10-21).
+
+Live, 2026-10-21 (run 8749d414, job c1d89868): the database dropped every connection for ten minutes, the reaper
+read the expired lease as a dead worker and requeued the job, and the SAME process started the run again while the
+first execution was still alive (hung on a store call). The first copy logged "finishing the in-flight run anyway"
+and kept its organization import slot, so the second copy waited on itself.
+
+The rule now (Arman, 2026-10-21): **a copy that loses its job lease stops itself.** The runner publishes the lease
+of the job it executes on this ContextVar; long, many-item work (an import writes item after item) calls
+:func:`check` before each item. Once the lease is gone — the renewal answered "not ours", or the job row says
+another claim holds it — ``check`` raises :class:`JobLeaseLost`, the copy stops without writing again, and the
+host skips its terminal write (the new copy owns the run). The runner also cancels the lost copy's task, so a copy
+parked between items stops too.
+
+``JobLeaseLost`` is a ``CancelledError``: to the engine and the host a lost lease is a cancellation of THIS copy,
+never a failure of the run (a failure would be written over the new copy's run).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import time
+from collections.abc import Iterator
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+from uuid import UUID
+
+log = logging.getLogger(__name__)
+
+class JobLeaseLost(asyncio.CancelledError):
+    """This copy's job lease was reaped or taken by another claim: stop now, write nothing more."""
+
+
+@dataclass
+class JobLease:
+    job_id: str
+    run_id: str
+    claimed_by: str | None
+    #: fewest seconds between two reads of the job row by ``check`` (a many-item run checks before every item); the
+    #: runner sets it from its lease (a tenth of the renewal beat), and the renewal's own answer is used in between
+    look_every: float = 0.0
+    #: the task running this copy of the job: whoever notices the loss first (the renewal, or the work's own
+    #: ``check``) cancels it, so the WHOLE copy stops at once (lane K1b, B2: run-ahead swallowed a ``JobLeaseLost``
+    #: raised inside one node, and the copy kept checkpointing and routing until the next renewal)
+    owner: asyncio.Task | None = field(default=None, repr=False)
+    #: the copy finished its work and is settling the job (``complete``): a renewal that lands after the settle
+    #: commits answers "not ours", and that is not a loss (K1b, M1)
+    settling: bool = False
+    #: per-copy fencing token for anything this copy holds (an import slot): a newer copy's is always greater
+    generation: float = field(default_factory=time.time)
+    visibility_timeout: int = 600
+    lost: bool = False
+    why: str = ""
+    _looked: float = field(default=0.0, repr=False)
+
+    def mark_lost(self, why: str) -> None:
+        if self.settling:
+            log.info("job %s: settled while a renewal was in flight; not a lost lease", self.job_id[:8])
+            return
+        if not self.lost:
+            log.error("job %s (run %s) lost its lease (%s): this copy stops and writes nothing more",
+                      self.job_id[:8], self.run_id[:8], why)
+        self.lost, self.why = True, why
+        owner = self.owner
+        if owner is not None and not owner.done() and owner is not _running_task():
+            owner.cancel(f"job {self.job_id[:8]} lease lost: {why}")
+
+    async def still_held(self) -> bool | None:
+        """The job row's answer: True while this claim holds it, False once it does not, None when unreadable."""
+        from matrx_graph.db._registry import get_model
+
+        try:
+            rows = await get_model("WfJob").filter(id=str(UUID(self.job_id))).all(use_cache=False)
+        except Exception:  # noqa: BLE001 — unreadable (an outage): the lease is not judged on a store that is down
+            log.warning("could not read job %s to check its lease", self.job_id[:8], exc_info=True)
+            return None
+        if not rows:
+            return False
+        row = rows[0]
+        get = row.get if isinstance(row, dict) else lambda k: getattr(row, k, None)
+        if get("claimed_at") is None:
+            return False
+        # every claim stamps a token: a token that is not this copy's (or one where this copy had none) is another
+        # claim (K1b, M4: a token-less copy used to read every later claim as its own)
+        return get("claimed_by") == self.claimed_by
+
+    async def not_ours(self, why: str) -> bool:
+        """The queue says this copy no longer holds its job. When the reaper only DEAD-LETTERED it (a false "dead" on
+        the last attempt), no other copy can ever own the run: take the job back and keep going, so the run is
+        finished and its terminal status written (K1b, B1). Otherwise mark the lease lost (which stops the copy).
+        Returns True when the copy keeps the job."""
+        from matrx_graph.workers.queue import take_back_false_dead_letter
+
+        if self.settling:
+            return True
+        try:
+            kept = await take_back_false_dead_letter(
+                self.job_id, claimed_by=self.claimed_by, visibility_timeout_seconds=self.visibility_timeout)
+        except Exception:  # noqa: BLE001 — unknown: the copy stops (the safe side for a run another copy may own)
+            log.warning("job %s: could not try to take back a dead-lettered job", self.job_id[:8], exc_info=True)
+            kept = False
+        if not kept:
+            self.mark_lost(why)
+        return kept
+
+    async def check(self) -> None:
+        """Raise :class:`JobLeaseLost` when this copy no longer holds its job; read the row at most every
+        ``look_every`` seconds."""
+        if not self.lost and time.monotonic() - self._looked >= self.look_every:
+            self._looked = time.monotonic()
+            if await self.still_held() is False:
+                await self.not_ours("the job row is claimed by another copy or was requeued")
+        if self.lost:
+            raise JobLeaseLost(f"job {self.job_id[:8]} lease lost: {self.why}")
+
+
+def _running_task() -> asyncio.Task | None:
+    try:
+        return asyncio.current_task()
+    except RuntimeError:
+        return None
+
+
+_current: ContextVar[JobLease | None] = ContextVar("matrx_graph_job_lease", default=None)
+
+
+def current() -> JobLease | None:
+    """The lease of the job this task executes; None outside a queued job (an inline run has no job lease)."""
+    return _current.get()
+
+
+async def check() -> None:
+    """Before each item of many-item work: raises :class:`JobLeaseLost` once this copy's job lease is gone."""
+    lease = _current.get()
+    if lease is not None:
+        await lease.check()
+
+
+def lost() -> bool:
+    lease = _current.get()
+    return bool(lease is not None and lease.lost)
+
+
+@contextlib.contextmanager
+def holding(lease: JobLease) -> Iterator[JobLease]:
+    token = _current.set(lease)
+    try:
+        yield lease
+    finally:
+        _current.reset(token)
+
+
+__all__ = ["JobLease", "JobLeaseLost", "check", "current", "holding", "lost"]

matrx_graph/workers/queue.py

@@ -707,19 +707,89 @@ async def renew_lease(
     return result.rows_affected > 0
 
 
+async def take_back_false_dead_letter(
+    job_id: str,
+    *,
+    claimed_by: str | None,
+    visibility_timeout_seconds: int = 600,
+) -> bool:
+    """A live copy whose job the reaper DEAD-LETTERED (a false "dead": an outage outlasted the lease on the job's last
+    attempt) takes the job back, so it finishes the run and writes its terminal status (lane K1b, B1, 2026-10-21).
+
+    Without this the live copy read "not ours", cancelled itself and skipped its terminal write, and no other copy would
+    ever claim a dead-lettered job: the run stayed non-terminal (stranded) for every job kind. A dead-lettered job has
+    no other owner, so taking it back can never run two copies. Compare-and-set: only a row that is still unclaimed,
+    still dead-lettered BY THE REAPER (``last_error.type == 'lease_expired'``) and still parked in the far future
+    (an operator's retry moves ``visible_at`` back) is taken; anything else answers False and the copy stops.
+    """
+    from matrx_graph.db._registry import get_model
+
+    model = get_model("WfJob")
+    try:
+        rows = await model.filter(id=str(UUID(job_id))).all(use_cache=False)
+    except Exception:  # noqa: BLE001 — unreadable: not taken back (the caller decides on the renewal's answer)
+        log.warning("could not read job %s to see whether the reaper dead-lettered it", job_id[:8], exc_info=True)
+        return False
+    if not rows:
+        return False
+    row = rows[0]
+    get = row.get if isinstance(row, dict) else lambda k: getattr(row, k, None)
+    note = _parse_json(get("last_error")) or {}
+    if get("claimed_at") is not None or not isinstance(note, dict):
+        return False
+    if note.get("queue_state") != "dead_letter" or note.get("type") != "lease_expired":
+        return False
+    now = datetime.now(UTC)
+    taken = (
+        await model.filter(
+            id=str(UUID(job_id)),
+            claimed_at__isnull=True,
+            visible_at__gt=now + timedelta(days=_FAR_FUTURE_DAYS // 2),
+        ).update(
+            claimed_at=now,
+            claimed_by=claimed_by,
+            visible_at=now + timedelta(seconds=visibility_timeout_seconds),
+            last_error={
+                "queue_state": "in_progress",
+                "type": "false_dead_taken_back",
+                "message": "the reaper dead-lettered a job its worker was still running (an outage outlasted the "
+                "lease); the live copy took it back to finish the run",
+            },
+        )
+    ).updated_rows
+    if taken:
+        log.warning("job %s: the reaper dead-lettered it while this worker still ran it; taken back to finish the run",
+                    job_id[:8])
+    return bool(taken)
+
+
 async def reap_expired_leases(*, grace_seconds: int = 120) -> int:
     """Requeue jobs whose lease expired ≥ ``grace_seconds`` ago (audit P1-14).
 
     A worker that dies mid-job leaves ``claimed_at`` set forever — only
     complete/fail/release null it — so the job is orphaned: never re-run,
     never dead-lettered, invisible to claim_one. This reaper is the one
-    mechanism that recovers those rows. Safe against false positives
-    because WorkerRunner renews the lease every lease/3 seconds while a
-    job executes: an expired-by-more-than-grace lease means the renewal
-    loop (and therefore the worker) is dead.
+    mechanism that recovers those rows. WorkerRunner renews the lease every
+    lease/3 seconds while a job executes, so an expired-by-more-than-grace
+    lease USUALLY means the worker is dead — but not always: a database
+    outage longer than the lease fails every renewal of every live worker
+    at once (live 2026-10-21, 15:49–15:59Z: job c1d89868 was requeued from
+    a worker that was alive).
+
+    Decision (lane K1, 2026-10-21): the reaper does NOT try to tell an outage
+    from a death (an outage-detection rule would itself have to read a store
+    that just failed, and would delay real recovery); instead a false "dead"
+    is made HARMLESS by the copies themselves: a copy whose renewal answers
+    "not ours" marks its lease lost, is cancelled and skips its terminal
+    write (``runner._renew_loop`` + ``job_lease``); many-item work checks the
+    lease before each item and each store attempt; and a slot held for the
+    run (``work_claims.hold_work_slot(holder=run_id)``) is taken over by the
+    new copy at once, so the run never waits on its own earlier copy.
 
     Jobs with attempts left are requeued (claim_one re-claims and bumps
-    ``attempt``); exhausted jobs are dead-lettered with the same far-future
+    ``attempt``); exhausted jobs are dead-lettered (a live copy whose job was
+    falsely dead-lettered takes it back, ``take_back_false_dead_letter``, lane
+    K1b, so its run is never stranded) with the same far-future
     visible_at shape ``fail()`` uses. Returns how many rows were touched.
     """
     from matrx_graph.db._registry import get_model

matrx_graph/workers/runner.py

@@ -30,6 +30,7 @@ from typing import TYPE_CHECKING, Any
 
 from matrx_utils import capture_error
 
+from matrx_graph.workers import job_lease
 from matrx_graph.workers.capability import (
     JobNeedsCapability,
     mark_run_awaiting_capability,
@@ -320,14 +321,36 @@ class WorkerRunner:
         await asyncio.gather(*still_running, return_exceptions=True)
 
     async def _process(self, job: Job) -> None:
-        renewer = asyncio.create_task(self._renew_loop(job.job_id, claimed_by=job.claimed_by))
+        # The lease this copy holds, published to the work (``job_lease.check`` before each item) and marked lost by
+        # the renewal the moment the queue says the job is no longer ours (lane K1, 2026-10-21).
+        lease = job_lease.JobLease(
+            job_id=job.job_id,
+            run_id=job.run_id,
+            claimed_by=job.claimed_by,
+            look_every=max(self._visibility_timeout // 3, 10) / 10,
+            owner=asyncio.current_task(),
+            visibility_timeout=self._visibility_timeout,
+            # the job's attempt: every re-claim bumps it, so a later copy's fencing token is always greater (K1b, B5)
+            generation=float(job.attempt or 0),
+        )
+        renewer = asyncio.create_task(
+            self._renew_loop(job.job_id, claimed_by=job.claimed_by, lease=lease, owner=asyncio.current_task())
+        )
         try:
             app = await self._app_factory(job)
-            await self._run_job(job, app)
+            with job_lease.holding(lease):
+                await self._run_job(job, app)
+            lease.settling = True  # a renewal answering "not ours" after this settles is not a loss (K1b, M1)
             await complete(job.job_id, claimed_by=job.claimed_by)
         except JobNeedsCapability as need:
             await hand_back_for_capability(job, need)
         except asyncio.CancelledError as exc:
+            if lease.lost:
+                # A copy whose lease was reaped stops here and settles NOTHING: the job row and the run belong to the
+                # copy that holds the job now (its fail() would be fenced anyway; this keeps the log honest).
+                log.warning("job %s: this copy stopped after losing its lease (%s); the current holder owns the run",
+                            job.job_id[:8], lease.why)
+                raise
             try:
                 await asyncio.shield(
                     fail(
@@ -396,7 +419,14 @@ class WorkerRunner:
             except asyncio.CancelledError:
                 pass
 
-    async def _renew_loop(self, job_id: str, *, claimed_by: str | None) -> None:
+    async def _renew_loop(
+        self,
+        job_id: str,
+        *,
+        claimed_by: str | None,
+        lease: job_lease.JobLease | None = None,
+        owner: asyncio.Task[Any] | None = None,
+    ) -> None:
         # Lease/3 keeps two renewal misses of headroom before expiry; the
         # reaper adds its own grace on top, so only a dead worker is reaped.
         interval = max(self._visibility_timeout // 3, 10)
@@ -421,16 +451,27 @@ class WorkerRunner:
                 )
                 continue
             if not still_ours:
-                # Reaped or completed elsewhere. Don't cancel the in-flight
-                # run — killing a paid LLM call mid-flight loses the spend
-                # AND the result; the run-store terminal-status guards
-                # arbitrate if a second worker re-claims and both finish.
+                # Reaped (a false "dead" after a database outage looks exactly like this) or settled elsewhere.
+                # Arman, 2026-10-21: a copy that loses its job lease STOPS ITSELF. Finishing "anyway" (the old rule)
+                # left two copies of one run alive: the old one, hung on a store call, kept its organization import
+                # slot and the restarted copy waited on itself (run 8749d414). Another copy may own the run now, so
+                # this one stops: the lease is marked lost (many-item work checks it before each item), the job
+                # task is cancelled, and the host skips its terminal write.
+                # Lane K1b, B1: when the reaper only DEAD-LETTERED the job (a false "dead" on its last attempt), no
+                # other copy can ever own the run, so this copy takes the job back and finishes it (stopping would
+                # strand the run: nothing would ever write its terminal status).
+                if lease is not None:
+                    if lease.settling:
+                        return
+                    if await lease.not_ours("the queue reaped or settled the job"):
+                        continue
                 log.error(
                     "job %s lease is no longer ours (reaped or deleted) — "
-                    "finishing the in-flight run anyway; status guards "
-                    "arbitrate duplicate terminal writes",
+                    "this copy stops; the current holder owns the run",
                     job_id,
                 )
+                if owner is not None and not owner.done():
+                    owner.cancel()
                 return
 
 

{matrx_graph-0.2.330.dist-info → matrx_graph-0.2.332.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: matrx-graph
-Version: 0.2.330
+Version: 0.2.332
 Summary: Channel-based, super-step workflow orchestration engine for the Matrx ecosystem
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current
@@ -19,7 +19,7 @@ Requires-Python: >=3.13
 Requires-Dist: croniter>=2.0
 Requires-Dist: httpx>=0.27
 Requires-Dist: jsonschema>=4.20
-Requires-Dist: matrx-connect>=0.1.235
+Requires-Dist: matrx-connect>=0.1.236
 Requires-Dist: matrx-utils>=2.0.83
 Requires-Dist: pydantic>=2.12
 Requires-Dist: pyyaml>=6.0

{matrx_graph-0.2.330.dist-info → matrx_graph-0.2.332.dist-info}/RECORD

@@ -1,7 +1,7 @@
 matrx_graph/__init__.py,sha256=6-2kvFL4Q1xbj21ZOqIGAvErgwYTStenEqzyrMdmY5o,7803
 matrx_graph/candidate.py,sha256=nWyD4FkELyT0aGAJG13OkcGjyLlqO_M7Jct4IIe8oJk,10497
 matrx_graph/checks.py,sha256=F5mDzoDAj5ArNuuehN88DntSk_iIWAq1WlBaMTXIUFE,12187
-matrx_graph/compile.py,sha256=Zm81mHPxOEyAyTVjp86pTZvOfa0h7Ba08IEWTy6WlQA,25630
+matrx_graph/compile.py,sha256=ewfpjIZtNBPvl0Kj_1jYx12KA1cB9NekdmuWWw_SOqA,27436
 matrx_graph/composition.py,sha256=PX3XAnH1v6jOJ1kYGVlJH3hzqQFTEtxR2uNzRObtdWA,9069
 matrx_graph/contract_kinds.py,sha256=xOlhlbwijClxeDzj12mxpbWqMc6k8bsz6UyHcjROqsc,11818
 matrx_graph/decision_spec.py,sha256=5QGYYa0xLqXidHbILwTjTkE9WbZzyKxLjz25CYVr2Kw,15764
@@ -15,7 +15,7 @@ matrx_graph/input_bindings.py,sha256=mweDa6SKo-I9PU7qJIz_jgruciUhtp7bV1KBBjla0TU
 matrx_graph/kind_control_keys.py,sha256=35LcweBvfqU2pz6Eu8vFn7kM-vh9FK1B4QPArZ76WS0,9444
 matrx_graph/kind_fields.py,sha256=6F9jmJNz9-104FSAqIwx8xcy5HqeeYxedgyoyPJf0mk,21123
 matrx_graph/kinds.py,sha256=Uh1KmVBL46EpC4ojKCq7eOom8SVleEROUmRYzz27BU4,37751
-matrx_graph/knobs.py,sha256=5C6wOykirXW6JuoCB1Zp8W6SjO1ga_aoAuvBa0D--Rc,21388
+matrx_graph/knobs.py,sha256=tZMlF7YV3GZw3liccEnFdOkYBbvOeFxgc25oetdEOGo,22463
 matrx_graph/mapping_sources.py,sha256=8VGTbPmib7uyh2L8ZnjsJa6h57lKgz0JOpJ9fmkcYxU,3910
 matrx_graph/overview.py,sha256=3_hxx3dFZQtZkVE-9TcdU8udK4GWndnHKKPNU__aTws,24406
 matrx_graph/patch.py,sha256=DCd7MioacEfcaPxuvrDZS9uxfUE-3EKN7FBL4vLc9y0,10675
@@ -147,7 +147,7 @@ matrx_graph/executor/resume_chain.py,sha256=YJv8P8Ca1wSAsUEjeii3Tg6Fgoy5N9Tf2Gs4
 matrx_graph/executor/resume_preflight.py,sha256=vz9_01QINvgwK9u2y7NcF7B9J2Yv7K1jHalRuZD7siw,11977
 matrx_graph/executor/review_flags.py,sha256=hMR8tPrSd_-vgsLiTc82MmEaIx-sMkC5yf53zIbhh8M,5542
 matrx_graph/executor/run_stop.py,sha256=HpdGE83MabnU_4rnf_Vn8zRyk5LBF3qbZQ2Ouvc36hY,3004
-matrx_graph/executor/scheduler.py,sha256=nqpaj7ZllrDf3twN8SHcl8BlXm4Ak8Ht593zJkDXwvM,322508
+matrx_graph/executor/scheduler.py,sha256=1ezi7TYpY-tLSk-dsGjuk38WIK3ivdPw8wUDFVCdLfw,323193
 matrx_graph/executor/schema_validation.py,sha256=X2CbDX7_vZGBtcw_L0BQcjDD-KEAxnGA8ruhm9Qp5CI,5250
 matrx_graph/nodes/__init__.py,sha256=ikOsSIu8aaSYAZXa_Uks-RW_BkIjwiO1TwTptLom0bY,10118
 matrx_graph/nodes/_sandbox.py,sha256=Uizb2QIbW9MgZ_4MtWoJlmjR1mMXZNb9oMYGhXNXeQ0,10517
@@ -229,7 +229,7 @@ matrx_graph/observability/__init__.py,sha256=h8iYxtLuNQg2CFepjbVzvx_dLOeA-iG3H-1
 matrx_graph/types/__init__.py,sha256=DsU_RAigVdPhLwzRDFU9M0bZ10uhYOhVveQOBh2vPr8,2335
 matrx_graph/types/canonical_payloads.py,sha256=c_R_EdjbXUyXcDUxMYTbi66oFJzzyTFmTIAKdQIB2gw,4925
 matrx_graph/types/channel.py,sha256=Tr0RjeUkhaXwW-DYnEhloWV2Zw-xbT6VmRbFqo6u0Zg,2385
-matrx_graph/types/compiled.py,sha256=GMjYLuPRde0f8kkmzegkokW9sbHB51jr24VK-JDhbf8,11164
+matrx_graph/types/compiled.py,sha256=9oCymfDLXEmxrezmfzTPV3EHK8ELtjdAG_u_cjXwsYM,12048
 matrx_graph/types/context.py,sha256=4AamW22VTE3K3IhGONg4gR3q8M9hrbMC9sFiMbYQU7M,18213
 matrx_graph/types/definition.py,sha256=noLq0ThxQQ1_N8ucLPXnXHXHmnyeKDwvSRcbSyNn2QU,12935
 matrx_graph/types/deliverables.py,sha256=-3kOWPSAKbSmqEFBhcVjYpjJ75mCycVrhOOSo3gEBOM,12550
@@ -246,14 +246,15 @@ matrx_graph/types/send.py,sha256=sKE_moS4uTB4P3L02Q7uKj0WhTzABm3vTk2o54EpoPA,254
 matrx_graph/types/usl.py,sha256=-zXXiHZpxpv7V7n_xgdcnY2l8XlWoM9ZUQNkwHYmqiQ,3376
 matrx_graph/types/value_objects.py,sha256=ylVpyj-_YQ2xhMAHVW2AdRo5yqzgXLajGUuaIZhJFQY,1295
 matrx_graph/types/variables.py,sha256=pCECEaDP3y2qwZkIlhpICRucFTMHRLW8wU3zXQBooFY,10896
-matrx_graph/workers/FEATURE.md,sha256=Hd6OFiBVG0HkzFt5WgJkTiDF6idX4t5CFYWLIRXqjME,16864
+matrx_graph/workers/FEATURE.md,sha256=yZXybUsAbUkYDwXBNnzoJXGCedQmVK_BQCZ3AXDlogk,18502
 matrx_graph/workers/__init__.py,sha256=G8e24ryKLafH_NggiiGIb9EZ7YparOJ5DaBOw0bbPv0,2888
 matrx_graph/workers/capability.py,sha256=9kFjZCfuie5GkevLLTBJVJ3m9NeeS36RJhbjK0r86fw,11058
 matrx_graph/workers/cron_watcher.py,sha256=JVzUJ7Wmtz9PPwUtkxnSyqkheK3nEM-7lHUlecdazYY,3988
 matrx_graph/workers/event_watcher.py,sha256=DYk4RgapiJJDyKY8zirBpp3HAlqi7BWExf4rWW97FHY,6263
-matrx_graph/workers/queue.py,sha256=pYt9Xl_eqJi6m_7-hN77I_XXR3UWMb-e33aS6PU8k6w,28756
-matrx_graph/workers/runner.py,sha256=Dn5WPfKT4iD_dSbKnfBE5O5pjUA24lnTdmJvOJWurZU,21479
+matrx_graph/workers/job_lease.py,sha256=pYZidqPB4hgHa8KKZqSJDGetsW7Wm_aVwSXM0ycrrfY,7213
+matrx_graph/workers/queue.py,sha256=GGzuJju1DvrZFPbfvGQEMuWay-2m18NzZPNlWligS4Q,32413
+matrx_graph/workers/runner.py,sha256=6ALoNst6LY9pK-8I7r0QPLjdKpSfyimp5JwFb1PFdmI,23946
 matrx_graph/workers/sleep_wake.py,sha256=1V-opM2zYtuN3n4ez85C4trYHoEapVwz8F7_MAex8HI,5970
-matrx_graph-0.2.330.dist-info/METADATA,sha256=E5NDq1pV9sKxeZvfKiVkcCAWoIXPDDUoS91bvYscY6c,5480
-matrx_graph-0.2.330.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-matrx_graph-0.2.330.dist-info/RECORD,,
+matrx_graph-0.2.332.dist-info/METADATA,sha256=Xtv8QD6KJcjVl9GHAwEdySTwocg45udFubgG2r2VWYk,5480
+matrx_graph-0.2.332.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+matrx_graph-0.2.332.dist-info/RECORD,,