flashruntime 0.6.0__tar.gz → 0.6.1__tar.gz

{flashruntime-0.6.0/flashruntime.egg-info → flashruntime-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashruntime
-Version: 0.6.0
+Version: 0.6.1
 Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
 Author: Zolli Labs
 License-Expression: Apache-2.0

{flashruntime-0.6.0 → flashruntime-0.6.1}/flashruntime/leases/manager.py

@@ -18,9 +18,26 @@ State transitions (task):
 
     PENDING ──claim──▶ LEASED ──complete (first valid)──▶ COMPLETED
        ▲                 │
-       └──sweep expiry / fail (attempts left)──┘
-    LEASED ──fail/expiry with no attempts left──▶ FAILED
+       └──sweep expiry / fail / revoked-silent / force_expire (budget left)──┘
+    LEASED ──fail, failure budget spent──▶ FAILED
+    LEASED ──expiry, expiry budget spent──▶ FAILED
     any non-terminal ──cancel──▶ CANCELLED
+
+Failure and expiry spend separate budgets (audit C12): a worker that
+reports N real failures exhausts at `max_attempts`; a lease that lapses
+unclaimed N times (pure churn — no worker ever ran it) exhausts at the
+separate, more generous `max_expiries`. `attempts_used` still numbers
+attempts but decides nothing terminal on its own.
+
+Two things can end a lease early besides its deadline (audit C11/§5.1).
+The *progress watchdog*: a heartbeat proves a process is alive, not that
+work is happening, so a holder silent past `spec.max_silent_seconds` has
+its next renewal refused and the task requeued — `note_progress` is what
+resets that clock. It is opt-in (`max_silent_seconds` defaults to 0, which
+disables it): a task that reports no progress signal is not thereby stuck,
+so whoever expands a job arms the window only for work that does report.
+And `force_expire`, the operator's manual version of the same revocation.
+Both spend the expiry budget, because both are churn.
 """
 
 from __future__ import annotations
@@ -80,6 +97,11 @@ class LeaseManager:
         record.state = TaskState.CANCELLED
         record.active_lease = None
         self._store.save(record)
+        # Emitted only on a real transition (audit §5.1): the ledger is the
+        # only place a cancellation was ever visible, and a task that
+        # vanished from the queue with no event read as a coordinator bug.
+        # An idempotent re-cancel transitions nothing, so it says nothing.
+        self._emit(EventType.TASK_CANCELLED, job_id, task_id, now=now)
 
     # -- worker side --------------------------------------------------------
 
@@ -125,6 +147,7 @@ class LeaseManager:
         if record is None:
             return None
         record.attempts_used += 1
+        record.last_progress_at = now
         lease = Lease(
             lease_id=f"ls-{uuid.uuid4().hex[:12]}",
             task_id=record.spec.task_id,
@@ -132,6 +155,9 @@ class LeaseManager:
             node_id=node_id,
             attempt_number=record.attempts_used,
             deadline=now + timedelta(seconds=record.spec.lease_seconds),
+            # On the wire so the holder can pick its own heartbeat interval
+            # from the real window instead of guessing one (audit §5.1).
+            lease_seconds=record.spec.lease_seconds,
             payload=record.spec.payload,
         )
         record.state = TaskState.LEASED
@@ -151,9 +177,38 @@ class LeaseManager:
     def heartbeat(self, lease_id: str, now: datetime | None = None) -> Lease:
         """Renew a live lease (extends the deadline by the task's lease
         window). Refused for unknown/expired/superseded leases — a worker
-        whose heartbeat is refused must stop working on the task."""
+        whose heartbeat is refused must stop working on the task.
+
+        Also refused, and the lease revoked, when the holder has been alive
+        but *silent* past `spec.max_silent_seconds` (audit C11). A heartbeat
+        proves a process is running; only `note_progress` proves the work
+        is. Without this, a stuck worker whose heartbeat thread survived
+        renewed its lease forever and the task never came back — the zombie
+        lease. Opt-in: `max_silent_seconds` defaults to 0, which disables
+        the watchdog, so it never fires on a workload that reports nothing.
+        """
         now = now or _utcnow()
         record, lease = self._require_live_lease(lease_id, now)
+        # After the liveness check on purpose: an already-dead lease is
+        # refused as dead ("no longer live"), not as silent. Only a lease
+        # that is still good takes the revocation path below.
+        window = record.spec.max_silent_seconds
+        baseline = record.last_progress_at
+        if window > 0 and baseline is not None and (now - baseline).total_seconds() > window:
+            self._emit(
+                EventType.LEASE_RENEWAL_REFUSED,
+                lease.job_id,
+                lease.task_id,
+                node_id=lease.node_id,
+                detail=f"no progress for {(now - baseline).total_seconds():.0f}s "
+                f"(budget {window:.0f}s) — lease revoked, task requeued",
+                now=now,
+            )
+            # Counter bumped at the call site, like fail()/sweep() (audit
+            # C12): a revoked lease is churn, so it spends the expiry budget.
+            record.expiries_seen += 1
+            self._release(record, now, EventType.TASK_REQUEUED, budget="expiry")
+            raise LeaseError(f"lease {lease_id} revoked: no progress within {window:.0f}s")
         renewed = lease.model_copy(
             update={"deadline": now + timedelta(seconds=record.spec.lease_seconds)}
         )
@@ -178,6 +233,13 @@ class LeaseManager:
         """
         now = now or _utcnow()
         record, lease = self._find_lease(lease_id)
+        if record.state == TaskState.COMPLETED and record.accepted_attempt_id == lease.lease_id:
+            # The winner asking again (its 200 was lost to a timeout, and it
+            # retried). Answering False told the one machine that actually
+            # did the work that it had lost — audit §2 "retried complete".
+            # Idempotent means same answer, and no second ACCEPTED event:
+            # the ledger already records the commit that happened.
+            return True
         if record.state == TaskState.COMPLETED or not self._is_live(record, lease, now):
             self._emit(
                 EventType.TASK_COMMIT_REJECTED,
@@ -202,11 +264,50 @@ class LeaseManager:
         )
         return True
 
+    def note_progress(
+        self, job_id: str, task_id: str, now: datetime | None = None
+    ) -> None:
+        """Record that the holder made *observable* progress on this task.
+
+        The other half of the C11 watchdog: heartbeats keep the deadline
+        alive, this keeps the silence budget alive. Callers are whatever the
+        workload reports through — a metric, a step counter, a partial
+        artifact write.
+
+        A silent no-op for an unknown task or one that is not LEASED: the
+        relay can always race an expiry, and a progress note arriving one
+        tick after the sweep took the lease away must not resurrect
+        anything, nor error at a reporter that did nothing wrong.
+        """
+        record = self._store.get(job_id, task_id)
+        if record is None or record.state != TaskState.LEASED:
+            return
+        record.last_progress_at = now or _utcnow()
+        self._store.save(record)
+
     def fail(self, lease_id: str, reason: str, now: datetime | None = None) -> None:
         """A worker reports its attempt failed. The task requeues while
-        attempts remain, otherwise it is FAILED for good."""
+        attempts remain, otherwise it is FAILED for good.
+
+        A report against a *known but dead* lease is recorded and ignored,
+        not raised (audit §5.1): the attempt really did fail, and that is
+        evidence worth keeping, but the task has already moved on — spending
+        its failure budget or touching its state would punish whoever holds
+        it now. Only a lease that never existed is an error.
+        """
         now = now or _utcnow()
-        record, lease = self._require_live_lease(lease_id, now)
+        record, lease = self._find_lease(lease_id)
+        if not self._is_live(record, lease, now):
+            self._emit(
+                EventType.TASK_ATTEMPT_FAILED,
+                lease.job_id,
+                lease.task_id,
+                node_id=lease.node_id,
+                detail=f"(late report — lease no longer live) {reason}",
+                now=now,
+            )
+            return
+        record.failures_seen += 1
         self._emit(
             EventType.TASK_ATTEMPT_FAILED,
             lease.job_id,
@@ -215,7 +316,7 @@ class LeaseManager:
             detail=reason,
             now=now,
         )
-        self._release(record, now, requeue_event=EventType.TASK_REQUEUED)
+        self._release(record, now, EventType.TASK_REQUEUED, budget="failure")
 
     # -- coordinator side ---------------------------------------------------
 
@@ -230,6 +331,7 @@ class LeaseManager:
             if lease is None or lease.deadline > now:
                 continue
             expired += 1
+            record.expiries_seen += 1
             self._emit(
                 EventType.LEASE_EXPIRED,
                 lease.job_id,
@@ -238,9 +340,37 @@ class LeaseManager:
                 detail=f"attempt {lease.attempt_number} heartbeat deadline passed",
                 now=now,
             )
-            self._release(record, now, requeue_event=EventType.TASK_REQUEUED)
+            self._release(record, now, EventType.TASK_REQUEUED, budget="expiry")
         return expired
 
+    def force_expire(
+        self, job_id: str, task_id: str, reason: str, now: datetime | None = None
+    ) -> bool:
+        """Revoke a task's live lease now, by operator decision.
+
+        The manual lever behind the automatic ones (audit §5.1): when a
+        machine is known-stuck but its deadline is minutes away, waiting is
+        the only thing an operator could do. Returns False when the task is
+        not LEASED — there is nothing to take away, and re-running the same
+        command is then a no-op rather than an error.
+        """
+        now = now or _utcnow()
+        record = self._require(job_id, task_id)
+        if record.state != TaskState.LEASED:
+            return False
+        lease = record.active_lease
+        self._emit(
+            EventType.LEASE_EXPIRED,
+            job_id,
+            task_id,
+            node_id=lease.node_id if lease is not None else None,
+            detail=f"force-expired: {reason}",
+            now=now,
+        )
+        record.expiries_seen += 1
+        self._release(record, now, EventType.TASK_REQUEUED, budget="expiry")
+        return True
+
     def job_state(self, job_id: str) -> dict[str, int]:
         """Task counts by state — the number the dashboard shows."""
         counts: dict[str, int] = {}
@@ -260,6 +390,20 @@ class LeaseManager:
             return None
         return lease
 
+    def lease_is_live(self, lease_id: str, now: datetime | None = None) -> bool:
+        """Is this lease still the good one? A question, so it answers.
+
+        Unlike `_find_lease`, an unknown id is False rather than an error:
+        callers ask this to decide whether to keep going, and "I have never
+        heard of that lease" is a perfectly good "stop".
+        """
+        now = now if now is not None else _utcnow()
+        try:
+            record, lease = self._find_lease(lease_id)
+        except LeaseError:
+            return False
+        return self._is_live(record, lease, now)
+
     def live_leases_for_node(
         self, node_id: str, now: datetime | None = None
     ) -> set[tuple[str, str]]:
@@ -294,15 +438,42 @@ class LeaseManager:
 
     # -- internals ----------------------------------------------------------
 
-    def _release(self, record: TaskRecord, now: datetime, requeue_event: EventType) -> None:
+    def _release(
+        self,
+        record: TaskRecord,
+        now: datetime,
+        requeue_event: EventType,
+        *,
+        budget: str,
+    ) -> None:
+        """Return a task to PENDING, or exhaust it — against the budget the
+        caller spent (audit C12).
+
+        `"failure"` and `"expiry"` are separate budgets on purpose: a
+        worker that keeps reporting real failures is different evidence
+        from a lease that keeps lapsing unclaimed (pure churn — no worker
+        ever ran it), so each is checked against its own counter
+        (`spec.max_attempts` / `spec.max_expiries`) and neither spends the
+        other's.
+        """
         record.active_lease = None
-        if record.attempts_used >= record.spec.max_attempts:
+        exhausted = (
+            record.failures_seen >= record.spec.max_attempts
+            if budget == "failure"
+            else record.expiries_seen >= record.spec.max_expiries
+        )
+        if exhausted:
             record.state = TaskState.FAILED
+            detail = (
+                f"all {record.spec.max_attempts} attempts failed"
+                if budget == "failure"
+                else f"lease expired {record.expiries_seen} times without completing"
+            )
             self._emit(
                 EventType.TASK_EXHAUSTED,
                 record.spec.job_id,
                 record.spec.task_id,
-                detail=f"all {record.spec.max_attempts} attempts used",
+                detail=detail,
                 now=now,
             )
         else:

{flashruntime-0.6.0 → flashruntime-0.6.1}/flashruntime/leases/sqlite_store.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 
 import json
 import sqlite3
+from datetime import datetime
 from pathlib import Path
 
 from flashruntime.leases.store import TaskRecord
@@ -49,6 +50,7 @@ class SqliteLeaseStore:
         self._conn = sqlite3.connect(str(path), check_same_thread=False)
         self._migrate()                 # legacy single-PK table → composite
         self._conn.executescript(_SCHEMA)
+        self._ensure_runtime_column()   # additive: budget-split counters (audit C12)
         self._cache: dict[tuple[str, str], TaskRecord] = {}
         self._seq = 0
         self._load()
@@ -79,15 +81,27 @@ class SqliteLeaseStore:
         )
         self._conn.commit()
 
+    def _ensure_runtime_column(self) -> None:
+        """Additive migration (audit C12): one nullable JSON column carries
+        the budget-split counters (`failures_seen`, `expiries_seen`,
+        `last_progress_at`). A plain `ADD COLUMN` — unlike the composite-key
+        rebuild above, no rewrite is needed since every existing row simply
+        reads back NULL, and `_load` treats that as the pre-upgrade
+        zeros/None defaults rather than failing to load."""
+        cols = {row[1] for row in self._conn.execute("PRAGMA table_info(lease_tasks)").fetchall()}
+        if "runtime_json" not in cols:
+            self._conn.execute("ALTER TABLE lease_tasks ADD COLUMN runtime_json TEXT")
+            self._conn.commit()
+
     # -- rehydration ---------------------------------------------------------
 
     def _load(self) -> None:
         rows = self._conn.execute(
             "SELECT spec_json, state, attempts_used, active_lease_json,"
-            " accepted_attempt_id, lease_history_json, seq"
+            " accepted_attempt_id, lease_history_json, runtime_json, seq"
             " FROM lease_tasks ORDER BY seq"
         ).fetchall()
-        for spec_json, state, attempts, lease_json, accepted, history_json, seq in rows:
+        for spec_json, state, attempts, lease_json, accepted, history_json, runtime_json, seq in rows:
             record = TaskRecord(TaskSpec.model_validate_json(spec_json))
             record.state = TaskState(state)
             record.attempts_used = attempts
@@ -97,6 +111,15 @@ class SqliteLeaseStore:
                 lid: Lease.model_validate(raw)
                 for lid, raw in json.loads(history_json).items()
             }
+            # `.get(...)` defaults: a row written before this column existed
+            # (or with it explicitly NULL) loads as zeros/None, not an error.
+            runtime = json.loads(runtime_json) if runtime_json else {}
+            record.failures_seen = runtime.get("failures_seen", 0)
+            record.expiries_seen = runtime.get("expiries_seen", 0)
+            last_progress_at = runtime.get("last_progress_at")
+            record.last_progress_at = (
+                datetime.fromisoformat(last_progress_at) if last_progress_at else None
+            )
             self._cache[(record.spec.job_id, record.spec.task_id)] = record
             self._seq = max(self._seq, seq or 0)
 
@@ -140,18 +163,28 @@ class SqliteLeaseStore:
         history = json.dumps(
             {lid: json.loads(lease.model_dump_json()) for lid, lease in record.lease_history.items()}
         )
+        runtime = json.dumps(
+            {
+                "failures_seen": record.failures_seen,
+                "expiries_seen": record.expiries_seen,
+                "last_progress_at": (
+                    record.last_progress_at.isoformat() if record.last_progress_at else None
+                ),
+            }
+        )
         self._conn.execute(
             "INSERT INTO lease_tasks"
             " (task_id, job_id, spec_json, state, attempts_used, active_lease_json,"
-            "  accepted_attempt_id, lease_history_json, seq)"
-            " VALUES (?, ?, ?, ?, ?, ?, ?, ?,"
+            "  accepted_attempt_id, lease_history_json, runtime_json, seq)"
+            " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,"
             "   COALESCE(?, (SELECT seq FROM lease_tasks"
             "                 WHERE job_id = ? AND task_id = ?)))"
             " ON CONFLICT(job_id, task_id) DO UPDATE SET"
             "  state=excluded.state, attempts_used=excluded.attempts_used,"
             "  active_lease_json=excluded.active_lease_json,"
             "  accepted_attempt_id=excluded.accepted_attempt_id,"
-            "  lease_history_json=excluded.lease_history_json",
+            "  lease_history_json=excluded.lease_history_json,"
+            "  runtime_json=excluded.runtime_json",
             (
                 record.spec.task_id,
                 record.spec.job_id,
@@ -161,6 +194,7 @@ class SqliteLeaseStore:
                 record.active_lease.model_dump_json() if record.active_lease else None,
                 record.accepted_attempt_id,
                 history,
+                runtime,
                 seq,
                 record.spec.job_id,
                 record.spec.task_id,

{flashruntime-0.6.0 → flashruntime-0.6.1}/flashruntime/leases/store.py

@@ -11,6 +11,7 @@ manager, never in a store, so every backend behaves identically.
 
 from __future__ import annotations
 
+from datetime import datetime
 from typing import Protocol
 
 from flashruntime.protocol.v1alpha1 import Lease, TaskSpec, TaskState
@@ -22,9 +23,28 @@ class TaskRecord:
     `lease_history` keeps every lease ever issued for the task so a late
     commit from a long-dead worker can be *identified and rejected* rather
     than erroring as unknown — rejected duplicates are recovery evidence.
+
+    `failures_seen` and `expiries_seen` (audit C12) are separate budgets:
+    a worker that reports a failure spends `failures_seen` against
+    `spec.max_attempts`; a lease that lapses unclaimed spends
+    `expiries_seen` against `spec.max_expiries`. `attempts_used` still
+    numbers attempts (and drives `Lease.attempt_number`), but no longer by
+    itself decides anything terminal. `last_progress_at` is stamped on
+    every claim — the last time some worker showed up for this task at
+    all, regardless of how that attempt ended.
     """
 
-    __slots__ = ("spec", "state", "attempts_used", "active_lease", "accepted_attempt_id", "lease_history")
+    __slots__ = (
+        "spec",
+        "state",
+        "attempts_used",
+        "active_lease",
+        "accepted_attempt_id",
+        "lease_history",
+        "failures_seen",
+        "expiries_seen",
+        "last_progress_at",
+    )
 
     def __init__(self, spec: TaskSpec):
         self.spec = spec
@@ -33,6 +53,9 @@ class TaskRecord:
         self.active_lease: Lease | None = None
         self.accepted_attempt_id: str | None = None
         self.lease_history: dict[str, Lease] = {}
+        self.failures_seen = 0
+        self.expiries_seen = 0
+        self.last_progress_at: datetime | None = None
 
 
 class LeaseStore(Protocol):

{flashruntime-0.6.0 → flashruntime-0.6.1}/flashruntime/protocol/v1alpha1.py

@@ -233,6 +233,9 @@ class EventType(str, Enum):
     FAILURE_CLASSIFIED = "FAILURE_CLASSIFIED"
     RECOVERY_ACTION_SELECTED = "RECOVERY_ACTION_SELECTED"
     RECOVERY_FROZEN = "RECOVERY_FROZEN"
+    # Lease hardening (additive, August 2026)
+    TASK_CANCELLED = "TASK_CANCELLED"
+    LEASE_RENEWAL_REFUSED = "LEASE_RENEWAL_REFUSED"
 
 
 class Event(BaseModel):
@@ -462,6 +465,8 @@ class TaskSpec(BaseModel):
     commit_key: str
     max_attempts: int = Field(default=3, ge=1)
     lease_seconds: float = Field(default=60.0, gt=0)
+    max_expiries: int = Field(default=9, ge=1)          # lease-expiry budget, separate from max_attempts
+    max_silent_seconds: float = Field(default=0.0, ge=0)  # renewal watchdog window; 0 (default) disables — the coordinator arms this at task-expansion time for tasks that produce progress signals
 
     @field_validator("lease_seconds")
     @classmethod
@@ -492,6 +497,7 @@ class Lease(BaseModel):
     node_id: str
     attempt_number: int
     deadline: datetime
+    lease_seconds: float = 0.0  # issued by a coordinator that predates this field if 0.0
     payload: dict[str, Any] = Field(default_factory=dict)
 
 

{flashruntime-0.6.0 → flashruntime-0.6.1}/flashruntime/recipes/command.py

@@ -233,6 +233,22 @@ class CommandRecipe(WorkloadRecipe):
                 # dies on ModuleNotFoundError, and both ends' unit tests
                 # still pass.
                 payload["dependencies"] = list(p["dependencies"])
+            if p.get("python") is not None:
+                # The interpreter `dependencies` above was resolved against
+                # ("3.11", major.minor, CPython) — declared by the submitter
+                # in flashml.yaml or derived by the compiler from the curated
+                # image the submitter chose. A no-container host provisions
+                # exactly this interpreter for the job's venv instead of
+                # trusting whatever python it happens to run.
+                #
+                # This forward exists because its absence was observed live
+                # on 2026-08-13: an Alibaba FC sandbox on Python 3.13 pulled
+                # pins wheeled only to cp312, every environment build died in
+                # a source compile, and the scheduler re-fed the same node
+                # until a healthy task was EXHAUSTED. Same failure shape as
+                # `dependencies` above: drop this line and both ends' unit
+                # tests still pass while the fleet fragments by interpreter.
+                payload["python"] = p["python"]
             if p.get("extra_dependencies"):
                 # The extras alone, named separately from `dependencies`
                 # above: the eighth placement gate keys on THIS field, not

{flashruntime-0.6.0 → flashruntime-0.6.1}/flashruntime/service/checkpoints.py

@@ -26,6 +26,33 @@ def _scope(job_id: str, task_id: str) -> str:
     return f"{job_id}::{task_id}"
 
 
+def _require_live_lease_on_this_task(manager, attempt_id: str, job_id: str, task_id: str) -> None:
+    """410 unless `attempt_id` is a lease that is both LIVE and *this
+    path's* (job_id, task_id) — not merely live somewhere.
+
+    `manager.lease_is_live(attempt_id)` alone answers a global question:
+    "is this lease id still the good one for whatever task it was issued
+    against?" That is true for any node currently holding any lease at
+    all, so checking liveness alone let a caller present a live lease from
+    task A against task B's checkpoint path — writing into a foreign
+    task's checkpoint lineage and resetting a foreign task's watchdog
+    baseline via `note_progress`. Binding the lease to the path pair
+    closes that: the presented attempt must be *this* task's own current
+    attempt, not merely proof that the caller holds work somewhere.
+    """
+    lease = manager.lease_info(attempt_id)
+    if (
+        lease is None
+        or lease.job_id != job_id
+        or lease.task_id != task_id
+        or not manager.lease_is_live(attempt_id)
+    ):
+        raise HTTPException(
+            status_code=410,
+            detail="lease is no longer live — checkpoint write refused",
+        )
+
+
 class RegisterPartRequest(BaseModel):
     attempt_id: str
     step: int = Field(ge=0)
@@ -48,12 +75,29 @@ def build_router(catalog: CheckpointCatalog, state, manager) -> APIRouter:
     @router.post("/parts")
     async def register_part(job_id: str, task_id: str, req: RegisterPartRequest, request: Request):
         authorize_task_write(state, manager, request, job_id, task_id)
+        # Fencing (audit C13/§5.4): `attempt_id` must name the lease that is
+        # STILL the live one, AND still this task's own current attempt —
+        # not a live lease borrowed from somewhere else. A zombie whose
+        # lease already expired — the heartbeat thread survived past the
+        # sweep, or the watchdog revoked it — must not be able to keep
+        # writing checkpoint bytes for a task another node now holds, and
+        # a caller holding a DIFFERENT live task must not be able to write
+        # into or reset the watchdog for THIS one. 410, same as a refused
+        # heartbeat: the caller must stop.
+        _require_live_lease_on_this_task(manager, req.attempt_id, job_id, task_id)
         catalog.register_part(_scope(job_id, task_id), req.attempt_id, req.step, req.part)
+        # Progress signal for the watchdog (audit C11), only now: after the
+        # fencing check passed and the write actually landed. Called with no
+        # `now` so the server clock stamps it — a client-supplied timestamp
+        # would let a stalled worker manufacture its own liveness.
+        manager.note_progress(job_id, task_id)
         return {"status": "registered", "key": req.part.key}
 
     @router.post("/commit")
     async def commit(job_id: str, task_id: str, req: CommitRequest, request: Request):
         authorize_task_write(state, manager, request, job_id, task_id)
+        # Same fencing as register_part above — see that comment.
+        _require_live_lease_on_this_task(manager, req.attempt_id, job_id, task_id)
         try:
             manifest = catalog.commit(
                 job_id=_scope(job_id, task_id),
@@ -67,7 +111,12 @@ def build_router(catalog: CheckpointCatalog, state, manager) -> APIRouter:
             )
         except CheckpointError as exc:
             # 409: the manifest was refused — parts missing or hashes wrong.
+            # No progress note: a rejected commit proved nothing about the
+            # work, so it must not reset the silence clock either.
             raise HTTPException(status_code=409, detail=str(exc))
+        # Only after the catalog accepted the manifest — a rejected commit
+        # above never reaches this line.
+        manager.note_progress(job_id, task_id)
         return manifest
 
     @router.get("/latest")