flashruntime 0.5.0__tar.gz → 0.6.1__tar.gz

{flashruntime-0.5.0/flashruntime.egg-info → flashruntime-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashruntime
-Version: 0.5.0
+Version: 0.6.1
 Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
 Author: Zolli Labs
 License-Expression: Apache-2.0

{flashruntime-0.5.0 → flashruntime-0.6.1}/flashruntime/leases/manager.py

@@ -18,9 +18,26 @@ State transitions (task):
 
     PENDING ──claim──▶ LEASED ──complete (first valid)──▶ COMPLETED
        ▲                 │
-       └──sweep expiry / fail (attempts left)──┘
-    LEASED ──fail/expiry with no attempts left──▶ FAILED
+       └──sweep expiry / fail / revoked-silent / force_expire (budget left)──┘
+    LEASED ──fail, failure budget spent──▶ FAILED
+    LEASED ──expiry, expiry budget spent──▶ FAILED
     any non-terminal ──cancel──▶ CANCELLED
+
+Failure and expiry spend separate budgets (audit C12): a worker that
+reports N real failures exhausts at `max_attempts`; a lease that lapses
+unclaimed N times (pure churn — no worker ever ran it) exhausts at the
+separate, more generous `max_expiries`. `attempts_used` still numbers
+attempts but decides nothing terminal on its own.
+
+Two things can end a lease early besides its deadline (audit C11/§5.1).
+The *progress watchdog*: a heartbeat proves a process is alive, not that
+work is happening, so a holder silent past `spec.max_silent_seconds` has
+its next renewal refused and the task requeued — `note_progress` is what
+resets that clock. It is opt-in (`max_silent_seconds` defaults to 0, which
+disables it): a task that reports no progress signal is not thereby stuck,
+so whoever expands a job arms the window only for work that does report.
+And `force_expire`, the operator's manual version of the same revocation.
+Both spend the expiry budget, because both are churn.
 """
 
 from __future__ import annotations
@@ -80,6 +97,11 @@ class LeaseManager:
         record.state = TaskState.CANCELLED
         record.active_lease = None
         self._store.save(record)
+        # Emitted only on a real transition (audit §5.1): the ledger is the
+        # only place a cancellation was ever visible, and a task that
+        # vanished from the queue with no event read as a coordinator bug.
+        # An idempotent re-cancel transitions nothing, so it says nothing.
+        self._emit(EventType.TASK_CANCELLED, job_id, task_id, now=now)
 
     # -- worker side --------------------------------------------------------
 
@@ -125,6 +147,7 @@ class LeaseManager:
         if record is None:
             return None
         record.attempts_used += 1
+        record.last_progress_at = now
         lease = Lease(
             lease_id=f"ls-{uuid.uuid4().hex[:12]}",
             task_id=record.spec.task_id,
@@ -132,6 +155,9 @@ class LeaseManager:
             node_id=node_id,
             attempt_number=record.attempts_used,
             deadline=now + timedelta(seconds=record.spec.lease_seconds),
+            # On the wire so the holder can pick its own heartbeat interval
+            # from the real window instead of guessing one (audit §5.1).
+            lease_seconds=record.spec.lease_seconds,
             payload=record.spec.payload,
         )
         record.state = TaskState.LEASED
@@ -151,9 +177,38 @@ class LeaseManager:
     def heartbeat(self, lease_id: str, now: datetime | None = None) -> Lease:
         """Renew a live lease (extends the deadline by the task's lease
         window). Refused for unknown/expired/superseded leases — a worker
-        whose heartbeat is refused must stop working on the task."""
+        whose heartbeat is refused must stop working on the task.
+
+        Also refused, and the lease revoked, when the holder has been alive
+        but *silent* past `spec.max_silent_seconds` (audit C11). A heartbeat
+        proves a process is running; only `note_progress` proves the work
+        is. Without this, a stuck worker whose heartbeat thread survived
+        renewed its lease forever and the task never came back — the zombie
+        lease. Opt-in: `max_silent_seconds` defaults to 0, which disables
+        the watchdog, so it never fires on a workload that reports nothing.
+        """
         now = now or _utcnow()
         record, lease = self._require_live_lease(lease_id, now)
+        # After the liveness check on purpose: an already-dead lease is
+        # refused as dead ("no longer live"), not as silent. Only a lease
+        # that is still good takes the revocation path below.
+        window = record.spec.max_silent_seconds
+        baseline = record.last_progress_at
+        if window > 0 and baseline is not None and (now - baseline).total_seconds() > window:
+            self._emit(
+                EventType.LEASE_RENEWAL_REFUSED,
+                lease.job_id,
+                lease.task_id,
+                node_id=lease.node_id,
+                detail=f"no progress for {(now - baseline).total_seconds():.0f}s "
+                f"(budget {window:.0f}s) — lease revoked, task requeued",
+                now=now,
+            )
+            # Counter bumped at the call site, like fail()/sweep() (audit
+            # C12): a revoked lease is churn, so it spends the expiry budget.
+            record.expiries_seen += 1
+            self._release(record, now, EventType.TASK_REQUEUED, budget="expiry")
+            raise LeaseError(f"lease {lease_id} revoked: no progress within {window:.0f}s")
         renewed = lease.model_copy(
             update={"deadline": now + timedelta(seconds=record.spec.lease_seconds)}
         )
@@ -178,6 +233,13 @@ class LeaseManager:
         """
         now = now or _utcnow()
         record, lease = self._find_lease(lease_id)
+        if record.state == TaskState.COMPLETED and record.accepted_attempt_id == lease.lease_id:
+            # The winner asking again (its 200 was lost to a timeout, and it
+            # retried). Answering False told the one machine that actually
+            # did the work that it had lost — audit §2 "retried complete".
+            # Idempotent means same answer, and no second ACCEPTED event:
+            # the ledger already records the commit that happened.
+            return True
         if record.state == TaskState.COMPLETED or not self._is_live(record, lease, now):
             self._emit(
                 EventType.TASK_COMMIT_REJECTED,
@@ -202,11 +264,50 @@ class LeaseManager:
         )
         return True
 
+    def note_progress(
+        self, job_id: str, task_id: str, now: datetime | None = None
+    ) -> None:
+        """Record that the holder made *observable* progress on this task.
+
+        The other half of the C11 watchdog: heartbeats keep the deadline
+        alive, this keeps the silence budget alive. Callers are whatever the
+        workload reports through — a metric, a step counter, a partial
+        artifact write.
+
+        A silent no-op for an unknown task or one that is not LEASED: the
+        relay can always race an expiry, and a progress note arriving one
+        tick after the sweep took the lease away must not resurrect
+        anything, nor error at a reporter that did nothing wrong.
+        """
+        record = self._store.get(job_id, task_id)
+        if record is None or record.state != TaskState.LEASED:
+            return
+        record.last_progress_at = now or _utcnow()
+        self._store.save(record)
+
     def fail(self, lease_id: str, reason: str, now: datetime | None = None) -> None:
         """A worker reports its attempt failed. The task requeues while
-        attempts remain, otherwise it is FAILED for good."""
+        attempts remain, otherwise it is FAILED for good.
+
+        A report against a *known but dead* lease is recorded and ignored,
+        not raised (audit §5.1): the attempt really did fail, and that is
+        evidence worth keeping, but the task has already moved on — spending
+        its failure budget or touching its state would punish whoever holds
+        it now. Only a lease that never existed is an error.
+        """
         now = now or _utcnow()
-        record, lease = self._require_live_lease(lease_id, now)
+        record, lease = self._find_lease(lease_id)
+        if not self._is_live(record, lease, now):
+            self._emit(
+                EventType.TASK_ATTEMPT_FAILED,
+                lease.job_id,
+                lease.task_id,
+                node_id=lease.node_id,
+                detail=f"(late report — lease no longer live) {reason}",
+                now=now,
+            )
+            return
+        record.failures_seen += 1
         self._emit(
             EventType.TASK_ATTEMPT_FAILED,
             lease.job_id,
@@ -215,7 +316,7 @@ class LeaseManager:
             detail=reason,
             now=now,
         )
-        self._release(record, now, requeue_event=EventType.TASK_REQUEUED)
+        self._release(record, now, EventType.TASK_REQUEUED, budget="failure")
 
     # -- coordinator side ---------------------------------------------------
 
@@ -230,6 +331,7 @@ class LeaseManager:
             if lease is None or lease.deadline > now:
                 continue
             expired += 1
+            record.expiries_seen += 1
             self._emit(
                 EventType.LEASE_EXPIRED,
                 lease.job_id,
@@ -238,9 +340,37 @@ class LeaseManager:
                 detail=f"attempt {lease.attempt_number} heartbeat deadline passed",
                 now=now,
             )
-            self._release(record, now, requeue_event=EventType.TASK_REQUEUED)
+            self._release(record, now, EventType.TASK_REQUEUED, budget="expiry")
         return expired
 
+    def force_expire(
+        self, job_id: str, task_id: str, reason: str, now: datetime | None = None
+    ) -> bool:
+        """Revoke a task's live lease now, by operator decision.
+
+        The manual lever behind the automatic ones (audit §5.1): when a
+        machine is known-stuck but its deadline is minutes away, waiting is
+        the only thing an operator could do. Returns False when the task is
+        not LEASED — there is nothing to take away, and re-running the same
+        command is then a no-op rather than an error.
+        """
+        now = now or _utcnow()
+        record = self._require(job_id, task_id)
+        if record.state != TaskState.LEASED:
+            return False
+        lease = record.active_lease
+        self._emit(
+            EventType.LEASE_EXPIRED,
+            job_id,
+            task_id,
+            node_id=lease.node_id if lease is not None else None,
+            detail=f"force-expired: {reason}",
+            now=now,
+        )
+        record.expiries_seen += 1
+        self._release(record, now, EventType.TASK_REQUEUED, budget="expiry")
+        return True
+
     def job_state(self, job_id: str) -> dict[str, int]:
         """Task counts by state — the number the dashboard shows."""
         counts: dict[str, int] = {}
@@ -260,6 +390,20 @@ class LeaseManager:
             return None
         return lease
 
+    def lease_is_live(self, lease_id: str, now: datetime | None = None) -> bool:
+        """Is this lease still the good one? A question, so it answers.
+
+        Unlike `_find_lease`, an unknown id is False rather than an error:
+        callers ask this to decide whether to keep going, and "I have never
+        heard of that lease" is a perfectly good "stop".
+        """
+        now = now if now is not None else _utcnow()
+        try:
+            record, lease = self._find_lease(lease_id)
+        except LeaseError:
+            return False
+        return self._is_live(record, lease, now)
+
     def live_leases_for_node(
         self, node_id: str, now: datetime | None = None
     ) -> set[tuple[str, str]]:
@@ -294,15 +438,42 @@ class LeaseManager:
 
     # -- internals ----------------------------------------------------------
 
-    def _release(self, record: TaskRecord, now: datetime, requeue_event: EventType) -> None:
+    def _release(
+        self,
+        record: TaskRecord,
+        now: datetime,
+        requeue_event: EventType,
+        *,
+        budget: str,
+    ) -> None:
+        """Return a task to PENDING, or exhaust it — against the budget the
+        caller spent (audit C12).
+
+        `"failure"` and `"expiry"` are separate budgets on purpose: a
+        worker that keeps reporting real failures is different evidence
+        from a lease that keeps lapsing unclaimed (pure churn — no worker
+        ever ran it), so each is checked against its own counter
+        (`spec.max_attempts` / `spec.max_expiries`) and neither spends the
+        other's.
+        """
         record.active_lease = None
-        if record.attempts_used >= record.spec.max_attempts:
+        exhausted = (
+            record.failures_seen >= record.spec.max_attempts
+            if budget == "failure"
+            else record.expiries_seen >= record.spec.max_expiries
+        )
+        if exhausted:
             record.state = TaskState.FAILED
+            detail = (
+                f"all {record.spec.max_attempts} attempts failed"
+                if budget == "failure"
+                else f"lease expired {record.expiries_seen} times without completing"
+            )
             self._emit(
                 EventType.TASK_EXHAUSTED,
                 record.spec.job_id,
                 record.spec.task_id,
-                detail=f"all {record.spec.max_attempts} attempts used",
+                detail=detail,
                 now=now,
             )
         else:

{flashruntime-0.5.0 → flashruntime-0.6.1}/flashruntime/leases/sqlite_store.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 
 import json
 import sqlite3
+from datetime import datetime
 from pathlib import Path
 
 from flashruntime.leases.store import TaskRecord
@@ -49,6 +50,7 @@ class SqliteLeaseStore:
         self._conn = sqlite3.connect(str(path), check_same_thread=False)
         self._migrate()                 # legacy single-PK table → composite
         self._conn.executescript(_SCHEMA)
+        self._ensure_runtime_column()   # additive: budget-split counters (audit C12)
         self._cache: dict[tuple[str, str], TaskRecord] = {}
         self._seq = 0
         self._load()
@@ -79,15 +81,27 @@ class SqliteLeaseStore:
         )
         self._conn.commit()
 
+    def _ensure_runtime_column(self) -> None:
+        """Additive migration (audit C12): one nullable JSON column carries
+        the budget-split counters (`failures_seen`, `expiries_seen`,
+        `last_progress_at`). A plain `ADD COLUMN` — unlike the composite-key
+        rebuild above, no rewrite is needed since every existing row simply
+        reads back NULL, and `_load` treats that as the pre-upgrade
+        zeros/None defaults rather than failing to load."""
+        cols = {row[1] for row in self._conn.execute("PRAGMA table_info(lease_tasks)").fetchall()}
+        if "runtime_json" not in cols:
+            self._conn.execute("ALTER TABLE lease_tasks ADD COLUMN runtime_json TEXT")
+            self._conn.commit()
+
     # -- rehydration ---------------------------------------------------------
 
     def _load(self) -> None:
         rows = self._conn.execute(
             "SELECT spec_json, state, attempts_used, active_lease_json,"
-            " accepted_attempt_id, lease_history_json, seq"
+            " accepted_attempt_id, lease_history_json, runtime_json, seq"
             " FROM lease_tasks ORDER BY seq"
         ).fetchall()
-        for spec_json, state, attempts, lease_json, accepted, history_json, seq in rows:
+        for spec_json, state, attempts, lease_json, accepted, history_json, runtime_json, seq in rows:
             record = TaskRecord(TaskSpec.model_validate_json(spec_json))
             record.state = TaskState(state)
             record.attempts_used = attempts
@@ -97,6 +111,15 @@ class SqliteLeaseStore:
                 lid: Lease.model_validate(raw)
                 for lid, raw in json.loads(history_json).items()
             }
+            # `.get(...)` defaults: a row written before this column existed
+            # (or with it explicitly NULL) loads as zeros/None, not an error.
+            runtime = json.loads(runtime_json) if runtime_json else {}
+            record.failures_seen = runtime.get("failures_seen", 0)
+            record.expiries_seen = runtime.get("expiries_seen", 0)
+            last_progress_at = runtime.get("last_progress_at")
+            record.last_progress_at = (
+                datetime.fromisoformat(last_progress_at) if last_progress_at else None
+            )
             self._cache[(record.spec.job_id, record.spec.task_id)] = record
             self._seq = max(self._seq, seq or 0)
 
@@ -140,18 +163,28 @@ class SqliteLeaseStore:
         history = json.dumps(
             {lid: json.loads(lease.model_dump_json()) for lid, lease in record.lease_history.items()}
         )
+        runtime = json.dumps(
+            {
+                "failures_seen": record.failures_seen,
+                "expiries_seen": record.expiries_seen,
+                "last_progress_at": (
+                    record.last_progress_at.isoformat() if record.last_progress_at else None
+                ),
+            }
+        )
         self._conn.execute(
             "INSERT INTO lease_tasks"
             " (task_id, job_id, spec_json, state, attempts_used, active_lease_json,"
-            "  accepted_attempt_id, lease_history_json, seq)"
-            " VALUES (?, ?, ?, ?, ?, ?, ?, ?,"
+            "  accepted_attempt_id, lease_history_json, runtime_json, seq)"
+            " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,"
             "   COALESCE(?, (SELECT seq FROM lease_tasks"
             "                 WHERE job_id = ? AND task_id = ?)))"
             " ON CONFLICT(job_id, task_id) DO UPDATE SET"
             "  state=excluded.state, attempts_used=excluded.attempts_used,"
             "  active_lease_json=excluded.active_lease_json,"
             "  accepted_attempt_id=excluded.accepted_attempt_id,"
-            "  lease_history_json=excluded.lease_history_json",
+            "  lease_history_json=excluded.lease_history_json,"
+            "  runtime_json=excluded.runtime_json",
             (
                 record.spec.task_id,
                 record.spec.job_id,
@@ -161,6 +194,7 @@ class SqliteLeaseStore:
                 record.active_lease.model_dump_json() if record.active_lease else None,
                 record.accepted_attempt_id,
                 history,
+                runtime,
                 seq,
                 record.spec.job_id,
                 record.spec.task_id,

{flashruntime-0.5.0 → flashruntime-0.6.1}/flashruntime/leases/store.py

@@ -11,6 +11,7 @@ manager, never in a store, so every backend behaves identically.
 
 from __future__ import annotations
 
+from datetime import datetime
 from typing import Protocol
 
 from flashruntime.protocol.v1alpha1 import Lease, TaskSpec, TaskState
@@ -22,9 +23,28 @@ class TaskRecord:
     `lease_history` keeps every lease ever issued for the task so a late
     commit from a long-dead worker can be *identified and rejected* rather
     than erroring as unknown — rejected duplicates are recovery evidence.
+
+    `failures_seen` and `expiries_seen` (audit C12) are separate budgets:
+    a worker that reports a failure spends `failures_seen` against
+    `spec.max_attempts`; a lease that lapses unclaimed spends
+    `expiries_seen` against `spec.max_expiries`. `attempts_used` still
+    numbers attempts (and drives `Lease.attempt_number`), but no longer by
+    itself decides anything terminal. `last_progress_at` is stamped on
+    every claim — the last time some worker showed up for this task at
+    all, regardless of how that attempt ended.
     """
 
-    __slots__ = ("spec", "state", "attempts_used", "active_lease", "accepted_attempt_id", "lease_history")
+    __slots__ = (
+        "spec",
+        "state",
+        "attempts_used",
+        "active_lease",
+        "accepted_attempt_id",
+        "lease_history",
+        "failures_seen",
+        "expiries_seen",
+        "last_progress_at",
+    )
 
     def __init__(self, spec: TaskSpec):
         self.spec = spec
@@ -33,6 +53,9 @@ class TaskRecord:
         self.active_lease: Lease | None = None
         self.accepted_attempt_id: str | None = None
         self.lease_history: dict[str, Lease] = {}
+        self.failures_seen = 0
+        self.expiries_seen = 0
+        self.last_progress_at: datetime | None = None
 
 
 class LeaseStore(Protocol):

{flashruntime-0.5.0 → flashruntime-0.6.1}/flashruntime/protocol/v1alpha1.py

@@ -233,6 +233,9 @@ class EventType(str, Enum):
     FAILURE_CLASSIFIED = "FAILURE_CLASSIFIED"
     RECOVERY_ACTION_SELECTED = "RECOVERY_ACTION_SELECTED"
     RECOVERY_FROZEN = "RECOVERY_FROZEN"
+    # Lease hardening (additive, August 2026)
+    TASK_CANCELLED = "TASK_CANCELLED"
+    LEASE_RENEWAL_REFUSED = "LEASE_RENEWAL_REFUSED"
 
 
 class Event(BaseModel):
@@ -332,6 +335,17 @@ class NodeCapabilities(BaseModel):
     #: this runtime cannot parse — never a guess. Placement reads the LENGTH
     #: of this list and nothing else in v1.
     gpus: list[GpuInfo] = Field(default_factory=list)
+    #: Bytes this host will lend to cached dataset shards. Advertised so the
+    #: control plane can refuse a job at SUBMIT time when no host in the pool
+    #: can hold a slice — twenty machines each downloading for forty minutes
+    #: and then refusing is the failure this exists to replace.
+    #:
+    #: Defaults to 0, and 0 means "send me no dataset work". Fail closed, the
+    #: same polarity as `can_install_dependencies`: an agent whose
+    #: registration predates this field has nowhere to put a shard and must
+    #: never be handed one. Bytes rather than GB to match `memory_bytes` —
+    #: one unit for every capacity on this model.
+    dataset_cache_bytes: int = 0
     #: Team pools this node serves, as pool ids minted by the cloud control
     #: plane. Stamped SERVER-SIDE by the cloud API's agent proxy from the
     #: machine owner's memberships — an agent's self-reported value is
@@ -451,6 +465,8 @@ class TaskSpec(BaseModel):
     commit_key: str
     max_attempts: int = Field(default=3, ge=1)
     lease_seconds: float = Field(default=60.0, gt=0)
+    max_expiries: int = Field(default=9, ge=1)          # lease-expiry budget, separate from max_attempts
+    max_silent_seconds: float = Field(default=0.0, ge=0)  # renewal watchdog window; 0 (default) disables — the coordinator arms this at task-expansion time for tasks that produce progress signals
 
     @field_validator("lease_seconds")
     @classmethod
@@ -481,6 +497,7 @@ class Lease(BaseModel):
     node_id: str
     attempt_number: int
     deadline: datetime
+    lease_seconds: float = 0.0  # issued by a coordinator that predates this field if 0.0
     payload: dict[str, Any] = Field(default_factory=dict)
 
 

{flashruntime-0.5.0 → flashruntime-0.6.1}/flashruntime/recipes/command.py

@@ -66,6 +66,18 @@ class CommandRecipe(WorkloadRecipe):
                 )
             elif not all(isinstance(d, str) for d in extra_deps):
                 problems.append("'extra_dependencies' must be a list of strings")
+        slices = params.get("dataset_slices")
+        if slices is not None:
+            if not isinstance(slices, list) or isinstance(slices, (str, bytes)):
+                problems.append(
+                    "'dataset_slices' must be a list with one entry per task "
+                    "(a list, not a single string)"
+                )
+            elif not all(isinstance(s, list) for s in slices):
+                problems.append(
+                    "'dataset_slices' must be a list of per-task lists, one "
+                    "list of dataset slices for each task"
+                )
         problems.extend(self._unpack_problems(params))
         problems.extend(partition_problems(params))
         problems.extend(declaration_problems(params.get("validators")))
@@ -221,6 +233,22 @@ class CommandRecipe(WorkloadRecipe):
                 # dies on ModuleNotFoundError, and both ends' unit tests
                 # still pass.
                 payload["dependencies"] = list(p["dependencies"])
+            if p.get("python") is not None:
+                # The interpreter `dependencies` above was resolved against
+                # ("3.11", major.minor, CPython) — declared by the submitter
+                # in flashml.yaml or derived by the compiler from the curated
+                # image the submitter chose. A no-container host provisions
+                # exactly this interpreter for the job's venv instead of
+                # trusting whatever python it happens to run.
+                #
+                # This forward exists because its absence was observed live
+                # on 2026-08-13: an Alibaba FC sandbox on Python 3.13 pulled
+                # pins wheeled only to cp312, every environment build died in
+                # a source compile, and the scheduler re-fed the same node
+                # until a healthy task was EXHAUSTED. Same failure shape as
+                # `dependencies` above: drop this line and both ends' unit
+                # tests still pass while the fleet fragments by interpreter.
+                payload["python"] = p["python"]
             if p.get("extra_dependencies"):
                 # The extras alone, named separately from `dependencies`
                 # above: the eighth placement gate keys on THIS field, not
@@ -239,6 +267,31 @@ class CommandRecipe(WorkloadRecipe):
                 # hop have tests that pass while it is broken, because each
                 # constructs the payload directly.
                 payload["extra_dependencies"] = list(p["extra_dependencies"])
+            if p.get("dataset_slices") is not None:
+                # The ONLY per-task payload key in this method. Every other
+                # forward above copies one value to every task; each task
+                # fetches a DIFFERENT slice of the manifest, so this one is
+                # indexed by task position exactly as `task_params` is.
+                #
+                # Computed by the flashml-cloud compiler and opaque here:
+                # the byte-weighted mapping lives beside `fleet_shape` in
+                # that repo's `elastic.py`, and a second implementation here
+                # is the drift that module's docstring warns about.
+                #
+                # Same failure shape as `local_inputs` and `dependencies`:
+                # drop this line and every host fetches nothing, the workload
+                # dies on a missing path, and both ends' unit tests still
+                # pass because each constructs the payload directly.
+                slices = p["dataset_slices"]
+                if i >= len(slices):
+                    raise ValueError(
+                        f"task {i}: 'dataset_slices' has {len(slices)} "
+                        f"entr{'y' if len(slices) == 1 else 'ies'} but this "
+                        f"job expanded to more tasks — every task needs a "
+                        f"slice, even an empty one"
+                    )
+                # `list(...)` so a payload never aliases the caller's spec.
+                payload["datasets"] = list(slices[i])
             gpus = spec.spec.resources.gpuPerTask
             if gpus:
                 # The GPU requirement is the ONLY payload key sourced from

{flashruntime-0.5.0 → flashruntime-0.6.1}/flashruntime/scheduler/__init__.py

@@ -386,6 +386,56 @@ class IsolationAwarePlacement(PlacementPolicy):
     waiver relaxes the sandbox tier and nothing else. No waiver makes a
     read-only container with no network able to run `pip install`.
 
+    A ninth gate applies to tasks whose payload carries `datasets`: the sum
+    of every slice entry's `size` must fit inside the claiming node's
+    `capabilities.dataset_cache_bytes`. This is CAPACITY, not capability —
+    every tier can fetch, because the AGENT fetches before the sandbox
+    closes, so unlike `extra_dependencies` there is no tier that physically
+    cannot. The only question is whether the bytes fit.
+
+    Not to be confused with the fourth gate. `local_inputs`/`local_datasets`
+    asks whether a host already HOLDS named data it will never upload; this
+    one asks whether a host has ROOM for data it is about to fetch from a
+    public origin. Different payload key, different capability, different
+    question.
+
+    It takes the argv/local-data/gpu/exclude_nodes/pool/dependencies
+    polarity — **fail closed** — and every leg of it:
+
+    - A node advertising nothing (an agent whose registration predates the
+      field) is refused, and so is one advertising `0`, which is the model
+      default and means "send me no dataset work". A shard with nowhere to
+      go is a task that dies after a forty-minute download rather than one
+      that never started, and twenty machines doing that at once is the
+      failure the advertisement exists to prevent.
+    - The advertisement must be a genuine `int`. `bool` is excluded
+      explicitly for the reason the `gpus` gate excludes it: it is an `int`
+      subclass, so a `dataset_cache_bytes: true` typo would read as the
+      capacity 1.
+    - `capabilities` itself may be absent or type-confused; that is read as
+      advertising nothing rather than allowed to raise. `isinstance`, never
+      `(node.get("capabilities") or {})` — a string `capabilities` value has
+      no `.get` and would crash the predicate inside the claim path, taking
+      the whole claim down rather than one task.
+    - A non-dict slice, a non-dict entry, or an entry whose `size` is not a
+      non-negative `int` makes the task ineligible everywhere. The manifest
+      is submitter-controlled, so an unreadable size must NOT contribute 0 to
+      the total: that is precisely how a hostile slice would buy itself an
+      unbounded fetch on a node that advertised a modest cache.
+    - The requirement is the sum over every entry of every slice. Checking
+      per-dataset, or against the largest entry, would place a task that then
+      fills a volunteer's disk halfway through the fetch.
+    - Empty and absent both mean "declares no data": `if
+      task.payload.get("datasets")` treats `None` and `[]` alike, so every
+      job deployed today is completely unaffected. A slice that is PRESENT
+      but happens to carry no entries is still dataset work and still needs
+      a readable advertisement — the fail-closed leg is the advertisement,
+      not the total.
+
+    `allowFallback` does not waive this gate either. It waives the sandbox
+    tier and nothing else; a submitter's isolation posture has nothing to say
+    about how much disk a stranger's machine offered.
+
     Everything genuinely standard keeps the fail-open placement default."""
 
     def eligible(self, task: TaskSpec, node: NodeView) -> bool:
@@ -516,6 +566,48 @@ class IsolationAwarePlacement(PlacementPolicy):
         if task.payload.get("extra_dependencies"):
             if node.get("can_install_dependencies") is not True:
                 return False
+        # Capacity, not capability, and the only gate in this class that is.
+        # Every tier's AGENT has network and a writable disk before the
+        # sandbox closes, so unlike `extra_dependencies` there is no tier
+        # that physically cannot fetch — the only question is whether the
+        # bytes fit. Distinct from the `local_inputs` gate above: that one
+        # asks whether a host already HOLDS named data, this one whether it
+        # has ROOM for data it is about to fetch.
+        #
+        # Fail-closed on an absent or type-confused advertisement, and
+        # checked before the allowFallback waiver below for the same reason
+        # as every gate above: the waiver relaxes the sandbox tier and has
+        # nothing to say about how much disk a stranger's machine offered.
+        if task.payload.get("datasets"):
+            needed = 0
+            for dataset in task.payload["datasets"]:
+                if not isinstance(dataset, dict):
+                    return False  # type-confused slice ⇒ fail closed, no crash
+                for entry in dataset.get("entries") or []:
+                    size = entry.get("size") if isinstance(entry, dict) else None
+                    # A non-int size is a malformed slice, not a free one:
+                    # letting it contribute 0 is how a hostile manifest buys
+                    # an unbounded fetch on a modest cache. `bool` is an
+                    # `int` subclass, so `size: true` is excluded explicitly.
+                    if (
+                        not isinstance(size, int)
+                        or isinstance(size, bool)
+                        or size < 0
+                    ):
+                        return False
+                    needed += size
+            capabilities = node.get("capabilities")
+            # isinstance, not `or {}` — a string capabilities value has no
+            # `.get` and must fail closed rather than crash the predicate.
+            advertised = (
+                capabilities.get("dataset_cache_bytes")
+                if isinstance(capabilities, dict)
+                else None
+            )
+            if not isinstance(advertised, int) or isinstance(advertised, bool):
+                return False  # absent/type-confused ⇒ advertises nothing
+            if needed > advertised:
+                return False
         isolation = task.payload.get("isolation")
         if isolation is None:
             return True  # no isolation payload ⇒ standard, runs anywhere