flashruntime 0.4.1__tar.gz → 0.4.3__tar.gz

{flashruntime-0.4.1/flashruntime.egg-info → flashruntime-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashruntime
-Version: 0.4.1
+Version: 0.4.3
 Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
 Author: Zolli Labs
 License-Expression: Apache-2.0

{flashruntime-0.4.1 → flashruntime-0.4.3}/flashml_workloads/fedavg_driver.py

@@ -29,7 +29,7 @@ from typing import Any, Callable, Protocol, Sequence, TypedDict
 
 from flashml_workloads.fedavg_weights import (
     apply_delta,
-    reduce_deltas,
+    reduce_deltas_with_report,
     require_finite,
 )
 
@@ -73,6 +73,26 @@ class RoundResult(TypedDict):
     mean_loss: float
     job_id: str
 
+    #: The round's contributions whose influence the aggregation capped, as
+    #: ``{"task_id", "norm", "cap", "scale"}`` — empty on an honest round,
+    #: which is every round unless somebody tried.
+    #:
+    #: ``task_id``, not a node id, because a task id is what this driver
+    #: actually holds. `fedavg_weights` reports positional indices; the round
+    #: knows which task each position came from, and a task is what a
+    #: coordinator leases. Resolving that to a MACHINE means deciding which
+    #: attempts count as accepted work, and that judgement already lives in
+    #: exactly one place — the cloud's round recorder, which reads the
+    #: coordinator's task view for provenance and joins on this task id.
+    #: Making a second copy of it here is how the two disagree.
+    #:
+    #: Recorded, never enforced. Nothing is quarantined, no credit withheld,
+    #: no lease refused: with a fleet this small a false positive costs a
+    #: volunteer their machine while a false negative costs one undeserved
+    #: credit, and that asymmetry only points one way. The row exists so the
+    #: owner can look.
+    clipped: list[dict]
+
 
 class RoundPlan(TypedDict):
     """What one round is: the job body to submit, and the task ids it will
@@ -122,7 +142,8 @@ class Coordinator(Protocol):
 
 def _round_body(round_idx: int, num_shards: int, worker_params: dict,
                 weights_uri: str | None, lease_seconds: float,
-                image: str, isolation_tier: str, allow_fallback: bool) -> dict:
+                image: str, isolation_tier: str, allow_fallback: bool,
+                pool: str | None = None) -> dict:
     params: dict[str, Any] = dict(worker_params)
     params.update({"round": round_idx, "num_shards": num_shards,
                    "lease_seconds": lease_seconds})
@@ -133,7 +154,7 @@ def _round_body(round_idx: int, num_shards: int, worker_params: dict,
         raise ValueError(
             f"image must be 'repository:tag' with a pinned tag, got {image!r}"
         )
-    return {
+    body = {
         "apiVersion": "flashml.dev/v1alpha1", "kind": "Job",
         "metadata": {"name": f"fedavg-r{round_idx:03d}"},
         "spec": {
@@ -143,6 +164,17 @@ def _round_body(round_idx: int, num_shards: int, worker_params: dict,
             "workload": {"type": "federated_averaging", "parameters": params},
         },
     }
+    if pool is not None and pool != "any":
+        # Same rule as the four expander stamps in recipes/command.py and
+        # service/modea.py: absent stays absent, and "any" is never stamped
+        # explicitly — the pool placement gate treats a stamped "any" as an
+        # ordinary pool id, which would make the round's tasks unplaceable
+        # everywhere rather than placeable anywhere. `None` (the caller made
+        # no pool statement) and the literal `"any"` (the caller stated the
+        # public queue explicitly) both mean "leave `placement` absent" —
+        # the default `PlacementSpec.pool` picks up "any" on its own.
+        body["spec"]["placement"] = {"pool": pool}
+    return body
 
 
 def _default_task_ids(num_shards: int) -> list[str]:
@@ -413,6 +445,7 @@ def run_fedavg(
     image: str = DEFAULT_IMAGE,
     isolation_tier: str = "standard",
     allow_fallback: bool = False,
+    pool: str | None = None,
     poll_attempts: int = 4,
     poll_backoff_s: float = 0.5,
     prior_job_ids: Sequence[tuple[int, str]] | None = None,
@@ -428,6 +461,17 @@ def run_fedavg(
     the same "two places, each correct in isolation" shape as the task-module
     allowlist drift that already caused an outage here.
 
+    `pool` is caller-settable for the same reason: the cloud control plane
+    owns pool ids, not this driver, so it must be free to say which pool (or
+    none) each round belongs to. `None` means the public queue, exactly as
+    before this parameter existed — every round of every run before pools
+    landed built a `placement`-less body, and that must stay byte-identical.
+    Round 0 and every later round go through the same `_round_body` call
+    inside this loop, so a pool named for round 0 travels to round 1, 2, …
+    without the caller repeating itself — the whole point being that pool
+    confinement must survive the round boundary, not just the round the job
+    was first submitted for.
+
     `build_round` replaces how a round becomes a job. The default builds the
     built-in `federated_averaging` body, whose tasks run
     `flashml_workloads.fedavg_worker`. A caller that wants the *user's own*
@@ -437,9 +481,9 @@ def run_fedavg(
     resume) is unchanged, because none of it depends on what ran inside the
     round, only on the task ids it produced and the `metrics.json` /
     `delta.json` pair each one committed. `worker_params`, `image`,
-    `isolation_tier`, `allow_fallback` and `lease_seconds` are inputs to the
-    *default* builder and are ignored when `build_round` is supplied — the
-    builder already knows all of it.
+    `isolation_tier`, `allow_fallback`, `pool` and `lease_seconds` are
+    inputs to the *default* builder and are ignored when `build_round` is
+    supplied — the builder already knows all of it.
 
     `initial_weights` may be `{}`, and that is not the same as "start from
     zeros": it means the driver holds no weights yet, so round 0's reduced
@@ -477,7 +521,7 @@ def run_fedavg(
             plan: RoundPlan = {
                 "body": _round_body(r, num_shards, worker_params, weights_uri,
                                     lease_seconds, image, isolation_tier,
-                                    allow_fallback),
+                                    allow_fallback, pool),
                 "task_ids": _default_task_ids(num_shards),
             }
         else:
@@ -535,7 +579,17 @@ def run_fedavg(
         # download. Anything committing from here on is discarded by
         # construction: we never re-read this job after aggregating.
         collected = _fetch(coord, keys)
-        reduced = reduce_deltas([(d, n) for d, n, _ in collected])
+        reduced, clip_events = reduce_deltas_with_report(
+            [(d, n) for d, n, _ in collected])
+        # `_fetch` preserves `keys`' order one-for-one, and every key in
+        # `keys` came out of the expected-key map, so a clip event's
+        # positional index resolves back to the exact task that sent it.
+        # Positional, NOT a shard number: a round aggregates on a quorum, so
+        # with shard-001 absent the third committer sits at position 2, and
+        # re-deriving `shard-{index:03d}` would name an honest volunteer.
+        by_key = _expected_metrics_keys(job_id, task_ids)
+        clipped = [{"task_id": by_key[keys[e.index]], "norm": e.norm,
+                    "cap": e.cap, "scale": e.scale} for e in clip_events]
         # No weights yet (`initial_weights={}` and nothing aggregated): the
         # round's workers were handed nothing, so what they reported as
         # "the change from what you were given" is the weights themselves.
@@ -561,6 +615,7 @@ def run_fedavg(
             "participants": len(collected),
             "mean_loss": sum(loss * n for _, n, loss in collected) / total_n,
             "job_id": job_id,
+            "clipped": clipped,
         }
         history.append(result)
         if on_round is not None:

{flashruntime-0.4.1 → flashruntime-0.4.3}/flashml_workloads/fedavg_weights.py

@@ -15,18 +15,48 @@ container.
 from __future__ import annotations
 
 import math
+from typing import NamedTuple
 
 __all__ = [
+    "CLIP_FACTOR",
+    "ClipEvent",
     "NonFiniteWeights",
     "WeightShapeMismatch",
     "apply_delta",
     "decode",
     "encode",
     "reduce_deltas",
+    "reduce_deltas_with_report",
     "require_finite",
     "subtract",
 ]
 
+#: How many times the round's MEDIAN contribution norm a single contribution
+#: may be before it is scaled back to that bound.
+#:
+#: 3.0, and the number is load-bearing. The governing property of the cap is
+#: that an honest round is bit-identical to a round with no cap at all:
+#: honest per-shard variation sits well inside 3x the median, so nothing
+#: fires and the arithmetic below is untouched. A factor of 1.0 would clip
+#: roughly half of every honest round and silently alter results that are
+#: correct today — a behaviour change wearing a safety net's clothes.
+CLIP_FACTOR: float = 3.0
+
+
+class ClipEvent(NamedTuple):
+    """One contribution that exceeded the round's cap, and by how much.
+
+    `index` is POSITIONAL into the `contributions` list the caller passed,
+    not a node id: this module is pure stdlib and knows nothing about
+    machines. Attribution is the driver's job — it holds the per-task
+    provenance and maps an index back to whoever sent it.
+    """
+
+    index: int
+    norm: float
+    cap: float
+    scale: float
+
 
 class WeightShapeMismatch(ValueError):
     """Two weight blobs do not describe the same parameter set.
@@ -138,13 +168,102 @@ def apply_delta(base: dict, delta: dict, scale: float = 1.0) -> dict:
     }, "apply_delta")
 
 
-def reduce_deltas(contributions: list[tuple[dict, int]]) -> dict:
+def _l2_norm(blob: dict) -> float:
+    """L2 norm of a delta, flattened across every parameter.
+
+    Only ever called on a blob `require_finite` has already accepted: the
+    multiplication below is a `TypeError` on the `None` a volunteer can put
+    in `data`, and a NaN anywhere would make the norm NaN, `norm > cap`
+    False, and the contribution sail through unscaled.
+    """
+    return math.sqrt(sum(v * v for p in blob.values() for v in p["data"]))
+
+
+def _median(values: list[float]) -> float:
+    """Median, with the even-length case spelled out: the mean of the two
+    middles.
+
+    Which is precisely why the cap is weak at two contributions — the
+    median of two values is their mean, and an attacker moves a mean
+    directly. Robust statistics need a majority to be honest, and with
+    `min_participants = 2` there is no majority to have. Documented, not
+    papered over: this does not fail closed at that quorum and must not be
+    described as protection there.
+    """
+    ordered = sorted(values)
+    mid = len(ordered) // 2
+    if len(ordered) % 2:
+        return ordered[mid]
+    return (ordered[mid - 1] + ordered[mid]) / 2
+
+
+def _scale_blob(blob: dict, scale: float) -> dict:
+    return {name: {"shape": list(p["shape"]),
+                   "data": [scale * v for v in p["data"]]}
+            for name, p in blob.items()}
+
+
+def reduce_deltas(contributions: list[tuple[dict, int]],
+                  *, clip_factor: float = CLIP_FACTOR) -> dict:
     """Sample-weighted mean of per-worker deltas (FedAvg).
 
     Weighting by sample count, not by worker, is what keeps the result
     equal to centralized training on the union of the shards when the
     shards are unequal — which they always are once machines differ.
+
+    A thin wrapper over `reduce_deltas_with_report`, kept because this name
+    has one production caller and 30+ tests pinning it. The clip report is
+    additive; callers that want it ask for it by name.
     """
+    return reduce_deltas_with_report(contributions, clip_factor=clip_factor)[0]
+
+
+def reduce_deltas_with_report(
+    contributions: list[tuple[dict, int]],
+    *,
+    clip_factor: float = CLIP_FACTOR,
+) -> tuple[dict, list[ClipEvent]]:
+    """`reduce_deltas`, plus the list of contributions the cap bound.
+
+    Everything MALFORMED is rejected below — non-positive, non-finite and
+    non-integer sample counts, mismatched shapes, NaN/Inf weights. What none
+    of those guards catch is a contribution that is perfectly well-formed
+    and adversarial: `delta = 1e6`, `n = 500`. Every check passes and the
+    sample-weighted mean moves the model by whatever the sender chose.
+
+    So, after validation and before the mean, each contribution's L2 norm is
+    compared against `C = median(norms) * clip_factor` and anything above it
+    is scaled to `C`. Median-anchored rather than a fixed constant because
+    the right magnitude depends on the model, the learning rate and the
+    round number, none of which this module knows — and because with a
+    majority of honest contributors the median is an honest value, which an
+    attacker-chosen mean is not.
+
+    Bounds MAGNITUDE, not direction. A small, consistently-biased delta
+    every round is unaffected, a node returning zeros still earns credit,
+    and a colluding majority defeats it by construction. It is a cap on how
+    far one contributor can move the model, not result verification.
+
+    Nothing is enforced here beyond the scaling: the events are returned so
+    the caller can record them. Revocation is a human decision.
+    """
+    # First, because it is the function's own configuration rather than
+    # untrusted input, and because the failure mode of getting it wrong is
+    # the worst one available: a non-positive or non-finite factor would
+    # disable the cap silently, the round would still reduce, and it would
+    # still report an empty clip list that an operator reads as "nobody
+    # tried". `clip_factor=0` would additionally zero every contribution.
+    try:
+        usable = math.isfinite(clip_factor) and clip_factor > 0
+    except TypeError:  # not a number at all
+        usable = False
+    if not usable:
+        raise ValueError(
+            f"reduce_deltas: clip_factor must be a finite number > 0, got "
+            f"{clip_factor!r}; a non-positive or non-finite value would "
+            "silently disable the influence cap rather than widening it"
+        )
+
     if not contributions:
         raise ValueError("reduce_deltas: no contributions")
     total = sum(n for _, n in contributions)
@@ -212,12 +331,38 @@ def reduce_deltas(contributions: list[tuple[dict, int]]) -> dict:
         _require_same_params(first, blob)
         require_finite(blob, f"reduce_deltas: contribution {i}")
 
+    # -- bounded influence, and ONLY here: after every guard above, because
+    # a malformed contribution must raise its own error rather than be
+    # quietly scaled into something plausible. Clipping caps a delta's
+    # magnitude and does nothing at all about a negative sample weight.
+    #
+    # `reduced` deliberately reuses the caller's own blob objects for every
+    # contribution that is not clipped, so an honest round accumulates the
+    # exact same float objects in the exact same order as it did before this
+    # existed. Byte-identical is the governing property; rebuilding every
+    # blob "harmlessly" would be the easiest way to lose it.
+    # Computed once, not once per use: deltas are megabytes, and this walks
+    # every weight in every contribution.
+    norms = [_l2_norm(blob) for blob, _ in contributions]
+    cap = _median(norms) * clip_factor
+    clipped: list[tuple[dict, int]] = list(contributions)
+    events: list[ClipEvent] = []
+    for i, ((blob, n), norm) in enumerate(zip(contributions, norms)):
+        # Strict `>` against a cap that is never negative — so a zero-norm
+        # contribution (a converged shard, or a lazy node returning zeros)
+        # is never the one being scaled, and `cap / norm` never divides by
+        # zero. An all-zero round puts the cap at 0.0 too and clips nothing.
+        if norm > cap:
+            scale = cap / norm
+            clipped[i] = (_scale_blob(blob, scale), n)
+            events.append(ClipEvent(index=i, norm=norm, cap=cap, scale=scale))
+
     out: dict = {}
     for name in first:
         acc = [0.0] * len(first[name]["data"])
-        for blob, n in contributions:
+        for blob, n in clipped:
             w = n / total
             for i, v in enumerate(blob[name]["data"]):
                 acc[i] += w * v
         out[name] = {"shape": list(first[name]["shape"]), "data": acc}
-    return out
+    return out, events

{flashruntime-0.4.1 → flashruntime-0.4.3}/flashruntime/protocol/v1alpha1.py

@@ -96,7 +96,13 @@ class ResourcesSpec(BaseModel):
 
 
 class PlacementSpec(BaseModel):
-    pool: Literal["any", "local", "edge", "standard-cloud", "secure-cloud"] = "any"
+    #: "any" (the default, and every pre-pools job) or a team-pool id minted
+    #: by the control plane. Until 0.4.3 this was a closed Literal of
+    #: infrastructure pool names that nothing ever read; widened when it
+    #: gained its first reader. Non-"any" makes every task of the job carry
+    #: payload["pool"] (recipes stamp it) and place only on nodes listing
+    #: that id in capabilities.pools.
+    pool: str = "any"
     architectures: list[Literal["amd64", "arm64"]] = Field(default_factory=lambda: ["amd64"])
 
 
@@ -309,6 +315,15 @@ class NodeCapabilities(BaseModel):
     #: this runtime cannot parse — never a guess. Placement reads the LENGTH
     #: of this list and nothing else in v1.
     gpus: list[GpuInfo] = Field(default_factory=list)
+    #: Team pools this node serves, as pool ids minted by the cloud control
+    #: plane. Stamped SERVER-SIDE by the cloud API's agent proxy from the
+    #: machine owner's memberships — an agent's self-reported value is
+    #: overwritten there, so the field is only as trustworthy as the
+    #: operator fronting registration. Empty means "serves no pool", and the
+    #: seventh placement gate refuses pool-scoped work (fail closed). NOT
+    #: the same thing as NodeRegistration.pool (singular), a deployment
+    #: profile label that predates teams; the two never interact.
+    pools: list[str] = Field(default_factory=list)
     os: str = ""
     architecture: str = ""
 
@@ -325,6 +340,14 @@ class NodeRegistration(BaseModel):
     #: every already-deployed agent is excluded from argv work until it is
     #: upgraded and explicitly opted in (security fields fail closed).
     argv_capable: bool = False
+    #: The operator explicitly opted this host into executing POOL-SCOPED
+    #: argv payloads without a container (`flashnode work --runner trusted`).
+    #: Defaults False so every existing agent is excluded until its owner
+    #: opts in (security fields fail closed). Distinct from argv_capable,
+    #: which asserts the CONTAINERISED argv contract — this asserts the
+    #: opposite: no sandbox, trusted-pool work only. The scheduler requires
+    #: pool + allowFallback + this flag together; any one alone places nothing.
+    unsandboxed_argv_capable: bool = False
     #: This node can run "module" (python -m <allowlisted module>) tasks.
     #: Defaults True — unlike argv_capable this is an AVAILABILITY gate, not
     #: a safety one: a module task placed on an incapable node just wastes
@@ -354,6 +377,12 @@ class NodeHeartbeat(BaseModel):
     node_id: str
     timestamp: datetime = Field(default_factory=utcnow)
     status: Literal["online", "draining", "terminating"] = "online"
+    #: Optional pool-membership refresh, stamped by the cloud API on the
+    #: heartbeat proxy so joining or leaving a pool reaches placement
+    #: without an agent restart. None means "no statement" — the
+    #: coordinator keeps whatever registration said. A list (even empty)
+    #: replaces `capabilities.pools` wholesale. Agents never set this.
+    pools: list[str] | None = None
 
 
 class NodeStatusView(BaseModel):
@@ -427,6 +456,60 @@ class TaskAttempt(BaseModel):
     accepted: bool = False
 
 
+class ExecutionEvidence(BaseModel):
+    """What the agent says about the run it is committing. **Evidence, never
+    proof** — the agent is the untrusted party, so every field here is
+    something a liar could fabricate. Its value is that a liar must now keep
+    two stories straight: these numbers have to stay consistent with the
+    elapsed time the *coordinator* measured between claim and commit, which
+    the agent does not control.
+
+    Optional on `CompleteRequest`, and it must stay optional — see the note
+    there.
+
+    NONE MEANS "NOT MEASURED". ZERO MEANS "MEASURED, AND IT WAS ZERO". These
+    are different facts and must never collapse onto one value: 0% GPU on a
+    task that asked for a GPU is the strongest single signal this block
+    carries, and a host with no sampler at all is no signal whatsoever. An
+    agent that cannot read a value sends `None`; a fabricated 0.0 turns "we
+    could not look" into "this node did nothing" and gets an honest
+    volunteer flagged. `image_digest` uses `""` for the same purpose, matching
+    `GpuInfo`'s string fields.
+
+    DELIBERATELY UNCONSTRAINED. No `ge=0`, no upper bound, no format check on
+    the digest. A validator here would let an agent's own self-report 422 the
+    agent's own work — inverting a design in which nothing is ever enforced
+    and everything is at most flagged. An implausible reading is worth more
+    recorded than refused, and refusing it mostly teaches a liar which values
+    pass.
+
+    Fields:
+    - `wall_seconds` — the agent's own clock around the run, excluding input
+      download and output upload. Cross-checked against coordinator-observed
+      elapsed, never trusted alone.
+    - `cpu_percent_mean` / `gpu_util_percent_mean` — mean utilisation over
+      the run, sampled by the agent. HOST-WIDE, not task-scoped: a volunteer
+      machine has other things running on it, so a high reading is weak
+      evidence and a *low* one is the interesting direction.
+    - `image_digest` — which image bytes actually executed, as the host
+      resolved them. `""` for a tier that ran no container at all.
+    - `exit_code` — the process/container exit status. On the reference agent
+      this is 0 whenever it is present at all, because a non-zero exit fails
+      the attempt and never reaches the commit path; it is carried so a
+      runner that commits partial results stays expressible, and so absence
+      still distinguishes an agent that measured from one that did not.
+
+    No `schema_version`: like `GpuInfo` and `CheckpointPart`, this is a
+    nested element of a wire message, not a wire message of its own.
+    """
+
+    wall_seconds: float | None = None
+    cpu_percent_mean: float | None = None
+    gpu_util_percent_mean: float | None = None
+    image_digest: str = ""
+    exit_code: int | None = None
+
+
 # ---------------------------------------------------------------------------
 # Checkpoint manifests (additive, July 2026)
 #

{flashruntime-0.4.1 → flashruntime-0.4.3}/flashruntime/recipes/command.py

@@ -82,13 +82,16 @@ class CommandRecipe(WorkloadRecipe):
 
     def expand(self, job_id: str, spec: JobSpec) -> list[TaskSpec]:
         isolation_spec = spec.spec.isolation
-        if isolation_spec.allowFallback:
-            # allowFallback waives the sandbox capability requirement at
-            # placement time. Honouring it for argv would let a submitter
-            # place arbitrary code on an unsandboxed node.
+        pool = spec.spec.placement.pool
+        if isolation_spec.allowFallback and pool == "any":
+            # The waiver is only acceptable INSIDE a pool: members chose to
+            # trust each other, and the seventh gate confines the task to
+            # them. Without a pool it would let a submitter place arbitrary
+            # code on any opted-in host — the exact thing the original
+            # unconditional refusal existed to prevent.
             raise ValueError(
-                "command jobs may not set isolation.allowFallback — "
-                "argv execution is container-only"
+                "command jobs may not set isolation.allowFallback without "
+                "placement.pool — unsandboxed argv is confined to team pools"
             )
         if isolation_spec.tier != "sandboxed":
             # Coordinator-side opt-in only: the operator running the pool
@@ -180,6 +183,12 @@ class CommandRecipe(WorkloadRecipe):
                 # exercising the key-missing branch, as unpack_inputs and
                 # local_inputs do.
                 payload["gpus"] = int(gpus)
+            if pool != "any":
+                # Same hop, same warning as local_inputs/gpus above: dropping
+                # this does NOT fail closed — the seventh gate would see a
+                # task requiring nothing and place it anywhere, carrying the
+                # waiver with it. Absent stays absent for "any".
+                payload["pool"] = pool
             tasks.append(
                 TaskSpec(
                     task_id=task_id,