flashruntime 0.3.0__tar.gz → 0.4.0__tar.gz

{flashruntime-0.3.0/flashruntime.egg-info → flashruntime-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashruntime
-Version: 0.3.0
+Version: 0.4.0
 Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
 Author: Zolli Labs
 License-Expression: Apache-2.0

{flashruntime-0.3.0 → flashruntime-0.4.0}/flashruntime/protocol/v1alpha1.py

@@ -297,6 +297,16 @@ class NodeRegistration(BaseModel):
     #: keeps receiving module work; only a node that explicitly opts into
     #: an argv-only runner sets this False (see scheduler.IsolationAwarePlacement).
     module_capable: bool = True
+    #: Names of datasets this node holds locally and offers to jobs. Names
+    #: only — never paths: a host path is host-private and must not travel to
+    #: the coordinator. Defaults to an empty list, so a node advertises
+    #: nothing until its operator explicitly maps a dataset: an agent that has
+    #: not opted in must never be eligible for local-data work, and an absent
+    #: field on an already-deployed agent's registration must read as "offers
+    #: nothing" rather than "offers anything" (security fields fail closed).
+    #: `default_factory` and not a bare `[]` — one shared list would let a
+    #: single node's advertisement leak into every other node's registration.
+    local_datasets: list[str] = Field(default_factory=list)
     pool: str = "local"
     runtime_profile: str = "kubernetes"
     labels: dict[str, str] = Field(default_factory=dict)

{flashruntime-0.3.0 → flashruntime-0.4.0}/flashruntime/recipes/command.py

@@ -146,6 +146,20 @@ class CommandRecipe(WorkloadRecipe):
                 # for byte. Emitting `[]` would mean the same thing today but
                 # would stop exercising that path.
                 payload["unpack_inputs"] = list(p["unpack_inputs"])
+            if p.get("local_inputs") is not None:
+                # Labels for host-supplied directories. This forward is what
+                # makes the local-data feature real: `IsolationAwarePlacement`
+                # reads `task.payload["local_inputs"]` to decide eligibility,
+                # and flashnode reads it again to know what to mount.
+                #
+                # Dropping it does NOT fail closed. The gate sees a task
+                # requiring nothing, places it on any node, and flashnode
+                # mounts nothing — so the task runs without the data it asked
+                # for. Both ends of this hop have tests that pass while it is
+                # broken, because each constructs the payload directly.
+                #
+                # `list(...)` so a payload never aliases the caller's spec.
+                payload["local_inputs"] = list(p["local_inputs"])
             tasks.append(
                 TaskSpec(
                     task_id=task_id,

{flashruntime-0.3.0 → flashruntime-0.4.0}/flashruntime/scheduler/__init__.py

@@ -143,6 +143,35 @@ class IsolationAwarePlacement(PlacementPolicy):
     cut every already-deployed node off from module work the moment this
     field shipped, without anyone having opted out of anything.
 
+    A fourth gate applies to tasks whose payload lists `local_inputs` (data
+    the host holds and never uploads): the claiming node must advertise
+    EVERY one of those names in its `local_datasets` capability. It shares
+    the argv gate's fail-closed polarity, not the module gate's, because
+    the host's data is the thing being protected — an absent, `None`, or
+    wrongly-typed capability counts as NOT capable:
+
+    - The capability must be a genuine *list* of names. A bare string does
+      not count, even when it looks right: ``"patients" in "patients"`` is
+      True in Python, so accepting a string would let a node advertise
+      every dataset whose name is a substring of anything it said. That is
+      the same class of type confusion `sandbox_capable is True` avoids
+      for booleans.
+    - The requirement itself must be a list. A `local_inputs` payload that
+      is present but not a list (e.g. the bare string ``"patients"``) makes
+      the task ineligible everywhere — fail closed without crashing the
+      predicate, exactly as a non-dict isolation payload does.
+    - An empty `local_inputs` requires nothing and so runs anywhere, like
+      tier ``"standard"``.
+
+    The `allowFallback` waiver does NOT apply here, for the same reason it
+    does not waive the argv gate: it waives the sandbox *tier* requirement
+    and nothing else. A waiver is the submitter's statement about their own
+    isolation posture; the local-data gate protects the HOST's data, which
+    the submitter has no standing to waive. A task placed on a node that
+    never advertised the dataset would fail on the agent anyway (the label
+    is not in its map) — failing closed here makes it fail before anything
+    touches the data, and before an attempt is burned.
+
     Everything genuinely standard keeps the fail-open placement default."""
 
     def eligible(self, task: TaskSpec, node: NodeView) -> bool:
@@ -158,6 +187,18 @@ class IsolationAwarePlacement(PlacementPolicy):
         # into the same node's path until attempts are exhausted.
         if "module" in task.payload and node.get("module_capable") is False:
             return False
+        # Fail-closed like the argv gate, and checked before the allowFallback
+        # waiver below for the same reason: the waiver covers the sandbox tier
+        # only, and a submitter cannot waive their way onto a host's data.
+        local_inputs = task.payload.get("local_inputs")
+        if local_inputs is not None:
+            if not isinstance(local_inputs, list):
+                return False  # type-confused requirement ⇒ fail closed, no crash
+            advertised = node.get("local_datasets")
+            if local_inputs and not isinstance(advertised, list):
+                return False  # absent/None/type-confused capability ⇒ not capable
+            if any(name not in advertised for name in local_inputs):
+                return False
         isolation = task.payload.get("isolation")
         if isolation is None:
             return True  # no isolation payload ⇒ standard, runs anywhere

{flashruntime-0.3.0 → flashruntime-0.4.0}/flashruntime/service/modea.py

@@ -643,6 +643,9 @@ def build_router(state: ModeAState) -> APIRouter:
             "sandbox_capable": entry.registration.sandbox_capable,
             "argv_capable": entry.registration.argv_capable,
             "module_capable": entry.registration.module_capable,
+            # Names only — the host paths behind them never reach us, and the
+            # placement gate needs the names to place local-data work at all.
+            "local_datasets": entry.registration.local_datasets,
             "capabilities": entry.registration.capabilities.model_dump(),
         }
         lease = manager.claim(

{flashruntime-0.3.0 → flashruntime-0.4.0/flashruntime.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashruntime
-Version: 0.3.0
+Version: 0.4.0
 Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
 Author: Zolli Labs
 License-Expression: Apache-2.0

{flashruntime-0.3.0 → flashruntime-0.4.0}/flashruntime.egg-info/SOURCES.txt

@@ -118,8 +118,11 @@ tests/test_leases.py
 tests/test_leases_scope.py
 tests/test_leases_sqlite.py
 tests/test_monitor_sampler.py
+tests/test_placement_local_data.py
 tests/test_planner.py
+tests/test_protocol_local_datasets.py
 tests/test_protocol_v1alpha1.py
+tests/test_recipe_local_inputs.py
 tests/test_recipes_command.py
 tests/test_recovery.py
 tests/test_run_json.py

{flashruntime-0.3.0 → flashruntime-0.4.0}/pyproject.toml

@@ -5,11 +5,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "flashruntime"
-# 0.3.0 is the FIRST published release. 0.1.0 and 0.2.0 were declared here but
-# never reached PyPI — the name 404'd until now — so nothing is superseded and
-# no installed version is being replaced. Released from the Zolli-Labs/flashml
-# monorepo by pushing a `flashruntime-v0.3.0` tag.
-version = "0.3.0"
+# 0.4.0 adds NodeRegistration.local_datasets — a new wire field, so consumers
+# must pin a range that includes it. flashnode’s floor moves to >=0.4,<0.5.
+# (0.3.0 was the first published release; 0.1.0 and 0.2.0 were declared here
+# but never reached PyPI.) Released from the Zolli-Labs/flashml monorepo by
+# pushing a `flashruntime-v0.4.0` tag.
+version = "0.4.0"
 description = "Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute."
 readme = "README.md"
 # PEP 639 SPDX expression (string form) — replaces the deprecated

flashruntime-0.4.0/tests/test_placement_local_data.py

@@ -0,0 +1,205 @@
+"""The fourth fail-closed placement gate: a task that names `local_inputs` may
+only be leased to a node advertising every one of those names in its
+`local_datasets` capability (AGENTS.md rule 3).
+
+The data never leaves the host, so placement is the only thing standing between
+a task and a machine that cannot serve it. An absent, None, or wrongly-typed
+capability counts as NOT capable, exactly like `sandbox_capable`/`argv_capable`.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+
+def _local_task(local_inputs=("patients",), **payload_extra):
+    """A task requiring local datasets. `local_inputs` is passed through RAW so
+    tests can poison it with type-confused values the protocol would never
+    emit (the same trick `_task_raw` plays on the isolation payload)."""
+    from flashruntime.protocol.v1alpha1 import TaskSpec
+
+    payload = {"module": "flashml_workloads.sklearn_trial"}
+    if local_inputs is not None:
+        payload["local_inputs"] = local_inputs
+    payload.update(payload_extra)
+    return TaskSpec(
+        task_id="task-000", job_id="job-a", commit_key="job-a/task-000/m.json",
+        payload=payload,
+    )
+
+
+def test_node_advertising_a_superset_is_eligible():
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    node = {"node_id": "n1", "local_datasets": ["patients", "labs"]}
+    assert IsolationAwarePlacement().eligible(_local_task(["patients"]), node) is True
+
+
+def test_node_advertising_every_required_name_is_eligible():
+    """Two requirements, both advertised — the gate is per-name, not per-task."""
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    node = {"node_id": "n1", "local_datasets": ["labs", "patients"]}
+    task = _local_task(["patients", "labs"])
+    assert IsolationAwarePlacement().eligible(task, node) is True
+
+
+def test_node_advertising_only_some_required_names_is_ineligible():
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    node = {"node_id": "n1", "local_datasets": ["patients"]}
+    task = _local_task(["patients", "labs"])
+    assert IsolationAwarePlacement().eligible(task, node) is False
+
+
+def test_node_advertising_a_different_dataset_is_ineligible():
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    node = {"node_id": "n1", "local_datasets": ["labs"]}
+    assert IsolationAwarePlacement().eligible(_local_task(["patients"]), node) is False
+
+
+@pytest.mark.parametrize(
+    "advertised",
+    [
+        pytest.param([], id="empty-list"),
+        pytest.param(None, id="explicit-none"),
+        pytest.param("patients", id="bare-string-type-confusion"),
+        pytest.param({"patients": "/srv/data"}, id="dict-type-confusion"),
+        pytest.param(("patients",), id="tuple-type-confusion"),
+        pytest.param(True, id="truthy-boolean"),
+    ],
+)
+def test_ineligible_unless_the_capability_is_a_list_containing_the_name(advertised):
+    """Fail closed on absence AND on type confusion. The bare string
+    "patients" must not count: `"patients" in "patients"` is True in Python,
+    which is exactly the substring trap `sandbox_capable is True` avoids for
+    booleans. Only a genuine list of names advertises anything."""
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    node = {"node_id": "n1", "local_datasets": advertised}
+    assert IsolationAwarePlacement().eligible(_local_task(["patients"]), node) is False
+
+
+def test_node_with_no_local_datasets_field_at_all_is_ineligible():
+    """An already-deployed agent whose registration predates the field
+    advertises nothing — absent must read as "offers nothing", never as
+    "offers anything"."""
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    node = {"node_id": "n1"}  # no local_datasets key at all
+    assert IsolationAwarePlacement().eligible(_local_task(["patients"]), node) is False
+
+
+def test_allow_fallback_cannot_bypass_the_local_data_gate():
+    """allowFallback waives the sandbox-tier requirement only. It must not also
+    waive the local-data gate — the same reasoning that stops it waiving the
+    argv gate: a submitter cannot grant themselves data a host never offered."""
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    task = _local_task(
+        ["patients"], isolation={"tier": "sandboxed", "allowFallback": True}
+    )
+    node = {"node_id": "n1", "sandbox_capable": False, "local_datasets": []}
+    assert IsolationAwarePlacement().eligible(task, node) is False
+
+
+def test_allow_fallback_does_not_bypass_it_on_a_node_missing_the_field():
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    task = _local_task(
+        ["patients"], isolation={"tier": "sandboxed", "allowFallback": True}
+    )
+    assert IsolationAwarePlacement().eligible(task, {"node_id": "n1"}) is False
+
+
+def test_type_confused_local_inputs_is_ineligible_everywhere():
+    """A poisoned requirement (present but not a list, e.g. the bare string
+    "patients") must fail closed on every node — including one that advertises
+    the name — and must never crash the predicate."""
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    policy = IsolationAwarePlacement()
+    node = {"node_id": "n1", "local_datasets": ["patients"]}
+    assert policy.eligible(_local_task("patients"), node) is False
+    assert policy.eligible(_local_task({"patients": 1}), node) is False
+
+
+def test_tasks_without_local_inputs_are_unaffected():
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    policy = IsolationAwarePlacement()
+    task = _local_task(None)  # no local_inputs key in the payload at all
+    assert policy.eligible(task, {"node_id": "n1"}) is True
+    assert policy.eligible(task, {"node_id": "n2", "local_datasets": ["labs"]}) is True
+
+
+def test_empty_local_inputs_list_requires_nothing():
+    """`local_inputs: []` is what a job that opted out of the feature compiles
+    to — it demands nothing, so it runs anywhere, like tier "standard"."""
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    assert IsolationAwarePlacement().eligible(_local_task([]), {"node_id": "n1"}) is True
+
+
+def test_claim_over_a_local_data_queue_serves_the_clean_task():
+    """A local-data task at the head of the queue must not block a node that
+    cannot serve it — the next clean task is still leased, and the local-data
+    task is never mis-leased."""
+    from flashruntime.leases import LeaseManager
+    from flashruntime.protocol.v1alpha1 import TaskSpec
+    from flashruntime.scheduler import IsolationAwarePlacement
+
+    mgr = LeaseManager()
+    mgr.add_task(_local_task(["patients"]))  # head of queue, needs data
+    mgr.add_task(
+        TaskSpec(task_id="task-001", job_id="job-a", commit_key="job-a/task-001/m.json")
+    )
+    policy = IsolationAwarePlacement()
+
+    lease = mgr.claim("n1", policy=policy, node={"node_id": "n1"})
+    assert lease is not None
+    assert lease.task_id == "task-001"
+
+    holder = {"node_id": "n2", "local_datasets": ["patients"]}
+    lease2 = mgr.claim("n2", policy=policy, node=holder)
+    assert lease2 is not None
+    assert lease2.task_id == "task-000"
+
+
+def test_claim_endpoint_leases_local_data_work_only_to_the_holder():
+    """End to end over the claim endpoint: the registry's claim-time node view
+    must forward the advertised names, or the gate reads an absent capability
+    on EVERY node and a local-data task becomes unplaceable everywhere — fail
+    closed, but also permanently stuck."""
+    import pathlib
+
+    import fastapi
+    from fastapi.testclient import TestClient
+
+    from flashruntime.leases import LeaseManager
+    from flashruntime.service.modea import ModeAState, build_router
+
+    state = ModeAState(LeaseManager(), artifacts_dir=pathlib.Path("/tmp"))
+    app = fastapi.FastAPI()
+    app.include_router(build_router(state))
+    client = TestClient(app)
+    state.manager.add_task(_local_task(["patients"]))
+
+    def register(node_id: str, local_datasets: list[str]):
+        r = client.post(
+            "/v1alpha1/nodes/register",
+            json={
+                "node_id": node_id, "kubernetes_node": "", "hostname": node_id,
+                "capabilities": {}, "local_datasets": local_datasets,
+            },
+        )
+        assert r.status_code == 200
+
+    register("no-data-node", [])
+    register("holder-node", ["patients", "labs"])
+
+    assert client.post("/v1alpha1/leases/claim", json={"node_id": "no-data-node"}).status_code == 204
+    r = client.post("/v1alpha1/leases/claim", json={"node_id": "holder-node"})
+    assert r.status_code == 200
+    assert r.json()["task_id"] == "task-000"

flashruntime-0.4.0/tests/test_protocol_local_datasets.py

@@ -0,0 +1,52 @@
+"""`NodeRegistration.local_datasets` — the names of on-host datasets a node
+advertises, and the fail-closed defaulting that governs them.
+
+These are protocol tests, not scheduler tests: they pin the *wire* behaviour
+that the placement gate is later allowed to trust. The gate can only fail
+closed if the field itself does.
+"""
+
+from flashruntime.protocol.v1alpha1 import NodeRegistration
+
+
+def make_registration(**overrides):
+    base = {
+        "node_id": "node-a",
+        "kubernetes_node": "worker-0",
+        "hostname": "worker-0.local",
+        "capabilities": {"cpu_cores": 4, "os": "linux", "architecture": "amd64"},
+    }
+    base.update(overrides)
+    return base
+
+
+def test_local_datasets_defaults_empty_when_absent_from_the_wire():
+    # An agent deployed before this field existed sends a registration with no
+    # `local_datasets` key at all. It must decode as advertising nothing, so it
+    # is never eligible for local-data work until it is upgraded and opts in
+    # explicitly (AGENTS.md rule 3: security-relevant fields fail closed).
+    reg = NodeRegistration.model_validate(make_registration())
+    assert reg.local_datasets == []
+
+
+def test_local_datasets_round_trips_through_json():
+    reg = NodeRegistration.model_validate(
+        make_registration(local_datasets=["patients", "labs"])
+    )
+    restored = NodeRegistration.model_validate_json(reg.model_dump_json())
+    assert restored.local_datasets == ["patients", "labs"]
+
+
+def test_local_datasets_default_is_not_shared_between_instances():
+    # A bare `[]` default would be one list shared by every NodeRegistration
+    # ever constructed: one node appending to it would silently advertise its
+    # datasets on behalf of every other node in the pool. `default_factory`
+    # is what stops that, and this test is what stops someone "simplifying"
+    # it back.
+    first = NodeRegistration.model_validate(make_registration(node_id="node-a"))
+    second = NodeRegistration.model_validate(make_registration(node_id="node-b"))
+
+    assert first.local_datasets is not second.local_datasets
+
+    first.local_datasets.append("patients")
+    assert second.local_datasets == []

flashruntime-0.4.0/tests/test_recipe_local_inputs.py

@@ -0,0 +1,68 @@
+"""`local_inputs` must survive the JobSpec -> task payload hop.
+
+The feature is built at both ends of this hop and was broken in the middle.
+`flashml.yaml` puts `local_inputs` into the workload parameters, and
+`IsolationAwarePlacement` reads it from `task.payload` — but `CommandRecipe`
+builds each payload from a fixed key list and dropped every parameter it did
+not recognise, `local_inputs` among them.
+
+The failure mode is the dangerous direction. The gate sees no requirement, so
+it does not fail closed and refuse the node — it fails OPEN, places the task
+anywhere, and flashnode mounts nothing. The task then runs without the data it
+asked for.
+
+Nothing caught it because both ends' tests construct the intermediate value
+directly: the compile tests assert the parameter lands in the JobSpec, and the
+placement tests build a payload containing `local_inputs` by hand. Each end was
+correct about itself. This test is about the hop between them.
+"""
+from __future__ import annotations
+
+from flashruntime.protocol.v1alpha1 import JobSpec
+from flashruntime.recipes.command import CommandRecipe
+
+
+def _spec(**params) -> JobSpec:
+    return JobSpec.model_validate({
+        "metadata": {"name": "local-data-job"},
+        "spec": {
+            "image": {"repository": "ghcr.io/zolli-labs/flashml-python-slim",
+                      "tag": "2026.08.1"},
+            "workload": {
+                "type": "command",
+                "parameters": {"command": ["python", "train.py"], **params},
+            },
+            "resources": {"minimumWorkers": 1, "maximumWorkers": 1},
+            "isolation": {"tier": "sandboxed", "allowFallback": False},
+        },
+    })
+
+
+def test_local_inputs_reaches_the_task_payload():
+    """Where the placement gate actually reads it."""
+    tasks = CommandRecipe().expand("job-1", _spec(local_inputs=["patients"]))
+
+    assert tasks, "expected at least one task"
+    for task in tasks:
+        assert task.payload.get("local_inputs") == ["patients"], (
+            "the placement gate reads task.payload['local_inputs']; without it "
+            "the gate fails OPEN and the task is placed on a node with no data"
+        )
+
+
+def test_absent_local_inputs_stays_absent():
+    """Never an empty list — same reasoning as `unpack_inputs` beside it.
+
+    An omitted key is the path where nothing about placement changes, and
+    emitting `[]` would mean the same thing today while quietly ceasing to
+    exercise that path."""
+    for task in CommandRecipe().expand("job-1", _spec()):
+        assert "local_inputs" not in task.payload
+
+
+def test_local_inputs_is_copied_not_aliased():
+    """A payload must not share a mutable list with the caller's spec."""
+    names = ["patients"]
+    tasks = CommandRecipe().expand("job-1", _spec(local_inputs=names))
+    names.append("labs")
+    assert tasks[0].payload["local_inputs"] == ["patients"]