flashnode 0.3.1__tar.gz → 0.3.3__tar.gz

{flashnode-0.3.1/flashnode.egg-info → flashnode-0.3.3}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: flashnode
-Version: 0.3.1
+Version: 0.3.3
 Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: flashruntime<0.5,>=0.4
+Requires-Dist: flashruntime<0.5,>=0.4.2
 Requires-Dist: psutil>=5.9
 Requires-Dist: websockets>=12
 Requires-Dist: cryptography>=42
@@ -65,6 +65,27 @@ image pull — a registry blip should not stop an agent whose images are
 already cached — and refuses to start if anything fails, because a host that
 cannot run tasks should not be claiming them.
 
+## While it runs
+
+On a terminal you get a live status block:
+
+```
+flashnode 0.3.2 · flashml-api.onrender.com · up 2h14m
+  running    fed-2e2d4d6ab57f  ·  attempt 1  ·  38s
+  session    12 accepted   0 failed
+  heartbeat  2s ago
+```
+
+`waiting · no work queued — this is normal` means exactly that: the pool has
+nothing for you right now, and your machine is fine. Pipe the output
+anywhere, or pass `--log-json`, and you get the machine-readable log instead.
+
+If three tasks in a row fail **on your machine**, the agent re-runs its own
+checks. Pass, and the jobs were broken rather than your host, so it carries
+on. Fail, and it stops claiming and tells you what to fix — instead of
+burning a job's retries on a machine that cannot run anything.
+`--max-consecutive-failures 0` turns that off.
+
 ## What it does today
 
 ```bash

flashnode-0.3.1/PKG-INFO → flashnode-0.3.3/README.md

@@ -1,20 +1,3 @@
-Metadata-Version: 2.4
-Name: flashnode
-Version: 0.3.1
-Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
-License: Apache-2.0
-Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: flashruntime<0.5,>=0.4
-Requires-Dist: psutil>=5.9
-Requires-Dist: websockets>=12
-Requires-Dist: cryptography>=42
-Provides-Extra: dev
-Requires-Dist: pytest; extra == "dev"
-Dynamic: license-file
-
 # FlashNode
 
 > **The open host agent of the FlashML system.** Install FlashNode on a
@@ -65,6 +48,27 @@ image pull — a registry blip should not stop an agent whose images are
 already cached — and refuses to start if anything fails, because a host that
 cannot run tasks should not be claiming them.
 
+## While it runs
+
+On a terminal you get a live status block:
+
+```
+flashnode 0.3.2 · flashml-api.onrender.com · up 2h14m
+  running    fed-2e2d4d6ab57f  ·  attempt 1  ·  38s
+  session    12 accepted   0 failed
+  heartbeat  2s ago
+```
+
+`waiting · no work queued — this is normal` means exactly that: the pool has
+nothing for you right now, and your machine is fine. Pipe the output
+anywhere, or pass `--log-json`, and you get the machine-readable log instead.
+
+If three tasks in a row fail **on your machine**, the agent re-runs its own
+checks. Pass, and the jobs were broken rather than your host, so it carries
+on. Fail, and it stops claiming and tells you what to fix — instead of
+burning a job's retries on a machine that cannot run anything.
+`--max-consecutive-failures 0` turns that off.
+
 ## What it does today
 
 ```bash

{flashnode-0.3.1 → flashnode-0.3.3}/flashnode/agent/cli.py

@@ -180,6 +180,16 @@ def _work(args: list[str]) -> int:
     )
     parser.add_argument("--max-tasks", type=int, default=None)
     parser.add_argument("--poll-seconds", type=float, default=1.0)
+    parser.add_argument(
+        "--log-json", action="store_true",
+        help="keep the machine-readable JSON log instead of the live status view",
+    )
+    parser.add_argument(
+        "--max-consecutive-failures", type=int,
+        default=int(os.environ.get("FLASHNODE_MAX_CONSECUTIVE_FAILURES", "3")),
+        help="host-side failures in a row before re-checking this machine and "
+             "stopping if it is broken (0 disables)",
+    )
     opts = parser.parse_args(args)
 
     runner = None
@@ -214,10 +224,18 @@ def _work(args: list[str]) -> int:
         # someone else's machine, and a transient registry blip must not
         # stop one whose images are already cached. `flashnode doctor` does
         # the pull.
-        from flashnode.doctor import format_results, run_checks
+        #
+        # The gate reads doctor's own NON_BLOCKING_STATUSES rather than
+        # testing `!= "ok"` here. Not every check is a gate: the GPU check is
+        # informational, because most volunteers have no GPU and blocking on
+        # it would lock the entire existing fleet out of CPU work the moment
+        # they upgraded. Keeping the predicate in one place is what stops
+        # `flashnode work` and `flashnode doctor` disagreeing about which
+        # verdicts are fatal.
+        from flashnode.doctor import NON_BLOCKING_STATUSES, format_results, run_checks
 
         results = run_checks(pull=False)
-        if any(r.status != "ok" for r in results):
+        if any(r.status not in NON_BLOCKING_STATUSES for r in results):
             print(
                 f"flashnode work: this machine cannot run tasks with "
                 f"--runner {opts.runner}.\n" + format_results(results)
@@ -264,10 +282,29 @@ def _work(args: list[str]) -> int:
         module_capable=(opts.runner != "argv"),
     )
     client.register(registration)
+    from flashnode.doctor import NON_BLOCKING_STATUSES, run_checks
+
+    def _blocking_problems():
+        """What the loop calls after a streak of host-side failures.
+
+        Filtered HERE, with the same set the startup gate reads, so the two
+        cannot drift — and so `loop.py` never has to import the doctor
+        (which would close a loop -> doctor -> executor -> loop cycle).
+        The GPU check reports "info" and never fails; a loop testing
+        `!= "ok"` itself would quarantine every CPU-only volunteer.
+
+        pull=False for the same reason the startup gate uses it: a registry
+        blip must not stop an agent whose images are already cached.
+        """
+        return [r for r in run_checks(pull=False)
+                if r.status not in NON_BLOCKING_STATUSES]
+
     loop = ExecutorLoop(
         client, node_id, runner=runner,
         poll_seconds=opts.poll_seconds, workdir_base=workdir_base,
         registration=registration,  # survives coordinator restarts
+        health_check=_blocking_problems,
+        max_consecutive_failures=opts.max_consecutive_failures,
     )
 
     def _stop(signum, frame):  # noqa: ARG001
@@ -275,7 +312,35 @@ def _work(args: list[str]) -> int:
 
     signal.signal(signal.SIGTERM, _stop)
     signal.signal(signal.SIGINT, _stop)
-    accepted = loop.run(max_tasks=opts.max_tasks)
+    view = None
+    if sys.stdout.isatty() and not opts.log_json:
+        # Two writers redrawing one terminal is unreadable, so the JSON
+        # handler goes when the view arrives. --log-json keeps it, and a
+        # non-TTY never gets here — ANSI cursor movement into a pipe or a
+        # systemd journal is corruption, not output.
+        from flashnode.status import StatusView
+
+        logging.getLogger().handlers.clear()
+        view = StatusView(loop, coordinator=opts.coordinator,
+                          version=__version__, stream=sys.stdout)
+        view.start()
+    try:
+        accepted = loop.run(max_tasks=opts.max_tasks)
+    finally:
+        if view is not None:
+            view.stop()
+
+    if getattr(loop, "quarantined", False):
+        from flashnode.doctor import format_results
+
+        print(
+            "\nflashnode work: stopping — this machine can no longer run "
+            "tasks.\n" + format_results(loop.health_report or [])
+            + "\n\nFix the above, then `flashnode doctor` to confirm before "
+              "restarting.",
+            file=sys.stderr,
+        )
+        return 2
     print(f"flashnode work: {accepted} task(s) accepted", file=sys.stderr)
     return 0
 

{flashnode-0.3.1 → flashnode-0.3.3}/flashnode/doctor.py

@@ -34,12 +34,16 @@ from flashnode.executor.hardening import (
     _bind_mount_source,
     harden_args,
 )
+from flashnode.inventory.gpu import probe_gpus
 
 __all__ = [
+    "IMAGE_TAG",
+    "NON_BLOCKING_STATUSES",
     "PROBE_IMAGE",
     "CheckResult",
     "check_cli_on_path",
     "check_engine",
+    "check_gpus",
     "check_hardened_run",
     "check_local_datasets",
     "check_pull",
@@ -52,15 +56,48 @@ __all__ = [
     "run_command",
 ]
 
+#: The curated-image tag this agent probes. It must equal `IMAGE_TAG` in
+#: `.github/workflows/images.yml` (which builds the images) and in
+#: flashml-cloud's `apps/api/flashml_cloud_api/images.py` (which hands
+#: references to jobs). All three are bumped together; the workflow's
+#: immutability guard refuses to repush an existing tag, so a bump is
+#: mandatory for any image change, never cosmetic.
+#:
+#: Named rather than baked into the string below because the drift is
+#: SILENT: old tags keep pulling forever, so a doctor probing an image two
+#: releases behind the fleet passes every check and certifies a host against
+#: an image no job will ever use. Nothing in either repo tests that the three
+#: agree — the test that would have is gone, deleted on 2026-08-01 when the
+#: image sources moved to this repo. One greppable constant per repo is what
+#: is left holding this together.
+IMAGE_TAG = "2026.08.2"
+
 #: The image every container-level check runs. python-slim, never
 #: pytorch-cpu: registry auth, TLS and the credential helper are properties
 #: of the REGISTRY, so the smallest curated image proves the same thing, and
 #: making a volunteer download gigabytes to learn their helper is missing is
-#: a hostile diagnostic. Kept in step with flashml-cloud's published tags.
-PROBE_IMAGE = "ghcr.io/zolli-labs/flashml-python-slim:2026.08.1"
+#: a hostile diagnostic. Never pytorch-cuda either, for the same reason
+#: doubled: a GPU host that cannot pull has the same broken registry as
+#: everyone else, and would find out 3 GB later.
+PROBE_IMAGE = f"ghcr.io/zolli-labs/flashml-python-slim:{IMAGE_TAG}"
 
 CommandRunner = Callable[..., subprocess.CompletedProcess]
 
+#: Statuses that do NOT stop a host from working. Everything else does, and
+#: an unrecognised status counts as blocking — this is the certification
+#: predicate, so it fails closed (see `exit_code`).
+#:
+#: The set exists because "info" does. Six of the seven checks answer "can
+#: this machine run a task at all", and blocking is the only correct answer
+#: when one of them fails. The GPU check answers something else — "is the
+#: hardware you think you contributed actually visible" — and most volunteers
+#: have no GPU and must keep taking CPU work. Blocking on that would lock the
+#: entire existing fleet out of the network on upgrade.
+#:
+#: `flashnode work` reads THIS set rather than testing `!= "ok"` itself, so
+#: the gate cannot drift from the exit code.
+NON_BLOCKING_STATUSES = frozenset({"ok", "info"})
+
 
 @dataclass(frozen=True)
 class CheckResult:
@@ -71,7 +108,7 @@ class CheckResult:
     """
 
     name: str
-    status: str  # "ok" | "fail" | "skip"
+    status: str  # "ok" | "fail" | "skip" | "info" (see NON_BLOCKING_STATUSES)
     detail: str = ""
     fix: str = ""
 
@@ -353,6 +390,44 @@ def check_local_datasets(raw: str | None = None) -> CheckResult:
     return CheckResult(name, "ok", detail=", ".join(sorted(mapping)))
 
 
+def check_gpus(probe: Callable[[], list] | None = None) -> CheckResult:
+    """What this host will advertise as GPUs — and NOTHING is a fine answer.
+
+    The only non-gating check in this module (spec §8). It never returns
+    "fail": a host with no GPU is the normal host, and the CPU work they
+    signed up for is unaffected. It exists for the other case — a host who
+    BELIEVES they contributed a GPU learning here that the agent cannot see
+    it, rather than wondering why no GPU job ever arrives. That is exactly
+    the class of silent failure `doctor` was written for.
+
+    It reports the same probe `discover()` advertises, not `nvidia-smi`
+    directly. A doctor that consults a different source than the agent is a
+    doctor that can pass while the agent registers `[]`.
+    """
+    name = "GPU devices"
+    probe = probe or probe_gpus
+    try:
+        gpus = list(probe())
+    except Exception as exc:  # noqa: BLE001
+        # probe_gpus already promises this cannot happen. If it ever does,
+        # a diagnostic that crashes has diagnosed nothing — and crashing on
+        # the one check that was never allowed to block would be perverse.
+        return CheckResult(name, "info", detail=f"could not probe for GPUs: {exc}")
+    if not gpus:
+        return CheckResult(
+            name, "info",
+            detail="no GPU detected — this host will take CPU work only",
+            fix="Nothing to do unless you expected a GPU here. If you did: "
+                "check `nvidia-smi` runs in this terminal, and install the "
+                "NVIDIA Container Toolkit so containers can see the device.",
+        )
+    lines = [f"{len(gpus)} GPU{'s' if len(gpus) != 1 else ''}"]
+    for gpu in gpus:
+        memory = f", {gpu.memory_total_mb} MiB" if gpu.memory_total_mb else ""
+        lines.append(f"{gpu.index}: {gpu.name or 'unnamed device'}{memory}")
+    return CheckResult(name, "ok", detail="\n".join(lines))
+
+
 def run_checks(
     *,
     pull: bool,
@@ -360,6 +435,7 @@ def run_checks(
     which: Callable[[str], str | None] | None = None,
     workdir: Path | None = None,
     raw_local_data: str | None = None,
+    gpu_probe: Callable[[], list] | None = None,
 ) -> list[CheckResult]:
     """Run every check, in order, skipping what a prior failure makes
     meaningless.
@@ -396,13 +472,18 @@ def run_checks(
         if results[-1].status != "ok":
             stopped = True
     results.append(check_local_datasets(raw=raw_local_data))
+    # Neither of the last two depends on Docker, so neither is skipped by a
+    # failure above: a host debugging their engine should still learn that
+    # their local-data label is a typo and that their GPU is invisible.
+    results.append(check_gpus(probe=gpu_probe))
     return results
 
 
 def format_results(results: Sequence[CheckResult]) -> str:
     lines = []
     for r in results:
-        tag = {"ok": "[ok]  ", "fail": "[FAIL]", "skip": "[skip]"}[r.status]
+        tag = {"ok": "[ok]  ", "fail": "[FAIL]", "skip": "[skip]",
+               "info": "[info]"}.get(r.status, "[????]")
         head = r.detail.splitlines()[0] if r.detail else ""
         lines.append(f"  {tag} {r.name:<30} {head}".rstrip())
         for extra in r.detail.splitlines()[1:]:
@@ -429,8 +510,14 @@ def format_results(results: Sequence[CheckResult]) -> str:
 def exit_code(results: Sequence[CheckResult]) -> int:
     """Skipped counts as not-passed. A host whose checks did not run has not
     been certified, and calling it healthy is the exact failure this command
-    removes."""
-    return 0 if all(r.status == "ok" for r in results) else 1
+    removes.
+
+    "info" does not count against a host — see NON_BLOCKING_STATUSES. The
+    membership test is deliberately the way round that makes an unknown
+    status BLOCK: a check whose verdict this function does not understand
+    has not certified anything.
+    """
+    return 0 if all(r.status in NON_BLOCKING_STATUSES for r in results) else 1
 
 
 def doctor_main(argv: list[str]) -> int:

{flashnode-0.3.1 → flashnode-0.3.3}/flashnode/executor/argv_runner.py

@@ -20,6 +20,7 @@ import re
 import subprocess
 from pathlib import Path
 
+from flashnode.executor.evidence import image_digest
 from flashnode.executor.hardening import container_name, harden_args
 from flashnode.executor.images import DEFAULT_ALLOWED_IMAGE_PREFIXES, image_is_allowed
 from flashnode.executor.runner import TaskExecutionError
@@ -43,8 +44,14 @@ class ArgvDockerRunner:
         self.memory_gb = memory_gb
         self.timeout_seconds = timeout_seconds
         self.max_output_bytes = max_output_bytes
+        # What the LAST run measured, read by ExecutorLoop for
+        # ExecutionEvidence — see SubprocessRunner for why both reset below.
+        self.last_exit_code: int | None = None
+        self.last_image_digest: str = ""
 
     def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
+        self.last_exit_code = None
+        self.last_image_digest = ""
         argv = payload.get("argv")
         if not argv or not isinstance(argv, list) or not all(isinstance(t, str) for t in argv):
             raise TaskExecutionError("payload 'argv' must be a non-empty list of strings")
@@ -76,6 +83,10 @@ class ArgvDockerRunner:
                 # here (before any subprocess) if this host does not lend them
                 # — see hardening.local_data_mounts.
                 local_inputs=payload.get("local_inputs"),
+                # The device count the job asked for. Absent for every job
+                # that does not ask, and validated in hardening.gpu_flags
+                # rather than here, so both runners cannot drift on it.
+                gpus=payload.get("gpus"),
             ),
             *env_args,
             image,          # argv follows the image, where docker treats it
@@ -101,9 +112,12 @@ class ArgvDockerRunner:
             # (a subclass of OSError). Degrade to a failed task, not a dead
             # agent — execute_one only catches TaskExecutionError/LeaseLost.
             raise TaskExecutionError(f"docker is unavailable: {exc}") from exc
+        self.last_exit_code = proc.returncode
         if proc.returncode != 0:
             tail = proc.stderr.decode(errors="replace")[-800:]
             raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
+        # Asked of the daemon, not read off the payload — see DockerRunner.
+        self.last_image_digest = image_digest(image)
 
         # metrics.json is load-bearing, not a preference: CommandRecipe sets
         # commit_key to <prefix>/metrics.json and the coordinator validates

{flashnode-0.3.1 → flashnode-0.3.3}/flashnode/executor/client.py

@@ -19,7 +19,12 @@ import urllib.error
 import urllib.request
 from pathlib import Path
 
-from flashruntime.protocol.v1alpha1 import Lease, NodeHeartbeat, NodeRegistration
+from flashruntime.protocol.v1alpha1 import (
+    ExecutionEvidence,
+    Lease,
+    NodeHeartbeat,
+    NodeRegistration,
+)
 
 
 class LeaseLost(Exception):
@@ -122,11 +127,25 @@ class CoordinatorClient:
         if status != 200:
             raise RuntimeError(f"attempt heartbeat failed ({status}): {body}")
 
-    def complete(self, lease_id: str, output_sha256: str) -> bool:
+    def complete(
+        self,
+        lease_id: str,
+        output_sha256: str,
+        evidence: ExecutionEvidence | None = None,
+    ) -> bool:
+        """Commit this attempt, optionally saying what the run looked like.
+
+        `evidence` is a keyword with a default so every existing caller keeps
+        working, and the body is byte-for-byte the one previous releases sent
+        when there is nothing to report — this agent may be talking to a
+        coordinator that predates the field, and an empty block is not the
+        same statement as no block.
+        """
+        payload: dict = {"output_sha256": output_sha256}
+        if evidence is not None:
+            payload["evidence"] = evidence.model_dump(mode="json")
         status, body = self._json(
-            "POST",
-            f"/v1alpha1/attempts/{lease_id}/complete",
-            {"output_sha256": output_sha256},
+            "POST", f"/v1alpha1/attempts/{lease_id}/complete", payload
         )
         if status != 200:
             raise RuntimeError(f"complete failed ({status}): {body}")

{flashnode-0.3.1 → flashnode-0.3.3}/flashnode/executor/docker_runner.py

@@ -19,6 +19,7 @@ import json
 import subprocess
 from pathlib import Path
 
+from flashnode.executor.evidence import image_digest
 from flashnode.executor.hardening import CONTAINER_WORKDIR, container_name, harden_args
 from flashnode.executor.images import DEFAULT_ALLOWED_IMAGE_PREFIXES, image_is_allowed
 from flashnode.executor.runner import DEFAULT_ALLOWED_MODULES, TaskExecutionError
@@ -44,8 +45,14 @@ class DockerRunner:
         self.cpus = cpus
         self.memory_gb = memory_gb
         self.timeout_seconds = timeout_seconds
+        # What the LAST run measured, read by ExecutorLoop for
+        # ExecutionEvidence — see SubprocessRunner for why both reset below.
+        self.last_exit_code: int | None = None
+        self.last_image_digest: str = ""
 
     def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
+        self.last_exit_code = None
+        self.last_image_digest = ""
         module = payload.get("module", "")
         if module not in self.allowed_modules:
             raise TaskExecutionError(f"module {module!r} is not allowlisted — refusing to run")
@@ -80,6 +87,11 @@ class DockerRunner:
                 # See hardening.local_data_mounts: read-only, only what the
                 # payload named, and a refusal if this host does not lend it.
                 local_inputs=payload.get("local_inputs"),
+                # The device count the job asked for — see
+                # hardening.gpu_flags. Both runners forward this; a runner
+                # that quietly does not is a GPU job silently running on the
+                # CPU, which still succeeds and so tells nobody.
+                gpus=payload.get("gpus"),
             ),
             image,
             "python", "-m", module,
@@ -105,9 +117,14 @@ class DockerRunner:
             # (a subclass of OSError). Degrade to a failed task, not a dead
             # agent — execute_one only catches TaskExecutionError/LeaseLost.
             raise TaskExecutionError(f"docker is unavailable: {exc}") from exc
+        self.last_exit_code = proc.returncode
         if proc.returncode != 0:
             tail = proc.stderr.decode(errors="replace")[-800:]
             raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
+        # Asked of the daemon, not read off the payload: the point of the
+        # field is which bytes ran here, and the payload only says which
+        # bytes were asked for. "" when the daemon will not say.
+        self.last_image_digest = image_digest(image)
         if not (outdir / "metrics.json").is_file():
             raise TaskExecutionError("task produced no metrics.json — nothing to commit")
         return outdir