flashnode 0.3.1__tar.gz → 0.3.2__tar.gz

{flashnode-0.3.1/flashnode.egg-info → flashnode-0.3.2}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: flashnode
-Version: 0.3.1
+Version: 0.3.2
 Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: flashruntime<0.5,>=0.4
+Requires-Dist: flashruntime<0.5,>=0.4.1
 Requires-Dist: psutil>=5.9
 Requires-Dist: websockets>=12
 Requires-Dist: cryptography>=42
@@ -65,6 +65,27 @@ image pull — a registry blip should not stop an agent whose images are
 already cached — and refuses to start if anything fails, because a host that
 cannot run tasks should not be claiming them.
 
+## While it runs
+
+On a terminal you get a live status block:
+
+```
+flashnode 0.3.2 · flashml-api.onrender.com · up 2h14m
+  running    fed-2e2d4d6ab57f  ·  attempt 1  ·  38s
+  session    12 accepted   0 failed
+  heartbeat  2s ago
+```
+
+`waiting · no work queued — this is normal` means exactly that: the pool has
+nothing for you right now, and your machine is fine. Pipe the output
+anywhere, or pass `--log-json`, and you get the machine-readable log instead.
+
+If three tasks in a row fail **on your machine**, the agent re-runs its own
+checks. Pass, and the jobs were broken rather than your host, so it carries
+on. Fail, and it stops claiming and tells you what to fix — instead of
+burning a job's retries on a machine that cannot run anything.
+`--max-consecutive-failures 0` turns that off.
+
 ## What it does today
 
 ```bash

flashnode-0.3.1/PKG-INFO → flashnode-0.3.2/README.md

@@ -1,20 +1,3 @@
-Metadata-Version: 2.4
-Name: flashnode
-Version: 0.3.1
-Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
-License: Apache-2.0
-Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: flashruntime<0.5,>=0.4
-Requires-Dist: psutil>=5.9
-Requires-Dist: websockets>=12
-Requires-Dist: cryptography>=42
-Provides-Extra: dev
-Requires-Dist: pytest; extra == "dev"
-Dynamic: license-file
-
 # FlashNode
 
 > **The open host agent of the FlashML system.** Install FlashNode on a
@@ -65,6 +48,27 @@ image pull — a registry blip should not stop an agent whose images are
 already cached — and refuses to start if anything fails, because a host that
 cannot run tasks should not be claiming them.
 
+## While it runs
+
+On a terminal you get a live status block:
+
+```
+flashnode 0.3.2 · flashml-api.onrender.com · up 2h14m
+  running    fed-2e2d4d6ab57f  ·  attempt 1  ·  38s
+  session    12 accepted   0 failed
+  heartbeat  2s ago
+```
+
+`waiting · no work queued — this is normal` means exactly that: the pool has
+nothing for you right now, and your machine is fine. Pipe the output
+anywhere, or pass `--log-json`, and you get the machine-readable log instead.
+
+If three tasks in a row fail **on your machine**, the agent re-runs its own
+checks. Pass, and the jobs were broken rather than your host, so it carries
+on. Fail, and it stops claiming and tells you what to fix — instead of
+burning a job's retries on a machine that cannot run anything.
+`--max-consecutive-failures 0` turns that off.
+
 ## What it does today
 
 ```bash

{flashnode-0.3.1 → flashnode-0.3.2}/flashnode/agent/cli.py

@@ -180,6 +180,16 @@ def _work(args: list[str]) -> int:
     )
     parser.add_argument("--max-tasks", type=int, default=None)
     parser.add_argument("--poll-seconds", type=float, default=1.0)
+    parser.add_argument(
+        "--log-json", action="store_true",
+        help="keep the machine-readable JSON log instead of the live status view",
+    )
+    parser.add_argument(
+        "--max-consecutive-failures", type=int,
+        default=int(os.environ.get("FLASHNODE_MAX_CONSECUTIVE_FAILURES", "3")),
+        help="host-side failures in a row before re-checking this machine and "
+             "stopping if it is broken (0 disables)",
+    )
     opts = parser.parse_args(args)
 
     runner = None
@@ -214,10 +224,18 @@ def _work(args: list[str]) -> int:
         # someone else's machine, and a transient registry blip must not
         # stop one whose images are already cached. `flashnode doctor` does
         # the pull.
-        from flashnode.doctor import format_results, run_checks
+        #
+        # The gate reads doctor's own NON_BLOCKING_STATUSES rather than
+        # testing `!= "ok"` here. Not every check is a gate: the GPU check is
+        # informational, because most volunteers have no GPU and blocking on
+        # it would lock the entire existing fleet out of CPU work the moment
+        # they upgraded. Keeping the predicate in one place is what stops
+        # `flashnode work` and `flashnode doctor` disagreeing about which
+        # verdicts are fatal.
+        from flashnode.doctor import NON_BLOCKING_STATUSES, format_results, run_checks
 
         results = run_checks(pull=False)
-        if any(r.status != "ok" for r in results):
+        if any(r.status not in NON_BLOCKING_STATUSES for r in results):
             print(
                 f"flashnode work: this machine cannot run tasks with "
                 f"--runner {opts.runner}.\n" + format_results(results)
@@ -264,10 +282,29 @@ def _work(args: list[str]) -> int:
         module_capable=(opts.runner != "argv"),
     )
     client.register(registration)
+    from flashnode.doctor import NON_BLOCKING_STATUSES, run_checks
+
+    def _blocking_problems():
+        """What the loop calls after a streak of host-side failures.
+
+        Filtered HERE, with the same set the startup gate reads, so the two
+        cannot drift — and so `loop.py` never has to import the doctor
+        (which would close a loop -> doctor -> executor -> loop cycle).
+        The GPU check reports "info" and never fails; a loop testing
+        `!= "ok"` itself would quarantine every CPU-only volunteer.
+
+        pull=False for the same reason the startup gate uses it: a registry
+        blip must not stop an agent whose images are already cached.
+        """
+        return [r for r in run_checks(pull=False)
+                if r.status not in NON_BLOCKING_STATUSES]
+
     loop = ExecutorLoop(
         client, node_id, runner=runner,
         poll_seconds=opts.poll_seconds, workdir_base=workdir_base,
         registration=registration,  # survives coordinator restarts
+        health_check=_blocking_problems,
+        max_consecutive_failures=opts.max_consecutive_failures,
     )
 
     def _stop(signum, frame):  # noqa: ARG001
@@ -275,7 +312,35 @@ def _work(args: list[str]) -> int:
 
     signal.signal(signal.SIGTERM, _stop)
     signal.signal(signal.SIGINT, _stop)
-    accepted = loop.run(max_tasks=opts.max_tasks)
+    view = None
+    if sys.stdout.isatty() and not opts.log_json:
+        # Two writers redrawing one terminal is unreadable, so the JSON
+        # handler goes when the view arrives. --log-json keeps it, and a
+        # non-TTY never gets here — ANSI cursor movement into a pipe or a
+        # systemd journal is corruption, not output.
+        from flashnode.status import StatusView
+
+        logging.getLogger().handlers.clear()
+        view = StatusView(loop, coordinator=opts.coordinator,
+                          version=__version__, stream=sys.stdout)
+        view.start()
+    try:
+        accepted = loop.run(max_tasks=opts.max_tasks)
+    finally:
+        if view is not None:
+            view.stop()
+
+    if getattr(loop, "quarantined", False):
+        from flashnode.doctor import format_results
+
+        print(
+            "\nflashnode work: stopping — this machine can no longer run "
+            "tasks.\n" + format_results(loop.health_report or [])
+            + "\n\nFix the above, then `flashnode doctor` to confirm before "
+              "restarting.",
+            file=sys.stderr,
+        )
+        return 2
     print(f"flashnode work: {accepted} task(s) accepted", file=sys.stderr)
     return 0
 

{flashnode-0.3.1 → flashnode-0.3.2}/flashnode/doctor.py

@@ -34,12 +34,16 @@ from flashnode.executor.hardening import (
     _bind_mount_source,
     harden_args,
 )
+from flashnode.inventory.gpu import probe_gpus
 
 __all__ = [
+    "IMAGE_TAG",
+    "NON_BLOCKING_STATUSES",
     "PROBE_IMAGE",
     "CheckResult",
     "check_cli_on_path",
     "check_engine",
+    "check_gpus",
     "check_hardened_run",
     "check_local_datasets",
     "check_pull",
@@ -52,15 +56,48 @@ __all__ = [
     "run_command",
 ]
 
+#: The curated-image tag this agent probes. It must equal `IMAGE_TAG` in
+#: `.github/workflows/images.yml` (which builds the images) and in
+#: flashml-cloud's `apps/api/flashml_cloud_api/images.py` (which hands
+#: references to jobs). All three are bumped together; the workflow's
+#: immutability guard refuses to repush an existing tag, so a bump is
+#: mandatory for any image change, never cosmetic.
+#:
+#: Named rather than baked into the string below because the drift is
+#: SILENT: old tags keep pulling forever, so a doctor probing an image two
+#: releases behind the fleet passes every check and certifies a host against
+#: an image no job will ever use. Nothing in either repo tests that the three
+#: agree — the test that would have is gone, deleted on 2026-08-01 when the
+#: image sources moved to this repo. One greppable constant per repo is what
+#: is left holding this together.
+IMAGE_TAG = "2026.08.2"
+
 #: The image every container-level check runs. python-slim, never
 #: pytorch-cpu: registry auth, TLS and the credential helper are properties
 #: of the REGISTRY, so the smallest curated image proves the same thing, and
 #: making a volunteer download gigabytes to learn their helper is missing is
-#: a hostile diagnostic. Kept in step with flashml-cloud's published tags.
-PROBE_IMAGE = "ghcr.io/zolli-labs/flashml-python-slim:2026.08.1"
+#: a hostile diagnostic. Never pytorch-cuda either, for the same reason
+#: doubled: a GPU host that cannot pull has the same broken registry as
+#: everyone else, and would find out 3 GB later.
+PROBE_IMAGE = f"ghcr.io/zolli-labs/flashml-python-slim:{IMAGE_TAG}"
 
 CommandRunner = Callable[..., subprocess.CompletedProcess]
 
+#: Statuses that do NOT stop a host from working. Everything else does, and
+#: an unrecognised status counts as blocking — this is the certification
+#: predicate, so it fails closed (see `exit_code`).
+#:
+#: The set exists because "info" does. Six of the seven checks answer "can
+#: this machine run a task at all", and blocking is the only correct answer
+#: when one of them fails. The GPU check answers something else — "is the
+#: hardware you think you contributed actually visible" — and most volunteers
+#: have no GPU and must keep taking CPU work. Blocking on that would lock the
+#: entire existing fleet out of the network on upgrade.
+#:
+#: `flashnode work` reads THIS set rather than testing `!= "ok"` itself, so
+#: the gate cannot drift from the exit code.
+NON_BLOCKING_STATUSES = frozenset({"ok", "info"})
+
 
 @dataclass(frozen=True)
 class CheckResult:
@@ -71,7 +108,7 @@ class CheckResult:
     """
 
     name: str
-    status: str  # "ok" | "fail" | "skip"
+    status: str  # "ok" | "fail" | "skip" | "info" (see NON_BLOCKING_STATUSES)
     detail: str = ""
     fix: str = ""
 
@@ -353,6 +390,44 @@ def check_local_datasets(raw: str | None = None) -> CheckResult:
     return CheckResult(name, "ok", detail=", ".join(sorted(mapping)))
 
 
+def check_gpus(probe: Callable[[], list] | None = None) -> CheckResult:
+    """What this host will advertise as GPUs — and NOTHING is a fine answer.
+
+    The only non-gating check in this module (spec §8). It never returns
+    "fail": a host with no GPU is the normal host, and the CPU work they
+    signed up for is unaffected. It exists for the other case — a host who
+    BELIEVES they contributed a GPU learning here that the agent cannot see
+    it, rather than wondering why no GPU job ever arrives. That is exactly
+    the class of silent failure `doctor` was written for.
+
+    It reports the same probe `discover()` advertises, not `nvidia-smi`
+    directly. A doctor that consults a different source than the agent is a
+    doctor that can pass while the agent registers `[]`.
+    """
+    name = "GPU devices"
+    probe = probe or probe_gpus
+    try:
+        gpus = list(probe())
+    except Exception as exc:  # noqa: BLE001
+        # probe_gpus already promises this cannot happen. If it ever does,
+        # a diagnostic that crashes has diagnosed nothing — and crashing on
+        # the one check that was never allowed to block would be perverse.
+        return CheckResult(name, "info", detail=f"could not probe for GPUs: {exc}")
+    if not gpus:
+        return CheckResult(
+            name, "info",
+            detail="no GPU detected — this host will take CPU work only",
+            fix="Nothing to do unless you expected a GPU here. If you did: "
+                "check `nvidia-smi` runs in this terminal, and install the "
+                "NVIDIA Container Toolkit so containers can see the device.",
+        )
+    lines = [f"{len(gpus)} GPU{'s' if len(gpus) != 1 else ''}"]
+    for gpu in gpus:
+        memory = f", {gpu.memory_total_mb} MiB" if gpu.memory_total_mb else ""
+        lines.append(f"{gpu.index}: {gpu.name or 'unnamed device'}{memory}")
+    return CheckResult(name, "ok", detail="\n".join(lines))
+
+
 def run_checks(
     *,
     pull: bool,
@@ -360,6 +435,7 @@ def run_checks(
     which: Callable[[str], str | None] | None = None,
     workdir: Path | None = None,
     raw_local_data: str | None = None,
+    gpu_probe: Callable[[], list] | None = None,
 ) -> list[CheckResult]:
     """Run every check, in order, skipping what a prior failure makes
     meaningless.
@@ -396,13 +472,18 @@ def run_checks(
         if results[-1].status != "ok":
             stopped = True
     results.append(check_local_datasets(raw=raw_local_data))
+    # Neither of the last two depends on Docker, so neither is skipped by a
+    # failure above: a host debugging their engine should still learn that
+    # their local-data label is a typo and that their GPU is invisible.
+    results.append(check_gpus(probe=gpu_probe))
     return results
 
 
 def format_results(results: Sequence[CheckResult]) -> str:
     lines = []
     for r in results:
-        tag = {"ok": "[ok]  ", "fail": "[FAIL]", "skip": "[skip]"}[r.status]
+        tag = {"ok": "[ok]  ", "fail": "[FAIL]", "skip": "[skip]",
+               "info": "[info]"}.get(r.status, "[????]")
         head = r.detail.splitlines()[0] if r.detail else ""
         lines.append(f"  {tag} {r.name:<30} {head}".rstrip())
         for extra in r.detail.splitlines()[1:]:
@@ -429,8 +510,14 @@ def format_results(results: Sequence[CheckResult]) -> str:
 def exit_code(results: Sequence[CheckResult]) -> int:
     """Skipped counts as not-passed. A host whose checks did not run has not
     been certified, and calling it healthy is the exact failure this command
-    removes."""
-    return 0 if all(r.status == "ok" for r in results) else 1
+    removes.
+
+    "info" does not count against a host — see NON_BLOCKING_STATUSES. The
+    membership test is deliberately the way round that makes an unknown
+    status BLOCK: a check whose verdict this function does not understand
+    has not certified anything.
+    """
+    return 0 if all(r.status in NON_BLOCKING_STATUSES for r in results) else 1
 
 
 def doctor_main(argv: list[str]) -> int:

{flashnode-0.3.1 → flashnode-0.3.2}/flashnode/executor/argv_runner.py

@@ -76,6 +76,10 @@ class ArgvDockerRunner:
                 # here (before any subprocess) if this host does not lend them
                 # — see hardening.local_data_mounts.
                 local_inputs=payload.get("local_inputs"),
+                # The device count the job asked for. Absent for every job
+                # that does not ask, and validated in hardening.gpu_flags
+                # rather than here, so both runners cannot drift on it.
+                gpus=payload.get("gpus"),
             ),
             *env_args,
             image,          # argv follows the image, where docker treats it

{flashnode-0.3.1 → flashnode-0.3.2}/flashnode/executor/docker_runner.py

@@ -80,6 +80,11 @@ class DockerRunner:
                 # See hardening.local_data_mounts: read-only, only what the
                 # payload named, and a refusal if this host does not lend it.
                 local_inputs=payload.get("local_inputs"),
+                # The device count the job asked for — see
+                # hardening.gpu_flags. Both runners forward this; a runner
+                # that quietly does not is a GPU job silently running on the
+                # CPU, which still succeeds and so tells nobody.
+                gpus=payload.get("gpus"),
             ),
             image,
             "python", "-m", module,

{flashnode-0.3.1 → flashnode-0.3.2}/flashnode/executor/hardening.py

@@ -183,6 +183,30 @@ def local_data_mounts(
     return args
 
 
+def gpu_flags(gpus: object) -> list[str]:
+    """`--gpus N`, or nothing at all.
+
+    Every other flag in this module NARROWS what a container may touch.
+    This one widens it — it hands a device on someone else's machine to a
+    stranger's code — so it is absent unless the payload asked, and the ask
+    has to be a plain positive integer.
+
+    Anything else emits nothing rather than raising. The coordinator's
+    placement gate already refused a task whose `gpus` was not a positive
+    int, so a bad value arriving here means gate and payload disagree; this
+    is the second lock, not the validation. Two values are worth naming:
+
+    - `True`. `bool` is an `int` subclass, so a naive check reads it as
+      "1 GPU" — and `--gpus True` is not something docker accepts anyway.
+    - `"all"`. This is *valid* docker syntax meaning every device on the
+      host, which is exactly why an untrusted string must never reach the
+      flag. A task asks for a count; it does not get to name a device.
+    """
+    if isinstance(gpus, bool) or not isinstance(gpus, int) or gpus <= 0:
+        return []
+    return ["--gpus", str(gpus)]
+
+
 def harden_args(
     workdir: Path,
     *,
@@ -191,6 +215,7 @@ def harden_args(
     pids_limit: int = 512,
     local_inputs: object = None,
     local_data: dict[str, str] | None = None,
+    gpus: object = None,
 ) -> list[str]:
     """Docker flags common to every sandboxed task.
 
@@ -199,6 +224,11 @@ def harden_args(
     byte-for-byte what they were before the feature existed). `local_data` is
     the host owner's label→path map; it defaults to their environment, so a
     runner never has to know where the map comes from.
+
+    `gpus` is the payload's device count, straight from the wire and
+    therefore typed `object` — None for every job that exists today, and
+    then, as with `local_inputs`, the flag list is byte-for-byte what it was
+    before this argument existed. `doctor.check_hardened_run` passes neither.
     """
     return [
         # the job never reaches the volunteer's LAN or the internet; the
@@ -215,6 +245,9 @@ def harden_args(
         # bypassable by swapping
         "--memory", f"{memory_gb}g",
         "--memory-swap", f"{memory_gb}g",
+        # With the other resource caps, and absent entirely for the job that
+        # asked for no device — which is every job that exists today.
+        *gpu_flags(gpus),
         "--ulimit", "nofile=1024:1024",
         "-v", f"{_bind_mount_source(workdir)}:{CONTAINER_WORKDIR}",
         # After the workdir mount, never before: these land *inside* it, at

{flashnode-0.3.1 → flashnode-0.3.2}/flashnode/executor/loop.py

@@ -141,6 +141,8 @@ class ExecutorLoop:
         registration=None,  # NodeRegistration; enables re-register after coordinator restart
         max_unpacked_bytes: int = DEFAULT_MAX_BYTES,
         max_unpacked_members: int = DEFAULT_MAX_MEMBERS,
+        health_check=None,
+        max_consecutive_failures: int = 0,
     ):
         self.client = client
         self.node_id = node_id
@@ -159,6 +161,30 @@ class ExecutorLoop:
         self.stop_event = threading.Event()
         self.tasks_accepted = 0
         self._last_node_hb = 0.0
+        # Host-facing state. Plain attributes on purpose: flashnode/status.py
+        # reads them from a separate thread on a timer, and putting a lock in
+        # the claim path to protect a counter would be trading a correctness
+        # risk for a cosmetic one.
+        self.tasks_failed = 0
+        self.consecutive_failures = 0
+        self.current_task: str | None = None
+        self.current_attempt: int | None = None
+        self.current_task_started: float | None = None
+        self.quarantined = False
+        self.health_report: list | None = None
+        # INJECTED, never imported. `doctor.py` imports
+        # `flashnode.executor.hardening`, which initialises this package,
+        # whose __init__ imports this module — loop -> doctor -> executor ->
+        # loop is a cycle that resolves or explodes by import order, which is
+        # the worst kind of bug to ship to machines we cannot reach.
+        #
+        # CONTRACT: health_check() returns the BLOCKING problems; an empty
+        # list means healthy. This loop never inspects a `.status`, because
+        # the doctor's GPU check reports "info" and never fails — a loop
+        # testing `!= "ok"` itself would quarantine every CPU-only volunteer
+        # on their third unlucky job.
+        self.health_check = health_check
+        self.max_consecutive_failures = max_consecutive_failures
 
     # -- inputs --------------------------------------------------------------
 
@@ -212,6 +238,44 @@ class ExecutorLoop:
     # -- one task ------------------------------------------------------------
 
     def execute_one(self, lease: Lease) -> bool:
+        """Run one lease, and record what its outcome says about this HOST.
+
+        Three different things return False here and only one implicates the
+        machine:
+
+            TaskExecutionError  — could not run it here            -> counts
+            LeaseLost           — someone else has the work        -> does not
+            accepted=False      — coordinator declined the result  -> does not
+
+        That last one is HTTP 200. Counting it would punish a healthy host
+        for losing a commit race — the same trap the contributions ledger hit
+        by crediting on 2xx.
+        """
+        self.current_task = lease.task_id
+        self.current_attempt = lease.attempt_number
+        self.current_task_started = time.monotonic()
+        try:
+            accepted = self._execute_inner(lease)
+        except TaskExecutionError:
+            # _execute_inner already reported fail() and logged the cause.
+            self.tasks_failed += 1
+            self.consecutive_failures += 1
+            return False
+        except LeaseLost:
+            # _execute_inner normally swallows this and returns False; the
+            # belt-and-braces handler makes the contract above true wherever
+            # it is raised from, and costs a healthy host nothing.
+            return False
+        else:
+            if accepted:
+                self.consecutive_failures = 0
+            return accepted
+        finally:
+            self.current_task = None
+            self.current_attempt = None
+            self.current_task_started = None
+
+    def _execute_inner(self, lease: Lease) -> bool:
         """Run a claimed lease end-to-end. Returns True if the commit was
         accepted. Never raises for task-level problems — they are reported."""
         payload = lease.payload
@@ -311,7 +375,10 @@ class ExecutorLoop:
                 self.client.fail(lease.lease_id, str(exc)[:500])
             except Exception:
                 pass  # lease will expire on its own — same outcome, slower
-            return False
+            # Re-raised, not returned: execute_one counts THIS outcome
+            # against the host and the other two against nobody. Caught one
+            # frame up, so callers still see False.
+            raise
         except LeaseLost:
             log.warning(_jlog("lease lost", task=lease.task_id))
             return False
@@ -330,6 +397,37 @@ class ExecutorLoop:
                 self.client.register(self.registration)
             self._last_node_hb = time.monotonic()
 
+    def _should_stop_volunteering(self) -> bool:
+        """After a streak of host-side failures, ask whether it is the HOST.
+
+        A counter alone would guess. This measures: re-run the same checks
+        `flashnode doctor` runs, and let the answer decide.
+
+        - nothing blocking -> this machine is fine and the JOBS are failing.
+          Say so, reset, keep working. A host that stops because of someone
+          else's broken job is a host that stops for no reason.
+        - blocking problems -> stop claiming. Continuing means burning this
+          job's retries on a machine that cannot run anything.
+        """
+        if self.health_check is None or self.max_consecutive_failures <= 0:
+            return False
+        if self.consecutive_failures < self.max_consecutive_failures:
+            return False
+        unhealthy = self.health_check()
+        if not unhealthy:
+            log.info(_jlog(
+                "consecutive task failures, but this host passes its own "
+                "checks — the jobs are failing, not the machine",
+                failures=self.consecutive_failures))
+            self.consecutive_failures = 0
+            return False
+        self.quarantined = True
+        self.health_report = unhealthy
+        log.error(_jlog("stopping: this host can no longer run tasks",
+                        failures=self.consecutive_failures,
+                        failed_checks=[getattr(r, "name", "?") for r in unhealthy]))
+        return True
+
     def run(self, max_tasks: int | None = None, idle_exit: bool = False) -> int:
         """Claim-and-execute until stopped, `max_tasks` accepted, or — with
         `idle_exit` — the queue drains (drain mode for tests/one-shot runs).
@@ -356,6 +454,8 @@ class ExecutorLoop:
                 continue
             log.info(_jlog("claimed", task=lease.task_id, attempt=lease.attempt_number))
             self.execute_one(lease)
+            if self._should_stop_volunteering():
+                break
         return self.tasks_accepted
 
 

{flashnode-0.3.1 → flashnode-0.3.2}/flashnode/inventory/capabilities.py

@@ -14,6 +14,7 @@ import socket
 import psutil
 
 from flashnode.config.local_data import load_local_data
+from flashnode.inventory.gpu import probe_gpus
 from flashruntime.protocol.v1alpha1 import (
     NodeCapabilities,
     NodeEnvironment,
@@ -90,6 +91,17 @@ def discover(node_id: str, kubernetes_node: str,
 
     from flashnode import __version__
 
+    # What `nvidia-smi` reports, or nothing. probe_gpus already promises it
+    # never raises; this is a second lock on the same door, because the cost
+    # of being wrong here is not "no GPU work" but "this agent does not
+    # start" — on a machine that was only ever going to run CPU tasks. Unlike
+    # local_datasets below, an unreadable GPU is not a misconfiguration the
+    # owner must fix; it is the normal state of most hosts.
+    try:
+        gpus = probe_gpus()
+    except Exception:
+        gpus = []
+
     return NodeRegistration(
         node_id=node_id,
         kubernetes_node=kubernetes_node,
@@ -97,7 +109,11 @@ def discover(node_id: str, kubernetes_node: str,
         capabilities=NodeCapabilities(
             cpu_cores=cpu,
             memory_bytes=memory,
-            gpus=[],  # GPU probing is a documented follow-up; never guess.
+            # One entry per device the driver reports; [] on the hosts that
+            # have no driver, which is most of them. Placement reads the
+            # LENGTH of this list, so an entry we are unsure of is worse
+            # than no entry — see inventory/gpu.py.
+            gpus=gpus,
             os=labels.get("kubernetes.io/os", platform.system().lower()),
             architecture=labels.get("kubernetes.io/arch", arch),
         ),