flashnode 0.3.0__tar.gz → 0.3.2__tar.gz

{flashnode-0.3.0 → flashnode-0.3.2}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: flashnode
-Version: 0.3.0
+Version: 0.3.2
 Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: flashruntime<0.5,>=0.4
+Requires-Dist: flashruntime<0.5,>=0.4.1
 Requires-Dist: psutil>=5.9
 Requires-Dist: websockets>=12
 Requires-Dist: cryptography>=42
@@ -49,10 +49,48 @@ verified results. Two profiles:
   inside managed pools (DaemonSet); KubeRay owns workload pods there.
   Implemented.
 
+## Check your machine first
+
+```bash
+flashnode doctor
+```
+
+Six checks: the `docker` CLI, the engine behind it, an anonymous pull of a
+curated image, whether a container can see your work directory, whether your
+Docker accepts the sandbox flags, and whether any directories you lend via
+`FLASHNODE_LOCAL_DATA` are readable. Every failure names the fix.
+
+Run it once before `flashnode work`. `work` repeats all of it **except** the
+image pull — a registry blip should not stop an agent whose images are
+already cached — and refuses to start if anything fails, because a host that
+cannot run tasks should not be claiming them.
+
+## While it runs
+
+On a terminal you get a live status block:
+
+```
+flashnode 0.3.2 · flashml-api.onrender.com · up 2h14m
+  running    fed-2e2d4d6ab57f  ·  attempt 1  ·  38s
+  session    12 accepted   0 failed
+  heartbeat  2s ago
+```
+
+`waiting · no work queued — this is normal` means exactly that: the pool has
+nothing for you right now, and your machine is fine. Pipe the output
+anywhere, or pass `--log-json`, and you get the machine-readable log instead.
+
+If three tasks in a row fail **on your machine**, the agent re-runs its own
+checks. Pass, and the jobs were broken rather than your host, so it carries
+on. Fail, and it stops claiming and tells you what to fix — instead of
+burning a job's retries on a machine that cannot run anything.
+`--max-consecutive-failures 0` turns that off.
+
 ## What it does today
 
 ```bash
 pip install -e .                      # plus: pip install -e ../flashruntime
+flashnode doctor                      # check this machine can run tasks
 flashnode work --coordinator http://<coordinator>:8100
 # optional hardening / pool config:
 #   FLASHNODE_JOIN_CODE=...          join-code-gated pools

{flashnode-0.3.0 → flashnode-0.3.2}/README.md

@@ -32,10 +32,48 @@ verified results. Two profiles:
   inside managed pools (DaemonSet); KubeRay owns workload pods there.
   Implemented.
 
+## Check your machine first
+
+```bash
+flashnode doctor
+```
+
+Six checks: the `docker` CLI, the engine behind it, an anonymous pull of a
+curated image, whether a container can see your work directory, whether your
+Docker accepts the sandbox flags, and whether any directories you lend via
+`FLASHNODE_LOCAL_DATA` are readable. Every failure names the fix.
+
+Run it once before `flashnode work`. `work` repeats all of it **except** the
+image pull — a registry blip should not stop an agent whose images are
+already cached — and refuses to start if anything fails, because a host that
+cannot run tasks should not be claiming them.
+
+## While it runs
+
+On a terminal you get a live status block:
+
+```
+flashnode 0.3.2 · flashml-api.onrender.com · up 2h14m
+  running    fed-2e2d4d6ab57f  ·  attempt 1  ·  38s
+  session    12 accepted   0 failed
+  heartbeat  2s ago
+```
+
+`waiting · no work queued — this is normal` means exactly that: the pool has
+nothing for you right now, and your machine is fine. Pipe the output
+anywhere, or pass `--log-json`, and you get the machine-readable log instead.
+
+If three tasks in a row fail **on your machine**, the agent re-runs its own
+checks. Pass, and the jobs were broken rather than your host, so it carries
+on. Fail, and it stops claiming and tells you what to fix — instead of
+burning a job's retries on a machine that cannot run anything.
+`--max-consecutive-failures 0` turns that off.
+
 ## What it does today
 
 ```bash
 pip install -e .                      # plus: pip install -e ../flashruntime
+flashnode doctor                      # check this machine can run tasks
 flashnode work --coordinator http://<coordinator>:8100
 # optional hardening / pool config:
 #   FLASHNODE_JOIN_CODE=...          join-code-gated pools

{flashnode-0.3.0 → flashnode-0.3.2}/flashnode/agent/cli.py

@@ -26,6 +26,7 @@ commands:
             (--coordinator URL; --token TOKEN to skip the browser step)
   logout    remove the saved bearer token for a FlashRuntime coordinator
             (--coordinator URL)
+  doctor    check this machine can run tasks (docker engine, images, mounts)
   join      connect this machine to a FlashML control plane (not yet implemented)
   status    show node identity, capabilities, and active leases (not yet implemented)
   leave     drain and disconnect (not yet implemented)
@@ -179,6 +180,16 @@ def _work(args: list[str]) -> int:
     )
     parser.add_argument("--max-tasks", type=int, default=None)
     parser.add_argument("--poll-seconds", type=float, default=1.0)
+    parser.add_argument(
+        "--log-json", action="store_true",
+        help="keep the machine-readable JSON log instead of the live status view",
+    )
+    parser.add_argument(
+        "--max-consecutive-failures", type=int,
+        default=int(os.environ.get("FLASHNODE_MAX_CONSECUTIVE_FAILURES", "3")),
+        help="host-side failures in a row before re-checking this machine and "
+             "stopping if it is broken (0 disables)",
+    )
     opts = parser.parse_args(args)
 
     runner = None
@@ -201,10 +212,35 @@ def _work(args: list[str]) -> int:
         # (subprocess.run(["docker", ...])); if it isn't installed that call
         # raises FileNotFoundError deep inside a task attempt. Check for it
         # here, at startup, rather than let the agent die on the first task.
-        if shutil.which("docker") is None:
+        # A `docker` binary on PATH says nothing about the daemon behind it,
+        # the credential helper Docker consults when it pulls, or whether
+        # this machine's work directory is even visible inside the VM. Both
+        # hosts that stopped the 2026-08-02 §10 run-through passed the old
+        # `shutil.which` check and then failed every task they claimed —
+        # docker_runner raises TaskExecutionError, loop.py calls fail() and
+        # claims the next one, forever, silently.
+        #
+        # pull=False deliberately: an agent is a long-running daemon on
+        # someone else's machine, and a transient registry blip must not
+        # stop one whose images are already cached. `flashnode doctor` does
+        # the pull.
+        #
+        # The gate reads doctor's own NON_BLOCKING_STATUSES rather than
+        # testing `!= "ok"` here. Not every check is a gate: the GPU check is
+        # informational, because most volunteers have no GPU and blocking on
+        # it would lock the entire existing fleet out of CPU work the moment
+        # they upgraded. Keeping the predicate in one place is what stops
+        # `flashnode work` and `flashnode doctor` disagreeing about which
+        # verdicts are fatal.
+        from flashnode.doctor import NON_BLOCKING_STATUSES, format_results, run_checks
+
+        results = run_checks(pull=False)
+        if any(r.status not in NON_BLOCKING_STATUSES for r in results):
             print(
-                f"flashnode work: --runner {opts.runner} requires the `docker` CLI "
-                "on PATH — refusing to start without it",
+                f"flashnode work: this machine cannot run tasks with "
+                f"--runner {opts.runner}.\n" + format_results(results)
+                + "\n\nRun `flashnode doctor` for the full check, including "
+                  "the image pull this skipped.",
                 file=sys.stderr,
             )
             return 2
@@ -246,10 +282,29 @@ def _work(args: list[str]) -> int:
         module_capable=(opts.runner != "argv"),
     )
     client.register(registration)
+    from flashnode.doctor import NON_BLOCKING_STATUSES, run_checks
+
+    def _blocking_problems():
+        """What the loop calls after a streak of host-side failures.
+
+        Filtered HERE, with the same set the startup gate reads, so the two
+        cannot drift — and so `loop.py` never has to import the doctor
+        (which would close a loop -> doctor -> executor -> loop cycle).
+        The GPU check reports "info" and never fails; a loop testing
+        `!= "ok"` itself would quarantine every CPU-only volunteer.
+
+        pull=False for the same reason the startup gate uses it: a registry
+        blip must not stop an agent whose images are already cached.
+        """
+        return [r for r in run_checks(pull=False)
+                if r.status not in NON_BLOCKING_STATUSES]
+
     loop = ExecutorLoop(
         client, node_id, runner=runner,
         poll_seconds=opts.poll_seconds, workdir_base=workdir_base,
         registration=registration,  # survives coordinator restarts
+        health_check=_blocking_problems,
+        max_consecutive_failures=opts.max_consecutive_failures,
     )
 
     def _stop(signum, frame):  # noqa: ARG001
@@ -257,7 +312,35 @@ def _work(args: list[str]) -> int:
 
     signal.signal(signal.SIGTERM, _stop)
     signal.signal(signal.SIGINT, _stop)
-    accepted = loop.run(max_tasks=opts.max_tasks)
+    view = None
+    if sys.stdout.isatty() and not opts.log_json:
+        # Two writers redrawing one terminal is unreadable, so the JSON
+        # handler goes when the view arrives. --log-json keeps it, and a
+        # non-TTY never gets here — ANSI cursor movement into a pipe or a
+        # systemd journal is corruption, not output.
+        from flashnode.status import StatusView
+
+        logging.getLogger().handlers.clear()
+        view = StatusView(loop, coordinator=opts.coordinator,
+                          version=__version__, stream=sys.stdout)
+        view.start()
+    try:
+        accepted = loop.run(max_tasks=opts.max_tasks)
+    finally:
+        if view is not None:
+            view.stop()
+
+    if getattr(loop, "quarantined", False):
+        from flashnode.doctor import format_results
+
+        print(
+            "\nflashnode work: stopping — this machine can no longer run "
+            "tasks.\n" + format_results(loop.health_report or [])
+            + "\n\nFix the above, then `flashnode doctor` to confirm before "
+              "restarting.",
+            file=sys.stderr,
+        )
+        return 2
     print(f"flashnode work: {accepted} task(s) accepted", file=sys.stderr)
     return 0
 
@@ -270,6 +353,10 @@ def main(argv: list[str] | None = None) -> int:
         return agent_main()
     if args and args[0] == "work":
         return _work(args[1:])
+    if args and args[0] == "doctor":
+        from flashnode.doctor import doctor_main
+
+        return doctor_main(args[1:])
     if args and args[0] == "login":
         return _login(args[1:])
     if args and args[0] == "logout":