flashnode 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flashnode-0.3.0 → flashnode-0.3.1}/PKG-INFO +18 -1
- {flashnode-0.3.0 → flashnode-0.3.1}/README.md +17 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/agent/cli.py +25 -3
- flashnode-0.3.1/flashnode/doctor.py +446 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode.egg-info/PKG-INFO +18 -1
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode.egg-info/SOURCES.txt +4 -1
- {flashnode-0.3.0 → flashnode-0.3.1}/pyproject.toml +1 -1
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_agent.py +14 -15
- flashnode-0.3.1/tests/test_doctor.py +410 -0
- flashnode-0.3.1/tests/test_work_gate.py +71 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/LICENSE +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/agent/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/agent/daemon.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/agent/kube.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/artifacts/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/benchmark/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/config/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/config/local_data.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/archives.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/argv_runner.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/client.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/docker_runner.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/hardening.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/images.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/loop.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/executor/runner.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/identity/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/identity/credentials.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/identity/enrol.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/identity/store.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/inventory/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/inventory/capabilities.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode/telemetry/__init__.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode.egg-info/dependency_links.txt +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode.egg-info/entry_points.txt +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode.egg-info/requires.txt +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/flashnode.egg-info/top_level.txt +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/setup.cfg +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_allowlist_drift.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_archives.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_argv_runner.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_capabilities.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_checkpoint_relay.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_credentials.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_docker_runner.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_enrol.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_executor.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_hardening.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_hardening_platform.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_identity.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_images_allowlist.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_interfaces.py +0 -0
- {flashnode-0.3.0 → flashnode-0.3.1}/tests/test_local_data.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flashnode
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
|
|
@@ -49,10 +49,27 @@ verified results. Two profiles:
|
|
|
49
49
|
inside managed pools (DaemonSet); KubeRay owns workload pods there.
|
|
50
50
|
Implemented.
|
|
51
51
|
|
|
52
|
+
## Check your machine first
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
flashnode doctor
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Six checks: the `docker` CLI, the engine behind it, an anonymous pull of a
|
|
59
|
+
curated image, whether a container can see your work directory, whether your
|
|
60
|
+
Docker accepts the sandbox flags, and whether any directories you lend via
|
|
61
|
+
`FLASHNODE_LOCAL_DATA` are readable. Every failure names the fix.
|
|
62
|
+
|
|
63
|
+
Run it once before `flashnode work`. `work` repeats all of it **except** the
|
|
64
|
+
image pull — a registry blip should not stop an agent whose images are
|
|
65
|
+
already cached — and refuses to start if anything fails, because a host that
|
|
66
|
+
cannot run tasks should not be claiming them.
|
|
67
|
+
|
|
52
68
|
## What it does today
|
|
53
69
|
|
|
54
70
|
```bash
|
|
55
71
|
pip install -e . # plus: pip install -e ../flashruntime
|
|
72
|
+
flashnode doctor # check this machine can run tasks
|
|
56
73
|
flashnode work --coordinator http://<coordinator>:8100
|
|
57
74
|
# optional hardening / pool config:
|
|
58
75
|
# FLASHNODE_JOIN_CODE=... join-code-gated pools
|
|
@@ -32,10 +32,27 @@ verified results. Two profiles:
|
|
|
32
32
|
inside managed pools (DaemonSet); KubeRay owns workload pods there.
|
|
33
33
|
Implemented.
|
|
34
34
|
|
|
35
|
+
## Check your machine first
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
flashnode doctor
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Six checks: the `docker` CLI, the engine behind it, an anonymous pull of a
|
|
42
|
+
curated image, whether a container can see your work directory, whether your
|
|
43
|
+
Docker accepts the sandbox flags, and whether any directories you lend via
|
|
44
|
+
`FLASHNODE_LOCAL_DATA` are readable. Every failure names the fix.
|
|
45
|
+
|
|
46
|
+
Run it once before `flashnode work`. `work` repeats all of it **except** the
|
|
47
|
+
image pull — a registry blip should not stop an agent whose images are
|
|
48
|
+
already cached — and refuses to start if anything fails, because a host that
|
|
49
|
+
cannot run tasks should not be claiming them.
|
|
50
|
+
|
|
35
51
|
## What it does today
|
|
36
52
|
|
|
37
53
|
```bash
|
|
38
54
|
pip install -e . # plus: pip install -e ../flashruntime
|
|
55
|
+
flashnode doctor # check this machine can run tasks
|
|
39
56
|
flashnode work --coordinator http://<coordinator>:8100
|
|
40
57
|
# optional hardening / pool config:
|
|
41
58
|
# FLASHNODE_JOIN_CODE=... join-code-gated pools
|
|
@@ -26,6 +26,7 @@ commands:
|
|
|
26
26
|
(--coordinator URL; --token TOKEN to skip the browser step)
|
|
27
27
|
logout remove the saved bearer token for a FlashRuntime coordinator
|
|
28
28
|
(--coordinator URL)
|
|
29
|
+
doctor check this machine can run tasks (docker engine, images, mounts)
|
|
29
30
|
join connect this machine to a FlashML control plane (not yet implemented)
|
|
30
31
|
status show node identity, capabilities, and active leases (not yet implemented)
|
|
31
32
|
leave drain and disconnect (not yet implemented)
|
|
@@ -201,10 +202,27 @@ def _work(args: list[str]) -> int:
|
|
|
201
202
|
# (subprocess.run(["docker", ...])); if it isn't installed that call
|
|
202
203
|
# raises FileNotFoundError deep inside a task attempt. Check for it
|
|
203
204
|
# here, at startup, rather than let the agent die on the first task.
|
|
204
|
-
|
|
205
|
+
# A `docker` binary on PATH says nothing about the daemon behind it,
|
|
206
|
+
# the credential helper Docker consults when it pulls, or whether
|
|
207
|
+
# this machine's work directory is even visible inside the VM. Both
|
|
208
|
+
# hosts that stopped the 2026-08-02 §10 run-through passed the old
|
|
209
|
+
# `shutil.which` check and then failed every task they claimed —
|
|
210
|
+
# docker_runner raises TaskExecutionError, loop.py calls fail() and
|
|
211
|
+
# claims the next one, forever, silently.
|
|
212
|
+
#
|
|
213
|
+
# pull=False deliberately: an agent is a long-running daemon on
|
|
214
|
+
# someone else's machine, and a transient registry blip must not
|
|
215
|
+
# stop one whose images are already cached. `flashnode doctor` does
|
|
216
|
+
# the pull.
|
|
217
|
+
from flashnode.doctor import format_results, run_checks
|
|
218
|
+
|
|
219
|
+
results = run_checks(pull=False)
|
|
220
|
+
if any(r.status != "ok" for r in results):
|
|
205
221
|
print(
|
|
206
|
-
f"flashnode work:
|
|
207
|
-
"
|
|
222
|
+
f"flashnode work: this machine cannot run tasks with "
|
|
223
|
+
f"--runner {opts.runner}.\n" + format_results(results)
|
|
224
|
+
+ "\n\nRun `flashnode doctor` for the full check, including "
|
|
225
|
+
"the image pull this skipped.",
|
|
208
226
|
file=sys.stderr,
|
|
209
227
|
)
|
|
210
228
|
return 2
|
|
@@ -270,6 +288,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
270
288
|
return agent_main()
|
|
271
289
|
if args and args[0] == "work":
|
|
272
290
|
return _work(args[1:])
|
|
291
|
+
if args and args[0] == "doctor":
|
|
292
|
+
from flashnode.doctor import doctor_main
|
|
293
|
+
|
|
294
|
+
return doctor_main(args[1:])
|
|
273
295
|
if args and args[0] == "login":
|
|
274
296
|
return _login(args[1:])
|
|
275
297
|
if args and args[0] == "logout":
|
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
"""Host health checks for the sandboxed execution tiers.
|
|
2
|
+
|
|
3
|
+
WHY THIS EXISTS. Two machines stopped the 2026-08-02 §10 run-through, and
|
|
4
|
+
neither was a distributed-systems problem: `docker-credential-desktop`
|
|
5
|
+
missing on macOS, and a Docker engine answering `_ping` with 500 on Windows.
|
|
6
|
+
The startup gate at the time was `shutil.which("docker")`, which BOTH
|
|
7
|
+
machines pass — the binary was on PATH in both cases.
|
|
8
|
+
|
|
9
|
+
What happened instead is worse than a crash. `docker_runner` turns a
|
|
10
|
+
non-zero `docker run` into TaskExecutionError; `loop.py` catches it, calls
|
|
11
|
+
fail() on the lease, and keeps claiming. A host with broken Docker therefore
|
|
12
|
+
claims a task, fails it, claims the next one, and never tells its owner. The
|
|
13
|
+
volunteer sees a healthy-looking agent; the submitter sees their job failing
|
|
14
|
+
with a Docker error tail from a stranger's laptop.
|
|
15
|
+
|
|
16
|
+
Every check takes its side effects as a parameter. That is not test
|
|
17
|
+
decoration: a diagnostic you can only exercise on a broken machine is one
|
|
18
|
+
nobody can keep correct.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import os
|
|
24
|
+
import shutil
|
|
25
|
+
import subprocess
|
|
26
|
+
import tempfile
|
|
27
|
+
from collections.abc import Sequence
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Callable
|
|
31
|
+
|
|
32
|
+
from flashnode.executor.hardening import (
|
|
33
|
+
CONTAINER_WORKDIR,
|
|
34
|
+
_bind_mount_source,
|
|
35
|
+
harden_args,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"PROBE_IMAGE",
|
|
40
|
+
"CheckResult",
|
|
41
|
+
"check_cli_on_path",
|
|
42
|
+
"check_engine",
|
|
43
|
+
"check_hardened_run",
|
|
44
|
+
"check_local_datasets",
|
|
45
|
+
"check_pull",
|
|
46
|
+
"check_workdir_mount",
|
|
47
|
+
"default_workdir",
|
|
48
|
+
"doctor_main",
|
|
49
|
+
"exit_code",
|
|
50
|
+
"format_results",
|
|
51
|
+
"run_checks",
|
|
52
|
+
"run_command",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
#: The image every container-level check runs. python-slim, never
|
|
56
|
+
#: pytorch-cpu: registry auth, TLS and the credential helper are properties
|
|
57
|
+
#: of the REGISTRY, so the smallest curated image proves the same thing, and
|
|
58
|
+
#: making a volunteer download gigabytes to learn their helper is missing is
|
|
59
|
+
#: a hostile diagnostic. Kept in step with flashml-cloud's published tags.
|
|
60
|
+
PROBE_IMAGE = "ghcr.io/zolli-labs/flashml-python-slim:2026.08.1"
|
|
61
|
+
|
|
62
|
+
CommandRunner = Callable[..., subprocess.CompletedProcess]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class CheckResult:
|
|
67
|
+
"""One check's verdict.
|
|
68
|
+
|
|
69
|
+
`fix` is the point of the whole module: a volunteer will not derive
|
|
70
|
+
"your credential helper is missing" from `task exited 1`.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
name: str
|
|
74
|
+
status: str # "ok" | "fail" | "skip"
|
|
75
|
+
detail: str = ""
|
|
76
|
+
fix: str = ""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def run_command(argv: Sequence[str], *, timeout: float = 300.0) -> subprocess.CompletedProcess:
|
|
80
|
+
"""The real side effect. Generous timeouts because a cold image pull is
|
|
81
|
+
slow on a home connection, and a doctor that times out on a healthy host
|
|
82
|
+
is worse than no doctor."""
|
|
83
|
+
return subprocess.run(list(argv), capture_output=True, timeout=timeout, check=False)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _text(raw: bytes | str | None) -> str:
|
|
87
|
+
if raw is None:
|
|
88
|
+
return ""
|
|
89
|
+
if isinstance(raw, str):
|
|
90
|
+
return raw.strip()
|
|
91
|
+
return raw.decode(errors="replace").strip()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def check_cli_on_path(which: Callable[[str], str | None] | None = None) -> CheckResult:
|
|
95
|
+
# Resolved at CALL time, never as a default argument. `def f(which=
|
|
96
|
+
# shutil.which)` captures the original function object at import, so
|
|
97
|
+
# `monkeypatch.setattr("shutil.which", ...)` — the idiom the rest of this
|
|
98
|
+
# suite uses, and documents in test_agent.py — silently fails to reach
|
|
99
|
+
# it. That is not merely a testing inconvenience: it let the `work` gate
|
|
100
|
+
# ignore a patched `which`, run a REAL `docker run` inside unit tests,
|
|
101
|
+
# and pass or fail with the machine's Docker state.
|
|
102
|
+
which = which or shutil.which
|
|
103
|
+
name = "docker CLI on PATH"
|
|
104
|
+
found = which("docker")
|
|
105
|
+
if found:
|
|
106
|
+
return CheckResult(name, "ok", detail=found)
|
|
107
|
+
return CheckResult(
|
|
108
|
+
name, "fail",
|
|
109
|
+
detail="`docker` was not found on PATH",
|
|
110
|
+
fix="Install Docker Desktop (macOS/Windows) or your distribution's "
|
|
111
|
+
"docker package, then re-run `flashnode doctor`.",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def check_engine(run: CommandRunner) -> CheckResult:
|
|
116
|
+
"""The binary existing says nothing about the daemon behind it. This is
|
|
117
|
+
the check the Windows machine needed and did not have."""
|
|
118
|
+
name = "docker engine reachable"
|
|
119
|
+
argv = ["docker", "version", "--format", "{{.Server.Version}}"]
|
|
120
|
+
try:
|
|
121
|
+
proc = run(argv, timeout=30.0)
|
|
122
|
+
except FileNotFoundError as exc:
|
|
123
|
+
# `docker` vanished between check 1 and here. Report, never raise:
|
|
124
|
+
# a diagnostic that crashes has diagnosed nothing.
|
|
125
|
+
return CheckResult(name, "fail", detail=str(exc),
|
|
126
|
+
fix="Install Docker and re-run `flashnode doctor`.")
|
|
127
|
+
except subprocess.TimeoutExpired:
|
|
128
|
+
return CheckResult(
|
|
129
|
+
name, "fail", detail="`docker version` did not answer within 30s",
|
|
130
|
+
fix="The Docker daemon is hung. Restart Docker Desktop (or "
|
|
131
|
+
"`systemctl restart docker`) and re-run `flashnode doctor`.",
|
|
132
|
+
)
|
|
133
|
+
if proc.returncode == 0 and _text(proc.stdout):
|
|
134
|
+
return CheckResult(name, "ok", detail=f"server {_text(proc.stdout)}")
|
|
135
|
+
return CheckResult(
|
|
136
|
+
name, "fail", detail=_text(proc.stderr) or _text(proc.stdout),
|
|
137
|
+
fix="The docker CLI is installed but no daemon answered. Start "
|
|
138
|
+
"Docker Desktop and wait for it to report Running, or start the "
|
|
139
|
+
"docker service, then re-run `flashnode doctor`.",
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def check_pull(run: CommandRunner, image: str = PROBE_IMAGE) -> CheckResult:
|
|
144
|
+
"""Pull the smallest curated image.
|
|
145
|
+
|
|
146
|
+
This is the check the Mac needed. The pull is the only step that touches
|
|
147
|
+
a registry and therefore the only one that consults a credential helper
|
|
148
|
+
— tasks themselves run `--network none`. It also re-catches the GHCR
|
|
149
|
+
visibility regression, where the images went private and every job died
|
|
150
|
+
at execution while signup, install and enrolment all looked fine.
|
|
151
|
+
"""
|
|
152
|
+
name = "pull a curated image"
|
|
153
|
+
try:
|
|
154
|
+
proc = run(["docker", "pull", image], timeout=600.0)
|
|
155
|
+
except subprocess.TimeoutExpired:
|
|
156
|
+
return CheckResult(
|
|
157
|
+
name, "fail", detail=f"`docker pull {image}` did not finish in 10 minutes",
|
|
158
|
+
fix="Check this machine's internet connection, then re-run "
|
|
159
|
+
"`flashnode doctor`.",
|
|
160
|
+
)
|
|
161
|
+
except OSError as exc:
|
|
162
|
+
return CheckResult(name, "fail", detail=str(exc),
|
|
163
|
+
fix="Install Docker and re-run `flashnode doctor`.")
|
|
164
|
+
if proc.returncode == 0:
|
|
165
|
+
return CheckResult(name, "ok", detail=image)
|
|
166
|
+
err = _text(proc.stderr) or _text(proc.stdout)
|
|
167
|
+
if "credential" in err or "docker-credential" in err:
|
|
168
|
+
fix = ("Your ~/.docker/config.json names a credential helper that is "
|
|
169
|
+
"not installed. Either start Docker Desktop, or remove the "
|
|
170
|
+
'"credsStore" line — these images are public and need no login.')
|
|
171
|
+
elif "denied" in err or "unauthorized" in err:
|
|
172
|
+
fix = ("The registry refused an anonymous pull. These images are "
|
|
173
|
+
"meant to be public; report this — it is our bug, not yours.")
|
|
174
|
+
else:
|
|
175
|
+
fix = ("Check this machine's internet connection and that "
|
|
176
|
+
"ghcr.io is reachable, then re-run `flashnode doctor`.")
|
|
177
|
+
return CheckResult(name, "fail", detail=f"{image}\n{err}", fix=fix)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
PROBE_FILENAME = "flashnode-doctor-probe.txt"
|
|
181
|
+
PROBE_CONTENT = "flashnode-doctor"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def default_workdir() -> Path:
|
|
185
|
+
"""Where the agent will actually stage task inputs.
|
|
186
|
+
|
|
187
|
+
Mirrors ExecutorLoop's own default (`workdir_base=None` → the system
|
|
188
|
+
temp dir) deliberately. On macOS that is /var/folders/…, which colima
|
|
189
|
+
cannot see — so the doctor fails on precisely the machines where the
|
|
190
|
+
agent would, which is the entire point.
|
|
191
|
+
"""
|
|
192
|
+
return Path(os.environ.get("FLASHNODE_WORKDIR") or tempfile.gettempdir())
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _mount_failure_fix(err: str) -> str:
|
|
196
|
+
if "No such image" in err or "not found" in err.lower():
|
|
197
|
+
return ("The probe image is not cached on this machine. Run "
|
|
198
|
+
"`flashnode doctor` once — it pulls it.")
|
|
199
|
+
return ("The container could not see this directory. On macOS, "
|
|
200
|
+
"colima and Docker Desktop share only $HOME: set "
|
|
201
|
+
"FLASHNODE_WORKDIR to a path under your home directory "
|
|
202
|
+
"(e.g. export FLASHNODE_WORKDIR=$HOME/.flashnode/work) and "
|
|
203
|
+
"re-run `flashnode doctor`.")
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def check_workdir_mount(
|
|
207
|
+
run: CommandRunner, workdir: Path, image: str = PROBE_IMAGE
|
|
208
|
+
) -> CheckResult:
|
|
209
|
+
"""Can a container see the directory the agent stages inputs in?
|
|
210
|
+
|
|
211
|
+
Minimum viable flags, on purpose. Check 5 runs the same probe with the
|
|
212
|
+
full hardening set, so a failure HERE is a mount problem and a failure
|
|
213
|
+
THERE is a flag problem — localised without pattern-matching stderr.
|
|
214
|
+
|
|
215
|
+
The container READS a file the host wrote rather than writing one: the
|
|
216
|
+
curated images end in a non-root USER and this flag set has no --user,
|
|
217
|
+
so a write would hit permission denied on a healthy Linux host.
|
|
218
|
+
"""
|
|
219
|
+
name = "workdir bind-mounts"
|
|
220
|
+
probe = Path(workdir) / PROBE_FILENAME
|
|
221
|
+
try:
|
|
222
|
+
probe.parent.mkdir(parents=True, exist_ok=True)
|
|
223
|
+
probe.write_text(PROBE_CONTENT)
|
|
224
|
+
except OSError as exc:
|
|
225
|
+
return CheckResult(
|
|
226
|
+
name, "fail", detail=f"{workdir}: {exc}",
|
|
227
|
+
fix="The agent cannot write to its own work directory. Set "
|
|
228
|
+
"FLASHNODE_WORKDIR to a writable path and re-run "
|
|
229
|
+
"`flashnode doctor`.",
|
|
230
|
+
)
|
|
231
|
+
argv = [
|
|
232
|
+
"docker", "run", "--rm", "--pull=never",
|
|
233
|
+
"-v", f"{_bind_mount_source(Path(workdir))}:{CONTAINER_WORKDIR}",
|
|
234
|
+
"-w", CONTAINER_WORKDIR,
|
|
235
|
+
image,
|
|
236
|
+
"python", "-c",
|
|
237
|
+
f"print(open('{CONTAINER_WORKDIR}/{PROBE_FILENAME}').read(), end='')",
|
|
238
|
+
]
|
|
239
|
+
try:
|
|
240
|
+
proc = run(argv, timeout=120.0)
|
|
241
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
242
|
+
return CheckResult(name, "fail", detail=str(exc),
|
|
243
|
+
fix=_mount_failure_fix(str(exc)))
|
|
244
|
+
finally:
|
|
245
|
+
probe.unlink(missing_ok=True)
|
|
246
|
+
if proc.returncode == 0 and _text(proc.stdout) == PROBE_CONTENT:
|
|
247
|
+
return CheckResult(name, "ok", detail=str(workdir))
|
|
248
|
+
err = _text(proc.stderr) or _text(proc.stdout) or "the container read nothing back"
|
|
249
|
+
return CheckResult(name, "fail", detail=f"{workdir}\n{err}",
|
|
250
|
+
fix=_mount_failure_fix(err))
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def check_hardened_run(
|
|
254
|
+
run: CommandRunner, workdir: Path, image: str = PROBE_IMAGE
|
|
255
|
+
) -> CheckResult:
|
|
256
|
+
"""The same probe as check 4, with the REAL sandbox flags.
|
|
257
|
+
|
|
258
|
+
Check 4 uses the minimum that can work; this uses everything a task
|
|
259
|
+
gets. So 4 passing and 5 failing localises the fault to a hardening
|
|
260
|
+
flag, with no stderr pattern-matching — and on Windows this is the first
|
|
261
|
+
thing in the system that has ever EXECUTED the platform-conditional
|
|
262
|
+
--user path from Plan 6, which until now was only argv-verified.
|
|
263
|
+
"""
|
|
264
|
+
name = "a hardened container runs"
|
|
265
|
+
probe = Path(workdir) / PROBE_FILENAME
|
|
266
|
+
try:
|
|
267
|
+
probe.parent.mkdir(parents=True, exist_ok=True)
|
|
268
|
+
probe.write_text(PROBE_CONTENT)
|
|
269
|
+
except OSError as exc:
|
|
270
|
+
return CheckResult(name, "fail", detail=f"{workdir}: {exc}",
|
|
271
|
+
fix="Set FLASHNODE_WORKDIR to a writable path.")
|
|
272
|
+
try:
|
|
273
|
+
flags = harden_args(Path(workdir), cpus=1.0, memory_gb=1.0)
|
|
274
|
+
except (RuntimeError, ValueError) as exc:
|
|
275
|
+
probe.unlink(missing_ok=True)
|
|
276
|
+
return CheckResult(
|
|
277
|
+
name, "fail", detail=str(exc),
|
|
278
|
+
fix="This platform is not one the sandbox can secure. Report "
|
|
279
|
+
"it — running your machine unprivileged-in-name-only is not "
|
|
280
|
+
"something we will do.",
|
|
281
|
+
)
|
|
282
|
+
argv = [
|
|
283
|
+
"docker", "run", "--rm", "--pull=never", *flags, image,
|
|
284
|
+
"python", "-c",
|
|
285
|
+
f"print(open('{CONTAINER_WORKDIR}/{PROBE_FILENAME}').read(), end='')",
|
|
286
|
+
]
|
|
287
|
+
try:
|
|
288
|
+
proc = run(argv, timeout=120.0)
|
|
289
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
290
|
+
return CheckResult(name, "fail", detail=str(exc),
|
|
291
|
+
fix=_mount_failure_fix(str(exc)))
|
|
292
|
+
finally:
|
|
293
|
+
probe.unlink(missing_ok=True)
|
|
294
|
+
if proc.returncode == 0 and _text(proc.stdout) == PROBE_CONTENT:
|
|
295
|
+
return CheckResult(name, "ok", detail="sandbox flags accepted")
|
|
296
|
+
err = _text(proc.stderr) or _text(proc.stdout) or "the container read nothing back"
|
|
297
|
+
if "No such image" in err:
|
|
298
|
+
return CheckResult(name, "fail", detail=err, fix=_mount_failure_fix(err))
|
|
299
|
+
return CheckResult(
|
|
300
|
+
name, "fail", detail=err,
|
|
301
|
+
fix="The workdir mounts (check above passed) but your Docker "
|
|
302
|
+
"rejected one of the sandbox flags. Please report this output — "
|
|
303
|
+
"we will not loosen the sandbox to work around it.",
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def check_local_datasets(raw: str | None = None) -> CheckResult:
|
|
308
|
+
"""Every label this host advertises must resolve to a readable directory.
|
|
309
|
+
|
|
310
|
+
parse_local_data checks the label charset, that the path is absolute,
|
|
311
|
+
that it carries no ':' that would re-read as a second mount, and that no
|
|
312
|
+
label is mapped twice. It never stats the path. So a typo advertises a
|
|
313
|
+
dataset this host cannot serve — and because the coordinator's
|
|
314
|
+
fail-closed placement gate trusts that advertisement, this host becomes
|
|
315
|
+
the ONLY one eligible for the job, and every retry comes back here.
|
|
316
|
+
"""
|
|
317
|
+
from flashnode.config.local_data import (
|
|
318
|
+
LOCAL_DATA_ENV,
|
|
319
|
+
LocalDataError,
|
|
320
|
+
parse_local_data,
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
name = "local datasets readable"
|
|
324
|
+
value = os.environ.get(LOCAL_DATA_ENV) if raw is None else raw
|
|
325
|
+
try:
|
|
326
|
+
mapping = parse_local_data(value)
|
|
327
|
+
except LocalDataError as exc:
|
|
328
|
+
return CheckResult(
|
|
329
|
+
name, "fail", detail=str(exc),
|
|
330
|
+
fix=f"{LOCAL_DATA_ENV} must look like "
|
|
331
|
+
"label=/absolute/path,other=/absolute/path2. Fix it and "
|
|
332
|
+
"re-run `flashnode doctor`.",
|
|
333
|
+
)
|
|
334
|
+
if not mapping:
|
|
335
|
+
return CheckResult(name, "ok", detail="none configured")
|
|
336
|
+
problems = []
|
|
337
|
+
for label, path in sorted(mapping.items()):
|
|
338
|
+
p = Path(path)
|
|
339
|
+
if not p.exists():
|
|
340
|
+
problems.append(f"{label}: {path} does not exist")
|
|
341
|
+
elif not p.is_dir():
|
|
342
|
+
problems.append(f"{label}: {path} is not a directory")
|
|
343
|
+
elif not os.access(p, os.R_OK | os.X_OK):
|
|
344
|
+
problems.append(f"{label}: {path} is not readable")
|
|
345
|
+
if problems:
|
|
346
|
+
return CheckResult(
|
|
347
|
+
name, "fail", detail="\n".join(problems),
|
|
348
|
+
fix=f"Point {LOCAL_DATA_ENV} at directories that exist and are "
|
|
349
|
+
"readable, or remove the labels you cannot serve — the "
|
|
350
|
+
"coordinator sends local-data jobs ONLY to hosts advertising "
|
|
351
|
+
"them, so a bad label strands the job here.",
|
|
352
|
+
)
|
|
353
|
+
return CheckResult(name, "ok", detail=", ".join(sorted(mapping)))
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def run_checks(
|
|
357
|
+
*,
|
|
358
|
+
pull: bool,
|
|
359
|
+
run: CommandRunner | None = None,
|
|
360
|
+
which: Callable[[str], str | None] | None = None,
|
|
361
|
+
workdir: Path | None = None,
|
|
362
|
+
raw_local_data: str | None = None,
|
|
363
|
+
) -> list[CheckResult]:
|
|
364
|
+
"""Run every check, in order, skipping what a prior failure makes
|
|
365
|
+
meaningless.
|
|
366
|
+
|
|
367
|
+
`pull=False` is the `flashnode work` path: an agent is a long-running
|
|
368
|
+
daemon on someone else's machine, and a transient registry blip must not
|
|
369
|
+
stop one whose images are already cached (spec §4.1).
|
|
370
|
+
"""
|
|
371
|
+
# Same call-time resolution as check_cli_on_path, for the same reason.
|
|
372
|
+
run = run or run_command
|
|
373
|
+
which = which or shutil.which
|
|
374
|
+
base = workdir if workdir is not None else default_workdir()
|
|
375
|
+
# Every container-level check, in order, each paired with the callable
|
|
376
|
+
# that runs it. A failure skips the rest of THIS list; the local-dataset
|
|
377
|
+
# check is independent of Docker and always runs, after the loop.
|
|
378
|
+
staged: list[tuple[str, Callable[[], CheckResult]]] = [
|
|
379
|
+
("docker CLI on PATH", lambda: check_cli_on_path(which=which)),
|
|
380
|
+
("docker engine reachable", lambda: check_engine(run)),
|
|
381
|
+
]
|
|
382
|
+
if pull:
|
|
383
|
+
staged.append(("pull a curated image", lambda: check_pull(run)))
|
|
384
|
+
staged += [
|
|
385
|
+
("workdir bind-mounts", lambda: check_workdir_mount(run, base)),
|
|
386
|
+
("a hardened container runs", lambda: check_hardened_run(run, base)),
|
|
387
|
+
]
|
|
388
|
+
|
|
389
|
+
results: list[CheckResult] = []
|
|
390
|
+
stopped = False
|
|
391
|
+
for label, check in staged:
|
|
392
|
+
if stopped:
|
|
393
|
+
results.append(CheckResult(label, "skip", detail="needs the check above"))
|
|
394
|
+
continue
|
|
395
|
+
results.append(check())
|
|
396
|
+
if results[-1].status != "ok":
|
|
397
|
+
stopped = True
|
|
398
|
+
results.append(check_local_datasets(raw=raw_local_data))
|
|
399
|
+
return results
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def format_results(results: Sequence[CheckResult]) -> str:
|
|
403
|
+
lines = []
|
|
404
|
+
for r in results:
|
|
405
|
+
tag = {"ok": "[ok] ", "fail": "[FAIL]", "skip": "[skip]"}[r.status]
|
|
406
|
+
head = r.detail.splitlines()[0] if r.detail else ""
|
|
407
|
+
lines.append(f" {tag} {r.name:<30} {head}".rstrip())
|
|
408
|
+
for extra in r.detail.splitlines()[1:]:
|
|
409
|
+
lines.append(f" {extra}")
|
|
410
|
+
if r.fix:
|
|
411
|
+
lines.append(f" fix: {r.fix}")
|
|
412
|
+
failed = sum(1 for r in results if r.status == "fail")
|
|
413
|
+
skipped = sum(1 for r in results if r.status == "skip")
|
|
414
|
+
if failed or skipped:
|
|
415
|
+
parts = []
|
|
416
|
+
if failed:
|
|
417
|
+
parts.append(f"{failed} check{'s' if failed != 1 else ''} failed")
|
|
418
|
+
if skipped:
|
|
419
|
+
parts.append(f"{skipped} skipped")
|
|
420
|
+
lines.append(
|
|
421
|
+
", ".join(parts) + ". Fix the above, then re-run `flashnode doctor`."
|
|
422
|
+
)
|
|
423
|
+
else:
|
|
424
|
+
lines.append("All checks passed. Start contributing with "
|
|
425
|
+
"`flashnode work --runner docker`.")
|
|
426
|
+
return "\n".join(lines)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def exit_code(results: Sequence[CheckResult]) -> int:
|
|
430
|
+
"""Skipped counts as not-passed. A host whose checks did not run has not
|
|
431
|
+
been certified, and calling it healthy is the exact failure this command
|
|
432
|
+
removes."""
|
|
433
|
+
return 0 if all(r.status == "ok" for r in results) else 1
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def doctor_main(argv: list[str]) -> int:
|
|
437
|
+
import argparse
|
|
438
|
+
|
|
439
|
+
parser = argparse.ArgumentParser(
|
|
440
|
+
prog="flashnode doctor",
|
|
441
|
+
description="Check this machine can actually run FlashML tasks.",
|
|
442
|
+
)
|
|
443
|
+
parser.parse_args(argv)
|
|
444
|
+
results = run_checks(pull=True)
|
|
445
|
+
print(format_results(results))
|
|
446
|
+
return exit_code(results)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flashnode
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
|
|
@@ -49,10 +49,27 @@ verified results. Two profiles:
|
|
|
49
49
|
inside managed pools (DaemonSet); KubeRay owns workload pods there.
|
|
50
50
|
Implemented.
|
|
51
51
|
|
|
52
|
+
## Check your machine first
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
flashnode doctor
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Six checks: the `docker` CLI, the engine behind it, an anonymous pull of a
|
|
59
|
+
curated image, whether a container can see your work directory, whether your
|
|
60
|
+
Docker accepts the sandbox flags, and whether any directories you lend via
|
|
61
|
+
`FLASHNODE_LOCAL_DATA` are readable. Every failure names the fix.
|
|
62
|
+
|
|
63
|
+
Run it once before `flashnode work`. `work` repeats all of it **except** the
|
|
64
|
+
image pull — a registry blip should not stop an agent whose images are
|
|
65
|
+
already cached — and refuses to start if anything fails, because a host that
|
|
66
|
+
cannot run tasks should not be claiming them.
|
|
67
|
+
|
|
52
68
|
## What it does today
|
|
53
69
|
|
|
54
70
|
```bash
|
|
55
71
|
pip install -e . # plus: pip install -e ../flashruntime
|
|
72
|
+
flashnode doctor # check this machine can run tasks
|
|
56
73
|
flashnode work --coordinator http://<coordinator>:8100
|
|
57
74
|
# optional hardening / pool config:
|
|
58
75
|
# FLASHNODE_JOIN_CODE=... join-code-gated pools
|
|
@@ -2,6 +2,7 @@ LICENSE
|
|
|
2
2
|
README.md
|
|
3
3
|
pyproject.toml
|
|
4
4
|
flashnode/__init__.py
|
|
5
|
+
flashnode/doctor.py
|
|
5
6
|
flashnode.egg-info/PKG-INFO
|
|
6
7
|
flashnode.egg-info/SOURCES.txt
|
|
7
8
|
flashnode.egg-info/dependency_links.txt
|
|
@@ -40,6 +41,7 @@ tests/test_capabilities.py
|
|
|
40
41
|
tests/test_checkpoint_relay.py
|
|
41
42
|
tests/test_credentials.py
|
|
42
43
|
tests/test_docker_runner.py
|
|
44
|
+
tests/test_doctor.py
|
|
43
45
|
tests/test_enrol.py
|
|
44
46
|
tests/test_executor.py
|
|
45
47
|
tests/test_hardening.py
|
|
@@ -47,4 +49,5 @@ tests/test_hardening_platform.py
|
|
|
47
49
|
tests/test_identity.py
|
|
48
50
|
tests/test_images_allowlist.py
|
|
49
51
|
tests/test_interfaces.py
|
|
50
|
-
tests/test_local_data.py
|
|
52
|
+
tests/test_local_data.py
|
|
53
|
+
tests/test_work_gate.py
|
|
@@ -10,7 +10,7 @@ name = "flashnode"
|
|
|
10
10
|
# hardcoded __version__ that made every agent register as 0.1.0.) Released from
|
|
11
11
|
# the Zolli-Labs/flashml monorepo by pushing a `flashnode-vX.Y.Z` tag, and only
|
|
12
12
|
# AFTER the flashruntime floor below is on PyPI.
|
|
13
|
-
version = "0.3.
|
|
13
|
+
version = "0.3.1"
|
|
14
14
|
description = "Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits."
|
|
15
15
|
readme = "README.md"
|
|
16
16
|
license = { text = "Apache-2.0" }
|
|
@@ -117,24 +117,23 @@ def test_work_cli_sets_module_capable_from_runner_choice(
|
|
|
117
117
|
monkeypatch.setenv("FLASHNODE_ALLOWED_IMAGES", "img:1")
|
|
118
118
|
monkeypatch.setenv("FLASHNODE_STATE_DIR", str(tmp_path))
|
|
119
119
|
|
|
120
|
-
# --runner docker and --runner argv both refuse to start
|
|
121
|
-
#
|
|
122
|
-
#
|
|
123
|
-
#
|
|
120
|
+
# --runner docker and --runner argv both refuse to start on a host that
|
|
121
|
+
# cannot run tasks — `flashnode work` now runs the doctor's checks as a
|
|
122
|
+
# fail-closed gate (flashnode/doctor.py) instead of the old
|
|
123
|
+
# `shutil.which("docker") is None`. That gate is correct and has its own
|
|
124
|
+
# tests in test_work_gate.py; here it is environment noise standing
|
|
125
|
+
# between us and the behaviour under test, which is purely how
|
|
124
126
|
# module_capable is derived from the runner choice.
|
|
125
127
|
#
|
|
126
|
-
#
|
|
127
|
-
# machine with Docker
|
|
128
|
-
# is
|
|
129
|
-
# (GitHub's macOS runners ship no Docker daemon).
|
|
128
|
+
# Report a healthy host. Without this the test passes on a developer
|
|
129
|
+
# machine with Docker AND the probe image cached, and fails everywhere
|
|
130
|
+
# else — which is the shape of the bug that first hit flashnode in CI on
|
|
131
|
+
# macOS (GitHub's macOS runners ship no Docker daemon).
|
|
130
132
|
#
|
|
131
|
-
#
|
|
132
|
-
#
|
|
133
|
-
#
|
|
134
|
-
|
|
135
|
-
import shutil
|
|
136
|
-
|
|
137
|
-
monkeypatch.setattr(shutil, "which", lambda name: f"/usr/local/bin/{name}")
|
|
133
|
+
# Stubbing run_checks rather than `shutil.which` because `which` is now
|
|
134
|
+
# only the first of six checks; faking it alone lets the gate proceed to
|
|
135
|
+
# `docker version` and a real `docker run`.
|
|
136
|
+
monkeypatch.setattr("flashnode.doctor.run_checks", lambda **kw: [])
|
|
138
137
|
|
|
139
138
|
captured = {}
|
|
140
139
|
real_discover = capabilities_mod.discover
|