flashnode 0.3.4__tar.gz → 0.3.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flashnode-0.3.4 → flashnode-0.3.5}/PKG-INFO +2 -2
- flashnode-0.3.5/flashnode/agent/cli.py +653 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/doctor.py +70 -4
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/argv_runner.py +19 -3
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/client.py +40 -5
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/docker_runner.py +13 -3
- flashnode-0.3.5/flashnode/executor/environments.py +423 -0
- flashnode-0.3.5/flashnode/executor/health.py +104 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/loop.py +126 -7
- flashnode-0.3.5/flashnode/executor/runner.py +229 -0
- flashnode-0.3.5/flashnode/executor/task_logs.py +218 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/trusted_runner.py +51 -3
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/identity/credentials.py +31 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/inventory/capabilities.py +6 -1
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode.egg-info/PKG-INFO +2 -2
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode.egg-info/SOURCES.txt +9 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode.egg-info/requires.txt +1 -1
- {flashnode-0.3.4 → flashnode-0.3.5}/pyproject.toml +20 -4
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_agent.py +8 -3
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_argv_runner.py +67 -7
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_capabilities.py +119 -0
- flashnode-0.3.5/tests/test_cli_env.py +178 -0
- flashnode-0.3.5/tests/test_cli_join.py +291 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_cli_trusted.py +44 -1
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_docker_runner.py +37 -1
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_doctor.py +75 -0
- flashnode-0.3.5/tests/test_environments.py +369 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_executor.py +415 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_loop_counters.py +78 -0
- flashnode-0.3.5/tests/test_task_logs.py +167 -0
- flashnode-0.3.5/tests/test_tier_health.py +126 -0
- flashnode-0.3.5/tests/test_trusted_runner.py +310 -0
- flashnode-0.3.5/tests/test_trusted_tier_endtoend.py +38 -0
- flashnode-0.3.5/tests/test_work_gate.py +287 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_work_status.py +66 -3
- flashnode-0.3.4/flashnode/agent/cli.py +0 -391
- flashnode-0.3.4/flashnode/executor/runner.py +0 -126
- flashnode-0.3.4/tests/test_trusted_runner.py +0 -92
- flashnode-0.3.4/tests/test_work_gate.py +0 -96
- {flashnode-0.3.4 → flashnode-0.3.5}/LICENSE +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/README.md +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/agent/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/agent/daemon.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/agent/kube.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/artifacts/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/benchmark/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/config/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/config/local_data.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/archives.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/evidence.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/hardening.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/executor/images.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/identity/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/identity/enrol.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/identity/store.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/inventory/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/inventory/gpu.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/status.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode/telemetry/__init__.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode.egg-info/dependency_links.txt +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode.egg-info/entry_points.txt +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/flashnode.egg-info/top_level.txt +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/setup.cfg +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_allowlist_drift.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_archives.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_checkpoint_relay.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_credentials.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_enrol.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_evidence.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_gpu_probe.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_hardening.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_hardening_platform.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_identity.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_images_allowlist.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_interfaces.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_local_data.py +0 -0
- {flashnode-0.3.4 → flashnode-0.3.5}/tests/test_status.py +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flashnode
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.5
|
|
4
4
|
Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
|
|
7
7
|
Requires-Python: >=3.10
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
License-File: LICENSE
|
|
10
|
-
Requires-Dist: flashruntime<0.
|
|
10
|
+
Requires-Dist: flashruntime<0.6,>=0.4.3
|
|
11
11
|
Requires-Dist: psutil>=5.9
|
|
12
12
|
Requires-Dist: websockets>=12
|
|
13
13
|
Requires-Dist: cryptography>=42
|
|
@@ -0,0 +1,653 @@
|
|
|
1
|
+
"""Command-line entry point for the FlashNode agent.
|
|
2
|
+
|
|
3
|
+
Target surface (see docs/SYSTEM_OVERVIEW.md §10):
|
|
4
|
+
|
|
5
|
+
flashnode join --code <one-time-code>
|
|
6
|
+
flashnode status
|
|
7
|
+
flashnode leave
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
from flashnode import __version__
|
|
15
|
+
|
|
16
|
+
USAGE = """\
|
|
17
|
+
flashnode {version} — FlashML open host agent (pre-release scaffold)
|
|
18
|
+
|
|
19
|
+
usage: flashnode <command>
|
|
20
|
+
|
|
21
|
+
commands:
|
|
22
|
+
agent run the node agent loop (register with FlashML Cloud + heartbeat)
|
|
23
|
+
work register with a FlashRuntime coordinator and execute leased tasks
|
|
24
|
+
(--coordinator URL | FLASHNODE_COORDINATOR_URL; --max-tasks N)
|
|
25
|
+
login enrol this machine — prints a code to approve in a browser
|
|
26
|
+
(--coordinator URL; --token TOKEN to skip the browser step)
|
|
27
|
+
logout remove the saved bearer token for a FlashRuntime coordinator
|
|
28
|
+
(--coordinator URL)
|
|
29
|
+
doctor check this machine can run tasks (docker engine, images, mounts)
|
|
30
|
+
env see and reclaim disk held by cached dependency environments
|
|
31
|
+
(list | purge — purge never touches node-id or credentials.json)
|
|
32
|
+
join connect this machine to a FlashML control plane (not yet implemented)
|
|
33
|
+
status show node identity, capabilities, and active leases (not yet implemented)
|
|
34
|
+
leave drain and disconnect (not yet implemented)
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _login(args: list[str]) -> int:
|
|
39
|
+
"""Enrol this machine.
|
|
40
|
+
|
|
41
|
+
The default path is device-code: print a short code, wait for a
|
|
42
|
+
signed-in human to approve it in a browser, save the token that comes
|
|
43
|
+
back. That is what the console tells volunteers to run, and until now it
|
|
44
|
+
could not work — `--token` was REQUIRED, and the enrolment flow issues
|
|
45
|
+
no token for anyone to paste. The API half (/v1alpha1/device/code,
|
|
46
|
+
/v1alpha1/device/token) had been built and simply had no client.
|
|
47
|
+
|
|
48
|
+
`--token` stays supported for a credential you already hold: CI, a
|
|
49
|
+
self-hosted coordinator, or re-pointing a machine with no browser to
|
|
50
|
+
hand.
|
|
51
|
+
"""
|
|
52
|
+
import argparse
|
|
53
|
+
|
|
54
|
+
from flashnode.identity.credentials import save_token
|
|
55
|
+
|
|
56
|
+
parser = argparse.ArgumentParser(
|
|
57
|
+
prog="flashnode login",
|
|
58
|
+
description="Enrol this machine with FlashML.",
|
|
59
|
+
)
|
|
60
|
+
parser.add_argument(
|
|
61
|
+
"--coordinator",
|
|
62
|
+
required=True,
|
|
63
|
+
help="FlashML Cloud API base URL (e.g. https://flashml-api.onrender.com)",
|
|
64
|
+
)
|
|
65
|
+
parser.add_argument(
|
|
66
|
+
"--token",
|
|
67
|
+
help="skip the browser step and save a token you already have",
|
|
68
|
+
)
|
|
69
|
+
opts = parser.parse_args(args)
|
|
70
|
+
|
|
71
|
+
if opts.token:
|
|
72
|
+
path = save_token(opts.coordinator, opts.token)
|
|
73
|
+
print(f"flashnode login: credential saved to {path}", file=sys.stderr)
|
|
74
|
+
return 0
|
|
75
|
+
|
|
76
|
+
from flashnode.identity.enrol import (
|
|
77
|
+
EnrolmentError,
|
|
78
|
+
describe_this_machine,
|
|
79
|
+
poll_for_token,
|
|
80
|
+
request_device_code,
|
|
81
|
+
)
|
|
82
|
+
from flashnode.identity.store import load_or_create_node_id
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
node_id = load_or_create_node_id()
|
|
86
|
+
except OSError as exc:
|
|
87
|
+
print(
|
|
88
|
+
f"flashnode login: cannot write this machine's identity: {exc}\n"
|
|
89
|
+
"Set FLASHNODE_STATE_DIR to a directory you can write to.",
|
|
90
|
+
file=sys.stderr,
|
|
91
|
+
)
|
|
92
|
+
return 1
|
|
93
|
+
|
|
94
|
+
hostname, platform_name = describe_this_machine()
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
start = request_device_code(
|
|
98
|
+
opts.coordinator, node_id, hostname, platform_name
|
|
99
|
+
)
|
|
100
|
+
except EnrolmentError as exc:
|
|
101
|
+
print(f"flashnode login: {exc}", file=sys.stderr)
|
|
102
|
+
return 1
|
|
103
|
+
|
|
104
|
+
# stdout, not stderr: this is the output the person is here for, and it
|
|
105
|
+
# should survive being piped.
|
|
106
|
+
#
|
|
107
|
+
# flush=True is load-bearing. Python block-buffers stdout when it is not
|
|
108
|
+
# a terminal, so piping `flashnode login` anywhere — tee, a log, a setup
|
|
109
|
+
# script — showed nothing at all while the process sat waiting for an
|
|
110
|
+
# approval of a code it had never displayed.
|
|
111
|
+
print(flush=True)
|
|
112
|
+
print(f" Your code: {start.user_code}", flush=True)
|
|
113
|
+
print(f" Approve at: {start.verification_uri}", flush=True)
|
|
114
|
+
print(flush=True)
|
|
115
|
+
print(
|
|
116
|
+
"Open that on any device you're signed in on — your phone is fine.",
|
|
117
|
+
flush=True,
|
|
118
|
+
)
|
|
119
|
+
print("Waiting for approval… (Ctrl-C to cancel)", flush=True)
|
|
120
|
+
|
|
121
|
+
try:
|
|
122
|
+
token = poll_for_token(
|
|
123
|
+
opts.coordinator, start.device_code, interval=start.interval
|
|
124
|
+
)
|
|
125
|
+
except EnrolmentError as exc:
|
|
126
|
+
print(f"\nflashnode login: {exc}", file=sys.stderr)
|
|
127
|
+
return 1
|
|
128
|
+
except KeyboardInterrupt:
|
|
129
|
+
# Cancelling is a normal act, not a crash. The code expires unused.
|
|
130
|
+
print("\nflashnode login: cancelled.", file=sys.stderr)
|
|
131
|
+
return 130
|
|
132
|
+
|
|
133
|
+
path = save_token(opts.coordinator, token)
|
|
134
|
+
print(f"\nApproved. This machine is enrolled — credential saved to {path}.")
|
|
135
|
+
# The coordinator has to be IN the printed command. A volunteer pastes
|
|
136
|
+
# this line verbatim; without it `work` defaults to localhost:8100,
|
|
137
|
+
# looks up a credential that was saved under a different key, and dies
|
|
138
|
+
# on a bare 401 running the exact command the CLI just gave them.
|
|
139
|
+
# Lazy import, matching this file's existing style (see the imports a few
|
|
140
|
+
# lines up): health.py pulls CheckResult/default_workdir/run_checks from
|
|
141
|
+
# flashnode.doctor at module scope, and this keeps that import graph from
|
|
142
|
+
# having to resolve until a login actually happens.
|
|
143
|
+
from flashnode.doctor import check_cli_on_path
|
|
144
|
+
from flashnode.executor.health import suggested_runner
|
|
145
|
+
|
|
146
|
+
# `shutil.which`, not the full docker set: suggested_runner reads only
|
|
147
|
+
# the CLI-on-PATH verdict, and probing the daemon here would make
|
|
148
|
+
# `flashnode login` hang for up to 30s against a wedged docker install
|
|
149
|
+
# to answer a question a PATH lookup already settles.
|
|
150
|
+
hint = suggested_runner([check_cli_on_path()])
|
|
151
|
+
print(
|
|
152
|
+
"Start contributing with: "
|
|
153
|
+
f"flashnode work --runner {hint} --coordinator {opts.coordinator}"
|
|
154
|
+
)
|
|
155
|
+
return 0
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
#: Tiers that shell out to the `docker` binary. Everything else runs the
|
|
159
|
+
#: workload on the host directly and has no container to diagnose.
|
|
160
|
+
_CONTAINER_TIERS = ("docker", "argv")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _next_step(runner: str, *, restarting: bool = False, docker_absent: bool = False) -> str:
|
|
164
|
+
"""What to tell an operator whose host just refused to work, in terms
|
|
165
|
+
their tier can actually act on.
|
|
166
|
+
|
|
167
|
+
`flashnode doctor` runs `run_checks(pull=True)` — the DOCKER set. Telling
|
|
168
|
+
a host that has no Docker, which is the entire reason it is on a
|
|
169
|
+
no-container tier, to run it hands back `[FAIL] docker CLI on PATH` and a
|
|
170
|
+
non-zero exit: the same misleading verdict this tier exists to remove,
|
|
171
|
+
reached one hop later. That is what a real pod was told on 2026-08-09.
|
|
172
|
+
The "image pull this skipped" clause is meaningless there too — the
|
|
173
|
+
no-container tiers never pull anything.
|
|
174
|
+
|
|
175
|
+
`docker_absent=True` is the narrower case: `runner` IS a container tier
|
|
176
|
+
(docker/argv), but the `docker` CLI itself was not found — the same
|
|
177
|
+
2026-08-09 shape one tier over. `flashnode doctor` is still the right
|
|
178
|
+
command here (this host really does need Docker), but "including the
|
|
179
|
+
image pull this skipped" is false on it: there was never going to be a
|
|
180
|
+
pull, because the CLI check that gates one already failed. Set by the
|
|
181
|
+
caller from the same startup check set this trailer follows, never
|
|
182
|
+
guessed here.
|
|
183
|
+
|
|
184
|
+
`restarting=True` is the quarantine path, where the host had already
|
|
185
|
+
started and the operator is being told what to do before starting again.
|
|
186
|
+
"""
|
|
187
|
+
if runner in _CONTAINER_TIERS:
|
|
188
|
+
if restarting:
|
|
189
|
+
return "Fix the above, then `flashnode doctor` to confirm before restarting."
|
|
190
|
+
if docker_absent:
|
|
191
|
+
return ("Run `flashnode doctor` for the full check. There is no "
|
|
192
|
+
"`docker` here at all, so there is no image pull to skip.")
|
|
193
|
+
return ("Run `flashnode doctor` for the full check, including the "
|
|
194
|
+
"image pull this skipped.")
|
|
195
|
+
verb = "start" if restarting else "run"
|
|
196
|
+
return (
|
|
197
|
+
f"Fix the above, then {verb} `flashnode work --runner {runner}` again.\n"
|
|
198
|
+
f"Do not run `flashnode doctor` here: it checks the Docker tier, which\n"
|
|
199
|
+
f"--runner {runner} never uses, so on this machine it fails by design."
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _logout(args: list[str]) -> int:
|
|
204
|
+
import argparse
|
|
205
|
+
|
|
206
|
+
from flashnode.identity.credentials import clear_token, credentials_path
|
|
207
|
+
|
|
208
|
+
parser = argparse.ArgumentParser(prog="flashnode logout")
|
|
209
|
+
parser.add_argument("--coordinator", required=True, help="FlashRuntime coordinator base URL")
|
|
210
|
+
opts = parser.parse_args(args)
|
|
211
|
+
|
|
212
|
+
removed = clear_token(opts.coordinator)
|
|
213
|
+
if removed:
|
|
214
|
+
print(f"flashnode logout: credential removed from {credentials_path()}", file=sys.stderr)
|
|
215
|
+
else:
|
|
216
|
+
print(f"flashnode logout: no saved credential for {opts.coordinator}", file=sys.stderr)
|
|
217
|
+
return 0
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _work(args: list[str]) -> int:
|
|
221
|
+
import argparse
|
|
222
|
+
import logging
|
|
223
|
+
import os
|
|
224
|
+
import shutil
|
|
225
|
+
import signal
|
|
226
|
+
|
|
227
|
+
from flashnode.executor import CoordinatorClient, ExecutorLoop
|
|
228
|
+
from flashnode.identity.store import load_or_create_node_id
|
|
229
|
+
from flashnode.inventory.capabilities import discover
|
|
230
|
+
|
|
231
|
+
logging.basicConfig(
|
|
232
|
+
level=logging.INFO,
|
|
233
|
+
format='{"ts":"%(asctime)s","level":"%(levelname)s","service":"flashnode","msg":%(message)s}',
|
|
234
|
+
)
|
|
235
|
+
parser = argparse.ArgumentParser(prog="flashnode work")
|
|
236
|
+
parser.add_argument(
|
|
237
|
+
"--coordinator",
|
|
238
|
+
# No default here: an omitted --coordinator is resolved below from
|
|
239
|
+
# the credentials this machine actually holds. Baking localhost in
|
|
240
|
+
# made every enrolled volunteer's first command 401.
|
|
241
|
+
default=os.environ.get("FLASHNODE_COORDINATOR_URL"),
|
|
242
|
+
help="FlashRuntime coordinator base URL (default: the one this "
|
|
243
|
+
"machine enrolled against with `flashnode login`)",
|
|
244
|
+
)
|
|
245
|
+
parser.add_argument(
|
|
246
|
+
"--runner",
|
|
247
|
+
choices=["subprocess", "docker", "argv", "trusted"],
|
|
248
|
+
default=os.environ.get("FLASHNODE_RUNNER", "subprocess"),
|
|
249
|
+
help="task execution tier (docker/argv need the docker CLI on PATH; "
|
|
250
|
+
"trusted runs pool jobs unsandboxed — opt-in only — and is the "
|
|
251
|
+
"only tier leased jobs that declare dependencies)",
|
|
252
|
+
)
|
|
253
|
+
parser.add_argument("--max-tasks", type=int, default=None)
|
|
254
|
+
parser.add_argument("--poll-seconds", type=float, default=1.0)
|
|
255
|
+
parser.add_argument(
|
|
256
|
+
"--log-json", action="store_true",
|
|
257
|
+
help="keep the machine-readable JSON log instead of the live status view",
|
|
258
|
+
)
|
|
259
|
+
parser.add_argument(
|
|
260
|
+
"--max-consecutive-failures", type=int,
|
|
261
|
+
default=int(os.environ.get("FLASHNODE_MAX_CONSECUTIVE_FAILURES", "3")),
|
|
262
|
+
help="host-side failures in a row before re-checking this machine and "
|
|
263
|
+
"stopping if it is broken (0 disables)",
|
|
264
|
+
)
|
|
265
|
+
opts = parser.parse_args(args)
|
|
266
|
+
|
|
267
|
+
# Resolved BEFORE the runner branch so the one health binding below can
|
|
268
|
+
# hand it to whichever tier's check set applies — the directory a check
|
|
269
|
+
# asks about must be the directory the loop will actually stage tasks in.
|
|
270
|
+
workdir_base = os.environ.get("FLASHNODE_WORKDIR") or None
|
|
271
|
+
|
|
272
|
+
# Every branch below assigns a runner — there is no `runner = None`
|
|
273
|
+
# fallthrough any more, because the health binding after them has to have
|
|
274
|
+
# something to ask.
|
|
275
|
+
if opts.runner in ("docker", "argv"):
|
|
276
|
+
# OPTIONAL, and additive. The built-in namespace allowlist
|
|
277
|
+
# (executor/images.py DEFAULT_ALLOWED_IMAGE_PREFIXES) is what a
|
|
278
|
+
# volunteer runs on, so this env var is no longer required and an
|
|
279
|
+
# empty value is no longer a refusal to start.
|
|
280
|
+
#
|
|
281
|
+
# It used to be mandatory, which quietly capped the project at the
|
|
282
|
+
# number of machines whose owners would hand-maintain a list of image
|
|
283
|
+
# references: every image we published stranded every host until its
|
|
284
|
+
# owner edited the variable, so security fixes would reach a fraction
|
|
285
|
+
# of the fleet. Setting it now means "also allow these", for
|
|
286
|
+
# self-hosting the stack or for integration tests.
|
|
287
|
+
images = frozenset(
|
|
288
|
+
i.strip() for i in os.environ.get("FLASHNODE_ALLOWED_IMAGES", "").split(",") if i.strip()
|
|
289
|
+
)
|
|
290
|
+
if opts.runner == "docker":
|
|
291
|
+
from flashnode.executor.docker_runner import DockerRunner
|
|
292
|
+
|
|
293
|
+
runner = DockerRunner(allowed_images=images)
|
|
294
|
+
else:
|
|
295
|
+
from flashnode.executor.argv_runner import ArgvDockerRunner
|
|
296
|
+
|
|
297
|
+
runner = ArgvDockerRunner(
|
|
298
|
+
allowed_images=images,
|
|
299
|
+
cpus=float(os.environ.get("FLASHNODE_MAX_CPUS", "2.0")),
|
|
300
|
+
memory_gb=float(os.environ.get("FLASHNODE_MAX_MEMORY_GB", "2.0")),
|
|
301
|
+
timeout_seconds=float(os.environ.get("FLASHNODE_TASK_TIMEOUT_S", "3600")),
|
|
302
|
+
max_output_bytes=int(
|
|
303
|
+
os.environ.get("FLASHNODE_MAX_OUTPUT_BYTES", str(2 * 1024**3))
|
|
304
|
+
),
|
|
305
|
+
)
|
|
306
|
+
elif opts.runner == "trusted":
|
|
307
|
+
# This tier exists for hosts that ARE a container already — a RunPod
|
|
308
|
+
# pod, a Colab VM — and so can never nest a Docker daemon. The docker
|
|
309
|
+
# checks are not merely unnecessary here, they are unsatisfiable.
|
|
310
|
+
from flashnode.executor.trusted_runner import TrustedArgvRunner
|
|
311
|
+
|
|
312
|
+
print(
|
|
313
|
+
"trusted runner: pool jobs from your team run UNSANDBOXED on this\n"
|
|
314
|
+
"machine — no container, no network isolation. Only continue if\n"
|
|
315
|
+
"every member of your pool is someone you trust to run code here."
|
|
316
|
+
)
|
|
317
|
+
runner = TrustedArgvRunner()
|
|
318
|
+
else:
|
|
319
|
+
# Constructed HERE rather than left to ExecutorLoop's own default, so
|
|
320
|
+
# that every tier reaches the health binding below with a runner to
|
|
321
|
+
# ask. The instance is what the loop would have built for itself.
|
|
322
|
+
from flashnode.executor.runner import SubprocessRunner
|
|
323
|
+
|
|
324
|
+
runner = SubprocessRunner()
|
|
325
|
+
|
|
326
|
+
# The tier is resolved ONCE, here, by asking the runner we just built.
|
|
327
|
+
# Two call sites each deciding their own tier is precisely the bug this
|
|
328
|
+
# replaced: the startup gate exempted `trusted` and the loop's
|
|
329
|
+
# post-failure re-check did not, so three failures of any cause
|
|
330
|
+
# quarantined a pod and blamed Docker for it (2026-08-09).
|
|
331
|
+
#
|
|
332
|
+
# What the docker tier asks is unchanged, and deliberately so. Both
|
|
333
|
+
# sandboxed tiers shell out to the `docker` binary directly
|
|
334
|
+
# (subprocess.run(["docker", ...])); if it isn't installed that call
|
|
335
|
+
# raises FileNotFoundError deep inside a task attempt. And a `docker`
|
|
336
|
+
# binary on PATH says nothing about the daemon behind it, the credential
|
|
337
|
+
# helper Docker consults when it pulls, or whether this machine's work
|
|
338
|
+
# directory is even visible inside the VM. Both hosts that stopped the
|
|
339
|
+
# 2026-08-02 §10 run-through passed the old `shutil.which` check and then
|
|
340
|
+
# failed every task they claimed — docker_runner raises
|
|
341
|
+
# TaskExecutionError, loop.py calls fail() and claims the next one,
|
|
342
|
+
# forever, silently. `docker_tier_checks` still runs with pull=False: an
|
|
343
|
+
# agent is a long-running daemon on someone else's machine, and a
|
|
344
|
+
# transient registry blip must not stop one whose images are already
|
|
345
|
+
# cached. `flashnode doctor` does the pull.
|
|
346
|
+
from flashnode.doctor import NON_BLOCKING_STATUSES, format_results
|
|
347
|
+
|
|
348
|
+
def _blocking_problems() -> list:
|
|
349
|
+
"""What the loop calls after a streak of host-side failures.
|
|
350
|
+
|
|
351
|
+
Bound to the same runner as the startup gate below, so the two
|
|
352
|
+
cannot disagree about which tier this host is in — and so `loop.py`
|
|
353
|
+
never has to import the doctor (which would close a
|
|
354
|
+
loop -> doctor -> executor -> loop cycle).
|
|
355
|
+
|
|
356
|
+
Filtered HERE with doctor's own NON_BLOCKING_STATUSES rather than a
|
|
357
|
+
`!= "ok"` test: the GPU check reports "info" and never fails,
|
|
358
|
+
because most volunteers have no GPU and blocking on it would lock
|
|
359
|
+
the entire existing fleet out of the CPU work they signed up for.
|
|
360
|
+
Keeping that predicate in one place is what stops `flashnode work`
|
|
361
|
+
and `flashnode doctor` disagreeing about which verdicts are fatal.
|
|
362
|
+
The membership test is the way round that makes an unknown status
|
|
363
|
+
BLOCK, matching doctor.exit_code.
|
|
364
|
+
"""
|
|
365
|
+
return [r for r in runner.health_checks(workdir=workdir_base)
|
|
366
|
+
if r.status not in NON_BLOCKING_STATUSES]
|
|
367
|
+
|
|
368
|
+
startup_problems = _blocking_problems()
|
|
369
|
+
if startup_problems:
|
|
370
|
+
# Read off the SAME check set the trailer follows, never guessed:
|
|
371
|
+
# a host whose `docker CLI on PATH` check failed has no image pull
|
|
372
|
+
# to have skipped, and the trailer below must not claim it did.
|
|
373
|
+
docker_absent = any(
|
|
374
|
+
r.name == "docker CLI on PATH" and r.status == "fail"
|
|
375
|
+
for r in startup_problems
|
|
376
|
+
)
|
|
377
|
+
# next_step="" because the trailer below IS the next step, and it is
|
|
378
|
+
# chosen by tier. Leaving format_results' default in place printed
|
|
379
|
+
# "re-run `flashnode doctor`" to a host the doctor cannot certify.
|
|
380
|
+
print(
|
|
381
|
+
f"flashnode work: this machine cannot run tasks with "
|
|
382
|
+
f"--runner {opts.runner}.\n"
|
|
383
|
+
+ format_results(startup_problems, next_step="")
|
|
384
|
+
+ "\n\n" + _next_step(opts.runner, docker_absent=docker_absent),
|
|
385
|
+
file=sys.stderr,
|
|
386
|
+
)
|
|
387
|
+
return 2
|
|
388
|
+
|
|
389
|
+
from flashnode.identity.credentials import load_token, resolve_coordinator
|
|
390
|
+
|
|
391
|
+
try:
|
|
392
|
+
opts.coordinator = resolve_coordinator(opts.coordinator)
|
|
393
|
+
except ValueError as exc:
|
|
394
|
+
print(f"flashnode work: {exc}", file=sys.stderr)
|
|
395
|
+
return 1
|
|
396
|
+
|
|
397
|
+
node_id = load_or_create_node_id()
|
|
398
|
+
client = CoordinatorClient(
|
|
399
|
+
opts.coordinator,
|
|
400
|
+
join_code=os.environ.get("FLASHNODE_JOIN_CODE") or None,
|
|
401
|
+
token=load_token(opts.coordinator),
|
|
402
|
+
)
|
|
403
|
+
registration = discover(
|
|
404
|
+
node_id, kubernetes_node="", node_meta=None,
|
|
405
|
+
argv_capable=(opts.runner == "argv"),
|
|
406
|
+
# An argv-only volunteer has no module runner behind it: advertise
|
|
407
|
+
# module_capable=False so the coordinator's placement gate stops
|
|
408
|
+
# routing "python -m <module>" tasks here (F1) — otherwise those
|
|
409
|
+
# tasks burn every attempt against ArgvDockerRunner's payload
|
|
410
|
+
# rejection before the job ever fails for real.
|
|
411
|
+
# `trusted` runs argv through TrustedArgvRunner, which refuses
|
|
412
|
+
# `python -m <module>` payloads exactly as the argv tier does. Both
|
|
413
|
+
# must advertise module_capable=False, or this machine is eligible
|
|
414
|
+
# for public MODULE tasks it fails every attempt of — burning the
|
|
415
|
+
# volunteer's electricity and the job's retries on a mismatch the
|
|
416
|
+
# registration already knew about.
|
|
417
|
+
module_capable=(opts.runner not in ("argv", "trusted")),
|
|
418
|
+
# Set true ONLY by the explicit --runner trusted opt-in, never
|
|
419
|
+
# inferred — the scheduler's placement gate treats this field as
|
|
420
|
+
# fail-closed (flashruntime.scheduler), so a node that merely has a
|
|
421
|
+
# shell available must not advertise it.
|
|
422
|
+
unsandboxed_argv_capable=(opts.runner == "trusted"),
|
|
423
|
+
# `trusted` ONLY, and deliberately narrower than "the tiers that
|
|
424
|
+
# could technically run pip".
|
|
425
|
+
#
|
|
426
|
+
# The feature exists for hosts that cannot run a container — a
|
|
427
|
+
# RunPod pod, a Colab notebook — and must execute the submitter's
|
|
428
|
+
# own argv; that is the trusted tier, and only there does the job's
|
|
429
|
+
# venv hold everything the job runs.
|
|
430
|
+
#
|
|
431
|
+
# `subprocess` is excluded even though it also runs on the host: it
|
|
432
|
+
# executes the AGENT's allowlisted `flashml_workloads.*` modules,
|
|
433
|
+
# which live in the agent's environment, not the job's. Sending it
|
|
434
|
+
# into a job venv built from the job's declared dependencies alone
|
|
435
|
+
# produces ModuleNotFoundError — the exact failure this feature
|
|
436
|
+
# exists to eliminate. Giving that venv access to the agent's own
|
|
437
|
+
# environment is not the fix either: a declared dependency would
|
|
438
|
+
# then shadow the agent's.
|
|
439
|
+
#
|
|
440
|
+
# The container tiers cannot install at all — --network none and a
|
|
441
|
+
# read-only rootfs. Physics, not policy.
|
|
442
|
+
#
|
|
443
|
+
# Fail closed and narrow. Widening this to another tier is a
|
|
444
|
+
# deliberate change that must arrive with its own tests.
|
|
445
|
+
can_install_dependencies=(opts.runner == "trusted"),
|
|
446
|
+
)
|
|
447
|
+
client.register(registration)
|
|
448
|
+
loop = ExecutorLoop(
|
|
449
|
+
client, node_id, runner=runner,
|
|
450
|
+
poll_seconds=opts.poll_seconds, workdir_base=workdir_base,
|
|
451
|
+
registration=registration, # survives coordinator restarts
|
|
452
|
+
health_check=_blocking_problems,
|
|
453
|
+
max_consecutive_failures=opts.max_consecutive_failures,
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
def _stop(signum, frame): # noqa: ARG001
|
|
457
|
+
loop.stop_event.set()
|
|
458
|
+
|
|
459
|
+
signal.signal(signal.SIGTERM, _stop)
|
|
460
|
+
signal.signal(signal.SIGINT, _stop)
|
|
461
|
+
view = None
|
|
462
|
+
if sys.stdout.isatty() and not opts.log_json:
|
|
463
|
+
# Two writers redrawing one terminal is unreadable, so the JSON
|
|
464
|
+
# handler goes when the view arrives. --log-json keeps it, and a
|
|
465
|
+
# non-TTY never gets here — ANSI cursor movement into a pipe or a
|
|
466
|
+
# systemd journal is corruption, not output.
|
|
467
|
+
from flashnode.status import StatusView
|
|
468
|
+
|
|
469
|
+
logging.getLogger().handlers.clear()
|
|
470
|
+
view = StatusView(loop, coordinator=opts.coordinator,
|
|
471
|
+
version=__version__, stream=sys.stdout)
|
|
472
|
+
view.start()
|
|
473
|
+
try:
|
|
474
|
+
accepted = loop.run(max_tasks=opts.max_tasks)
|
|
475
|
+
finally:
|
|
476
|
+
if view is not None:
|
|
477
|
+
view.stop()
|
|
478
|
+
|
|
479
|
+
if getattr(loop, "quarantined", False):
|
|
480
|
+
from flashnode.doctor import format_results
|
|
481
|
+
|
|
482
|
+
print(
|
|
483
|
+
"\nflashnode work: stopping — this machine can no longer run "
|
|
484
|
+
"tasks.\n"
|
|
485
|
+
+ format_results(loop.health_report or [], next_step="")
|
|
486
|
+
+ "\n\n" + _next_step(opts.runner, restarting=True),
|
|
487
|
+
file=sys.stderr,
|
|
488
|
+
)
|
|
489
|
+
return 2
|
|
490
|
+
print(f"flashnode work: {accepted} task(s) accepted", file=sys.stderr)
|
|
491
|
+
return 0
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _human_bytes(n: int) -> str:
|
|
495
|
+
"""`n` bytes, rendered the way a volunteer reads disk usage — whole B,
|
|
496
|
+
one decimal place from KB up."""
|
|
497
|
+
size = float(n)
|
|
498
|
+
for unit in ("B", "KB", "MB", "GB"):
|
|
499
|
+
if size < 1024 or unit == "GB":
|
|
500
|
+
return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
|
|
501
|
+
size /= 1024
|
|
502
|
+
return f"{size:.1f} GB" # pragma: no cover - unreachable, GB always returns above
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _env(args: list[str]) -> int:
|
|
506
|
+
"""`flashnode env list|purge` — the disk cached dependency environments
|
|
507
|
+
hold, and a way to give it back without `rm -rf ~/.flashnode`.
|
|
508
|
+
|
|
509
|
+
`purge` deletes only what `EnvironmentCache.entries()` reports — the
|
|
510
|
+
contents of `$FLASHNODE_STATE_DIR/envs/`. It never touches the state
|
|
511
|
+
directory itself, so `node-id` and `credentials.json`, its siblings, are
|
|
512
|
+
untouched: that is the entire difference between this command and
|
|
513
|
+
un-enrolling the machine.
|
|
514
|
+
"""
|
|
515
|
+
import argparse
|
|
516
|
+
import shutil
|
|
517
|
+
from datetime import datetime
|
|
518
|
+
|
|
519
|
+
from flashnode.executor.environments import EnvironmentCache
|
|
520
|
+
|
|
521
|
+
parser = argparse.ArgumentParser(prog="flashnode env")
|
|
522
|
+
sub = parser.add_subparsers(dest="subcommand", required=True)
|
|
523
|
+
sub.add_parser("list", help="show cached environments, their size, and last use")
|
|
524
|
+
sub.add_parser(
|
|
525
|
+
"purge",
|
|
526
|
+
help="delete all cached environments (identity and credentials "
|
|
527
|
+
"survive; stop `flashnode work` first — this can break a task "
|
|
528
|
+
"in flight)",
|
|
529
|
+
)
|
|
530
|
+
opts = parser.parse_args(args)
|
|
531
|
+
|
|
532
|
+
cache = EnvironmentCache()
|
|
533
|
+
entries = cache.entries() # most recently used first
|
|
534
|
+
|
|
535
|
+
# Scratch directories from builds that were KILLED — SIGKILL, the OOM
|
|
536
|
+
# killer, a power cut. `entries()` skips dotted names, so without this
|
|
537
|
+
# the two commands whose whole job is to account for disk were the two
|
|
538
|
+
# that could not see a multi-GB directory.
|
|
539
|
+
stale_bytes = cache.stale_build_bytes()
|
|
540
|
+
|
|
541
|
+
if opts.subcommand == "list":
|
|
542
|
+
if not entries and not stale_bytes:
|
|
543
|
+
print("flashnode env: no environments cached — nothing is using disk yet.")
|
|
544
|
+
return 0
|
|
545
|
+
for entry in entries:
|
|
546
|
+
when = datetime.fromtimestamp(entry.last_used).strftime("%Y-%m-%d %H:%M")
|
|
547
|
+
print(f" {entry.key} {_human_bytes(entry.size_bytes):>8} last used {when}")
|
|
548
|
+
total = sum(e.size_bytes for e in entries) + stale_bytes
|
|
549
|
+
if stale_bytes:
|
|
550
|
+
print(
|
|
551
|
+
f" leftover build directories {_human_bytes(stale_bytes):>8} "
|
|
552
|
+
"from interrupted builds"
|
|
553
|
+
)
|
|
554
|
+
print(
|
|
555
|
+
f"total: {_human_bytes(total)} across {len(entries)} environment(s) "
|
|
556
|
+
f"at {cache.root}"
|
|
557
|
+
)
|
|
558
|
+
if stale_bytes:
|
|
559
|
+
print(
|
|
560
|
+
" the leftover build directories are swept by the next build, "
|
|
561
|
+
"or now by `flashnode env purge`"
|
|
562
|
+
)
|
|
563
|
+
return 0
|
|
564
|
+
|
|
565
|
+
# purge
|
|
566
|
+
#
|
|
567
|
+
# This runs in a SEPARATE process from the agent. The in-process
|
|
568
|
+
# eviction path is safe only because ExecutorLoop holds one lease at a
|
|
569
|
+
# time and `_evict` protects the entry it just handed out; purge
|
|
570
|
+
# sidesteps both and can delete the venv a running task's interpreter
|
|
571
|
+
# lives in. Refusing would beat warning, but nothing in the state
|
|
572
|
+
# directory records a running agent — no pidfile, no lock — and a
|
|
573
|
+
# guessed signal (a stale pid, an mtime heuristic) would refuse on a
|
|
574
|
+
# machine with no agent at all. So: say it, plainly, every time, and
|
|
575
|
+
# before the deletion rather than after it.
|
|
576
|
+
print(
|
|
577
|
+
"flashnode env purge: warning — if `flashnode work` is running on "
|
|
578
|
+
"this machine, this can delete the environment a task is running in "
|
|
579
|
+
"and fail that task. This command cannot tell whether an agent is "
|
|
580
|
+
"running. Stop the agent first if you are not sure."
|
|
581
|
+
)
|
|
582
|
+
# Counted AFTER each removal, never before. The old version totalled the
|
|
583
|
+
# entries up front and deleted with `ignore_errors=True`, so a
|
|
584
|
+
# permission error printed the full figure as reclaimed — on the one
|
|
585
|
+
# command whose entire value is that number, and which a volunteer has
|
|
586
|
+
# no other way to check.
|
|
587
|
+
freed = 0
|
|
588
|
+
removed = 0
|
|
589
|
+
survivors = []
|
|
590
|
+
for entry in entries:
|
|
591
|
+
shutil.rmtree(entry.path, ignore_errors=True)
|
|
592
|
+
if entry.path.exists():
|
|
593
|
+
survivors.append(entry.path)
|
|
594
|
+
else:
|
|
595
|
+
freed += entry.size_bytes
|
|
596
|
+
removed += 1
|
|
597
|
+
stale_freed = cache.sweep_stale_builds() # counts only what actually went
|
|
598
|
+
freed += stale_freed
|
|
599
|
+
survivors.extend(cache.stale_build_dirs())
|
|
600
|
+
leftovers = (
|
|
601
|
+
f" (including {_human_bytes(stale_freed)} of leftover build "
|
|
602
|
+
"directories from interrupted builds)"
|
|
603
|
+
if stale_freed
|
|
604
|
+
else ""
|
|
605
|
+
)
|
|
606
|
+
print(
|
|
607
|
+
f"flashnode env purge: freed {_human_bytes(freed)} across "
|
|
608
|
+
f"{removed} environment(s){leftovers}. node-id and credentials.json "
|
|
609
|
+
"were not touched."
|
|
610
|
+
)
|
|
611
|
+
if survivors:
|
|
612
|
+
noun = "directory" if len(survivors) == 1 else "directories"
|
|
613
|
+
print(
|
|
614
|
+
f"flashnode env purge: could not remove {len(survivors)} {noun} "
|
|
615
|
+
"— the disk it holds was NOT reclaimed:"
|
|
616
|
+
)
|
|
617
|
+
for path in survivors:
|
|
618
|
+
print(f" {path}")
|
|
619
|
+
print(
|
|
620
|
+
" check the permissions on those paths, or whether another "
|
|
621
|
+
"process is holding them open, and run purge again."
|
|
622
|
+
)
|
|
623
|
+
return 1
|
|
624
|
+
return 0
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def main(argv: list[str] | None = None) -> int:
|
|
628
|
+
args = sys.argv[1:] if argv is None else argv
|
|
629
|
+
if args and args[0] == "agent":
|
|
630
|
+
from flashnode.agent.daemon import main as agent_main
|
|
631
|
+
|
|
632
|
+
return agent_main()
|
|
633
|
+
if args and args[0] == "work":
|
|
634
|
+
return _work(args[1:])
|
|
635
|
+
if args and args[0] == "doctor":
|
|
636
|
+
from flashnode.doctor import doctor_main
|
|
637
|
+
|
|
638
|
+
return doctor_main(args[1:])
|
|
639
|
+
if args and args[0] == "login":
|
|
640
|
+
return _login(args[1:])
|
|
641
|
+
if args and args[0] == "logout":
|
|
642
|
+
return _logout(args[1:])
|
|
643
|
+
if args and args[0] == "env":
|
|
644
|
+
return _env(args[1:])
|
|
645
|
+
print(USAGE.format(version=__version__), end="")
|
|
646
|
+
if args and args[0] in {"join", "status", "leave"}:
|
|
647
|
+
print(f"\nerror: '{args[0]}' is not implemented yet in this scaffold.", file=sys.stderr)
|
|
648
|
+
return 1
|
|
649
|
+
return 0
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
if __name__ == "__main__":
|
|
653
|
+
raise SystemExit(main())
|