flashnode 0.3.3__tar.gz → 0.3.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {flashnode-0.3.3 → flashnode-0.3.5}/PKG-INFO +2 -2
  2. flashnode-0.3.5/flashnode/agent/cli.py +653 -0
  3. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/doctor.py +70 -4
  4. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/argv_runner.py +19 -3
  5. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/client.py +40 -5
  6. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/docker_runner.py +13 -3
  7. flashnode-0.3.5/flashnode/executor/environments.py +423 -0
  8. flashnode-0.3.5/flashnode/executor/health.py +104 -0
  9. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/loop.py +126 -7
  10. flashnode-0.3.5/flashnode/executor/runner.py +229 -0
  11. flashnode-0.3.5/flashnode/executor/task_logs.py +218 -0
  12. flashnode-0.3.5/flashnode/executor/trusted_runner.py +131 -0
  13. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/identity/credentials.py +31 -0
  14. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/inventory/capabilities.py +9 -1
  15. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode.egg-info/PKG-INFO +2 -2
  16. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode.egg-info/SOURCES.txt +12 -0
  17. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode.egg-info/requires.txt +1 -1
  18. {flashnode-0.3.3 → flashnode-0.3.5}/pyproject.toml +33 -4
  19. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_agent.py +8 -3
  20. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_argv_runner.py +67 -7
  21. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_capabilities.py +119 -0
  22. flashnode-0.3.5/tests/test_cli_env.py +178 -0
  23. flashnode-0.3.5/tests/test_cli_join.py +291 -0
  24. flashnode-0.3.5/tests/test_cli_trusted.py +117 -0
  25. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_docker_runner.py +37 -1
  26. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_doctor.py +75 -0
  27. flashnode-0.3.5/tests/test_environments.py +369 -0
  28. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_executor.py +415 -0
  29. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_loop_counters.py +78 -0
  30. flashnode-0.3.5/tests/test_task_logs.py +167 -0
  31. flashnode-0.3.5/tests/test_tier_health.py +126 -0
  32. flashnode-0.3.5/tests/test_trusted_runner.py +310 -0
  33. flashnode-0.3.5/tests/test_trusted_tier_endtoend.py +38 -0
  34. flashnode-0.3.5/tests/test_work_gate.py +287 -0
  35. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_work_status.py +66 -3
  36. flashnode-0.3.3/flashnode/agent/cli.py +0 -372
  37. flashnode-0.3.3/flashnode/executor/runner.py +0 -126
  38. flashnode-0.3.3/tests/test_work_gate.py +0 -96
  39. {flashnode-0.3.3 → flashnode-0.3.5}/LICENSE +0 -0
  40. {flashnode-0.3.3 → flashnode-0.3.5}/README.md +0 -0
  41. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/__init__.py +0 -0
  42. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/agent/__init__.py +0 -0
  43. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/agent/daemon.py +0 -0
  44. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/agent/kube.py +0 -0
  45. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/artifacts/__init__.py +0 -0
  46. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/benchmark/__init__.py +0 -0
  47. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/config/__init__.py +0 -0
  48. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/config/local_data.py +0 -0
  49. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/__init__.py +0 -0
  50. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/archives.py +0 -0
  51. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/evidence.py +0 -0
  52. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/hardening.py +0 -0
  53. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/executor/images.py +0 -0
  54. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/identity/__init__.py +0 -0
  55. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/identity/enrol.py +0 -0
  56. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/identity/store.py +0 -0
  57. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/inventory/__init__.py +0 -0
  58. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/inventory/gpu.py +0 -0
  59. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/status.py +0 -0
  60. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode/telemetry/__init__.py +0 -0
  61. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode.egg-info/dependency_links.txt +0 -0
  62. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode.egg-info/entry_points.txt +0 -0
  63. {flashnode-0.3.3 → flashnode-0.3.5}/flashnode.egg-info/top_level.txt +0 -0
  64. {flashnode-0.3.3 → flashnode-0.3.5}/setup.cfg +0 -0
  65. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_allowlist_drift.py +0 -0
  66. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_archives.py +0 -0
  67. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_checkpoint_relay.py +0 -0
  68. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_credentials.py +0 -0
  69. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_enrol.py +0 -0
  70. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_evidence.py +0 -0
  71. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_gpu_probe.py +0 -0
  72. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_hardening.py +0 -0
  73. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_hardening_platform.py +0 -0
  74. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_identity.py +0 -0
  75. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_images_allowlist.py +0 -0
  76. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_interfaces.py +0 -0
  77. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_local_data.py +0 -0
  78. {flashnode-0.3.3 → flashnode-0.3.5}/tests/test_status.py +0 -0
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flashnode
3
- Version: 0.3.3
3
+ Version: 0.3.5
4
4
  Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
7
7
  Requires-Python: >=3.10
8
8
  Description-Content-Type: text/markdown
9
9
  License-File: LICENSE
10
- Requires-Dist: flashruntime<0.5,>=0.4.2
10
+ Requires-Dist: flashruntime<0.6,>=0.4.3
11
11
  Requires-Dist: psutil>=5.9
12
12
  Requires-Dist: websockets>=12
13
13
  Requires-Dist: cryptography>=42
@@ -0,0 +1,653 @@
1
+ """Command-line entry point for the FlashNode agent.
2
+
3
+ Target surface (see docs/SYSTEM_OVERVIEW.md §10):
4
+
5
+ flashnode join --code <one-time-code>
6
+ flashnode status
7
+ flashnode leave
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import sys
13
+
14
+ from flashnode import __version__
15
+
16
+ USAGE = """\
17
+ flashnode {version} — FlashML open host agent (pre-release scaffold)
18
+
19
+ usage: flashnode <command>
20
+
21
+ commands:
22
+ agent run the node agent loop (register with FlashML Cloud + heartbeat)
23
+ work register with a FlashRuntime coordinator and execute leased tasks
24
+ (--coordinator URL | FLASHNODE_COORDINATOR_URL; --max-tasks N)
25
+ login enrol this machine — prints a code to approve in a browser
26
+ (--coordinator URL; --token TOKEN to skip the browser step)
27
+ logout remove the saved bearer token for a FlashRuntime coordinator
28
+ (--coordinator URL)
29
+ doctor check this machine can run tasks (docker engine, images, mounts)
30
+ env see and reclaim disk held by cached dependency environments
31
+ (list | purge — purge never touches node-id or credentials.json)
32
+ join connect this machine to a FlashML control plane (not yet implemented)
33
+ status show node identity, capabilities, and active leases (not yet implemented)
34
+ leave drain and disconnect (not yet implemented)
35
+ """
36
+
37
+
38
+ def _login(args: list[str]) -> int:
39
+ """Enrol this machine.
40
+
41
+ The default path is device-code: print a short code, wait for a
42
+ signed-in human to approve it in a browser, save the token that comes
43
+ back. That is what the console tells volunteers to run, and until now it
44
+ could not work — `--token` was REQUIRED, and the enrolment flow issues
45
+ no token for anyone to paste. The API half (/v1alpha1/device/code,
46
+ /v1alpha1/device/token) had been built and simply had no client.
47
+
48
+ `--token` stays supported for a credential you already hold: CI, a
49
+ self-hosted coordinator, or re-pointing a machine with no browser to
50
+ hand.
51
+ """
52
+ import argparse
53
+
54
+ from flashnode.identity.credentials import save_token
55
+
56
+ parser = argparse.ArgumentParser(
57
+ prog="flashnode login",
58
+ description="Enrol this machine with FlashML.",
59
+ )
60
+ parser.add_argument(
61
+ "--coordinator",
62
+ required=True,
63
+ help="FlashML Cloud API base URL (e.g. https://flashml-api.onrender.com)",
64
+ )
65
+ parser.add_argument(
66
+ "--token",
67
+ help="skip the browser step and save a token you already have",
68
+ )
69
+ opts = parser.parse_args(args)
70
+
71
+ if opts.token:
72
+ path = save_token(opts.coordinator, opts.token)
73
+ print(f"flashnode login: credential saved to {path}", file=sys.stderr)
74
+ return 0
75
+
76
+ from flashnode.identity.enrol import (
77
+ EnrolmentError,
78
+ describe_this_machine,
79
+ poll_for_token,
80
+ request_device_code,
81
+ )
82
+ from flashnode.identity.store import load_or_create_node_id
83
+
84
+ try:
85
+ node_id = load_or_create_node_id()
86
+ except OSError as exc:
87
+ print(
88
+ f"flashnode login: cannot write this machine's identity: {exc}\n"
89
+ "Set FLASHNODE_STATE_DIR to a directory you can write to.",
90
+ file=sys.stderr,
91
+ )
92
+ return 1
93
+
94
+ hostname, platform_name = describe_this_machine()
95
+
96
+ try:
97
+ start = request_device_code(
98
+ opts.coordinator, node_id, hostname, platform_name
99
+ )
100
+ except EnrolmentError as exc:
101
+ print(f"flashnode login: {exc}", file=sys.stderr)
102
+ return 1
103
+
104
+ # stdout, not stderr: this is the output the person is here for, and it
105
+ # should survive being piped.
106
+ #
107
+ # flush=True is load-bearing. Python block-buffers stdout when it is not
108
+ # a terminal, so piping `flashnode login` anywhere — tee, a log, a setup
109
+ # script — showed nothing at all while the process sat waiting for an
110
+ # approval of a code it had never displayed.
111
+ print(flush=True)
112
+ print(f" Your code: {start.user_code}", flush=True)
113
+ print(f" Approve at: {start.verification_uri}", flush=True)
114
+ print(flush=True)
115
+ print(
116
+ "Open that on any device you're signed in on — your phone is fine.",
117
+ flush=True,
118
+ )
119
+ print("Waiting for approval… (Ctrl-C to cancel)", flush=True)
120
+
121
+ try:
122
+ token = poll_for_token(
123
+ opts.coordinator, start.device_code, interval=start.interval
124
+ )
125
+ except EnrolmentError as exc:
126
+ print(f"\nflashnode login: {exc}", file=sys.stderr)
127
+ return 1
128
+ except KeyboardInterrupt:
129
+ # Cancelling is a normal act, not a crash. The code expires unused.
130
+ print("\nflashnode login: cancelled.", file=sys.stderr)
131
+ return 130
132
+
133
+ path = save_token(opts.coordinator, token)
134
+ print(f"\nApproved. This machine is enrolled — credential saved to {path}.")
135
+ # The coordinator has to be IN the printed command. A volunteer pastes
136
+ # this line verbatim; without it `work` defaults to localhost:8100,
137
+ # looks up a credential that was saved under a different key, and dies
138
+ # on a bare 401 running the exact command the CLI just gave them.
139
+ # Lazy import, matching this file's existing style (see the imports a few
140
+ # lines up): health.py pulls CheckResult/default_workdir/run_checks from
141
+ # flashnode.doctor at module scope, and this keeps that import graph from
142
+ # having to resolve until a login actually happens.
143
+ from flashnode.doctor import check_cli_on_path
144
+ from flashnode.executor.health import suggested_runner
145
+
146
+ # `shutil.which`, not the full docker set: suggested_runner reads only
147
+ # the CLI-on-PATH verdict, and probing the daemon here would make
148
+ # `flashnode login` hang for up to 30s against a wedged docker install
149
+ # to answer a question a PATH lookup already settles.
150
+ hint = suggested_runner([check_cli_on_path()])
151
+ print(
152
+ "Start contributing with: "
153
+ f"flashnode work --runner {hint} --coordinator {opts.coordinator}"
154
+ )
155
+ return 0
156
+
157
+
158
+ #: Tiers that shell out to the `docker` binary. Everything else runs the
159
+ #: workload on the host directly and has no container to diagnose.
160
+ _CONTAINER_TIERS = ("docker", "argv")
161
+
162
+
163
+ def _next_step(runner: str, *, restarting: bool = False, docker_absent: bool = False) -> str:
164
+ """What to tell an operator whose host just refused to work, in terms
165
+ their tier can actually act on.
166
+
167
+ `flashnode doctor` runs `run_checks(pull=True)` — the DOCKER set. Telling
168
+ a host that has no Docker, which is the entire reason it is on a
169
+ no-container tier, to run it hands back `[FAIL] docker CLI on PATH` and a
170
+ non-zero exit: the same misleading verdict this tier exists to remove,
171
+ reached one hop later. That is what a real pod was told on 2026-08-09.
172
+ The "image pull this skipped" clause is meaningless there too — the
173
+ no-container tiers never pull anything.
174
+
175
+ `docker_absent=True` is the narrower case: `runner` IS a container tier
176
+ (docker/argv), but the `docker` CLI itself was not found — the same
177
+ 2026-08-09 shape one tier over. `flashnode doctor` is still the right
178
+ command here (this host really does need Docker), but "including the
179
+ image pull this skipped" is false on it: there was never going to be a
180
+ pull, because the CLI check that gates one already failed. Set by the
181
+ caller from the same startup check set this trailer follows, never
182
+ guessed here.
183
+
184
+ `restarting=True` is the quarantine path, where the host had already
185
+ started and the operator is being told what to do before starting again.
186
+ """
187
+ if runner in _CONTAINER_TIERS:
188
+ if restarting:
189
+ return "Fix the above, then `flashnode doctor` to confirm before restarting."
190
+ if docker_absent:
191
+ return ("Run `flashnode doctor` for the full check. There is no "
192
+ "`docker` here at all, so there is no image pull to skip.")
193
+ return ("Run `flashnode doctor` for the full check, including the "
194
+ "image pull this skipped.")
195
+ verb = "start" if restarting else "run"
196
+ return (
197
+ f"Fix the above, then {verb} `flashnode work --runner {runner}` again.\n"
198
+ f"Do not run `flashnode doctor` here: it checks the Docker tier, which\n"
199
+ f"--runner {runner} never uses, so on this machine it fails by design."
200
+ )
201
+
202
+
203
+ def _logout(args: list[str]) -> int:
204
+ import argparse
205
+
206
+ from flashnode.identity.credentials import clear_token, credentials_path
207
+
208
+ parser = argparse.ArgumentParser(prog="flashnode logout")
209
+ parser.add_argument("--coordinator", required=True, help="FlashRuntime coordinator base URL")
210
+ opts = parser.parse_args(args)
211
+
212
+ removed = clear_token(opts.coordinator)
213
+ if removed:
214
+ print(f"flashnode logout: credential removed from {credentials_path()}", file=sys.stderr)
215
+ else:
216
+ print(f"flashnode logout: no saved credential for {opts.coordinator}", file=sys.stderr)
217
+ return 0
218
+
219
+
220
+ def _work(args: list[str]) -> int:
221
+ import argparse
222
+ import logging
223
+ import os
224
+ import shutil
225
+ import signal
226
+
227
+ from flashnode.executor import CoordinatorClient, ExecutorLoop
228
+ from flashnode.identity.store import load_or_create_node_id
229
+ from flashnode.inventory.capabilities import discover
230
+
231
+ logging.basicConfig(
232
+ level=logging.INFO,
233
+ format='{"ts":"%(asctime)s","level":"%(levelname)s","service":"flashnode","msg":%(message)s}',
234
+ )
235
+ parser = argparse.ArgumentParser(prog="flashnode work")
236
+ parser.add_argument(
237
+ "--coordinator",
238
+ # No default here: an omitted --coordinator is resolved below from
239
+ # the credentials this machine actually holds. Baking localhost in
240
+ # made every enrolled volunteer's first command 401.
241
+ default=os.environ.get("FLASHNODE_COORDINATOR_URL"),
242
+ help="FlashRuntime coordinator base URL (default: the one this "
243
+ "machine enrolled against with `flashnode login`)",
244
+ )
245
+ parser.add_argument(
246
+ "--runner",
247
+ choices=["subprocess", "docker", "argv", "trusted"],
248
+ default=os.environ.get("FLASHNODE_RUNNER", "subprocess"),
249
+ help="task execution tier (docker/argv need the docker CLI on PATH; "
250
+ "trusted runs pool jobs unsandboxed — opt-in only — and is the "
251
+ "only tier leased jobs that declare dependencies)",
252
+ )
253
+ parser.add_argument("--max-tasks", type=int, default=None)
254
+ parser.add_argument("--poll-seconds", type=float, default=1.0)
255
+ parser.add_argument(
256
+ "--log-json", action="store_true",
257
+ help="keep the machine-readable JSON log instead of the live status view",
258
+ )
259
+ parser.add_argument(
260
+ "--max-consecutive-failures", type=int,
261
+ default=int(os.environ.get("FLASHNODE_MAX_CONSECUTIVE_FAILURES", "3")),
262
+ help="host-side failures in a row before re-checking this machine and "
263
+ "stopping if it is broken (0 disables)",
264
+ )
265
+ opts = parser.parse_args(args)
266
+
267
+ # Resolved BEFORE the runner branch so the one health binding below can
268
+ # hand it to whichever tier's check set applies — the directory a check
269
+ # asks about must be the directory the loop will actually stage tasks in.
270
+ workdir_base = os.environ.get("FLASHNODE_WORKDIR") or None
271
+
272
+ # Every branch below assigns a runner — there is no `runner = None`
273
+ # fallthrough any more, because the health binding after them has to have
274
+ # something to ask.
275
+ if opts.runner in ("docker", "argv"):
276
+ # OPTIONAL, and additive. The built-in namespace allowlist
277
+ # (executor/images.py DEFAULT_ALLOWED_IMAGE_PREFIXES) is what a
278
+ # volunteer runs on, so this env var is no longer required and an
279
+ # empty value is no longer a refusal to start.
280
+ #
281
+ # It used to be mandatory, which quietly capped the project at the
282
+ # number of machines whose owners would hand-maintain a list of image
283
+ # references: every image we published stranded every host until its
284
+ # owner edited the variable, so security fixes would reach a fraction
285
+ # of the fleet. Setting it now means "also allow these", for
286
+ # self-hosting the stack or for integration tests.
287
+ images = frozenset(
288
+ i.strip() for i in os.environ.get("FLASHNODE_ALLOWED_IMAGES", "").split(",") if i.strip()
289
+ )
290
+ if opts.runner == "docker":
291
+ from flashnode.executor.docker_runner import DockerRunner
292
+
293
+ runner = DockerRunner(allowed_images=images)
294
+ else:
295
+ from flashnode.executor.argv_runner import ArgvDockerRunner
296
+
297
+ runner = ArgvDockerRunner(
298
+ allowed_images=images,
299
+ cpus=float(os.environ.get("FLASHNODE_MAX_CPUS", "2.0")),
300
+ memory_gb=float(os.environ.get("FLASHNODE_MAX_MEMORY_GB", "2.0")),
301
+ timeout_seconds=float(os.environ.get("FLASHNODE_TASK_TIMEOUT_S", "3600")),
302
+ max_output_bytes=int(
303
+ os.environ.get("FLASHNODE_MAX_OUTPUT_BYTES", str(2 * 1024**3))
304
+ ),
305
+ )
306
+ elif opts.runner == "trusted":
307
+ # This tier exists for hosts that ARE a container already — a RunPod
308
+ # pod, a Colab VM — and so can never nest a Docker daemon. The docker
309
+ # checks are not merely unnecessary here, they are unsatisfiable.
310
+ from flashnode.executor.trusted_runner import TrustedArgvRunner
311
+
312
+ print(
313
+ "trusted runner: pool jobs from your team run UNSANDBOXED on this\n"
314
+ "machine — no container, no network isolation. Only continue if\n"
315
+ "every member of your pool is someone you trust to run code here."
316
+ )
317
+ runner = TrustedArgvRunner()
318
+ else:
319
+ # Constructed HERE rather than left to ExecutorLoop's own default, so
320
+ # that every tier reaches the health binding below with a runner to
321
+ # ask. The instance is what the loop would have built for itself.
322
+ from flashnode.executor.runner import SubprocessRunner
323
+
324
+ runner = SubprocessRunner()
325
+
326
+ # The tier is resolved ONCE, here, by asking the runner we just built.
327
+ # Two call sites each deciding their own tier is precisely the bug this
328
+ # replaced: the startup gate exempted `trusted` and the loop's
329
+ # post-failure re-check did not, so three failures of any cause
330
+ # quarantined a pod and blamed Docker for it (2026-08-09).
331
+ #
332
+ # What the docker tier asks is unchanged, and deliberately so. Both
333
+ # sandboxed tiers shell out to the `docker` binary directly
334
+ # (subprocess.run(["docker", ...])); if it isn't installed that call
335
+ # raises FileNotFoundError deep inside a task attempt. And a `docker`
336
+ # binary on PATH says nothing about the daemon behind it, the credential
337
+ # helper Docker consults when it pulls, or whether this machine's work
338
+ # directory is even visible inside the VM. Both hosts that stopped the
339
+ # 2026-08-02 §10 run-through passed the old `shutil.which` check and then
340
+ # failed every task they claimed — docker_runner raises
341
+ # TaskExecutionError, loop.py calls fail() and claims the next one,
342
+ # forever, silently. `docker_tier_checks` still runs with pull=False: an
343
+ # agent is a long-running daemon on someone else's machine, and a
344
+ # transient registry blip must not stop one whose images are already
345
+ # cached. `flashnode doctor` does the pull.
346
+ from flashnode.doctor import NON_BLOCKING_STATUSES, format_results
347
+
348
+ def _blocking_problems() -> list:
349
+ """What the loop calls after a streak of host-side failures.
350
+
351
+ Bound to the same runner as the startup gate below, so the two
352
+ cannot disagree about which tier this host is in — and so `loop.py`
353
+ never has to import the doctor (which would close a
354
+ loop -> doctor -> executor -> loop cycle).
355
+
356
+ Filtered HERE with doctor's own NON_BLOCKING_STATUSES rather than a
357
+ `!= "ok"` test: the GPU check reports "info" and never fails,
358
+ because most volunteers have no GPU and blocking on it would lock
359
+ the entire existing fleet out of the CPU work they signed up for.
360
+ Keeping that predicate in one place is what stops `flashnode work`
361
+ and `flashnode doctor` disagreeing about which verdicts are fatal.
362
+ The membership test is the way round that makes an unknown status
363
+ BLOCK, matching doctor.exit_code.
364
+ """
365
+ return [r for r in runner.health_checks(workdir=workdir_base)
366
+ if r.status not in NON_BLOCKING_STATUSES]
367
+
368
+ startup_problems = _blocking_problems()
369
+ if startup_problems:
370
+ # Read off the SAME check set the trailer follows, never guessed:
371
+ # a host whose `docker CLI on PATH` check failed has no image pull
372
+ # to have skipped, and the trailer below must not claim it did.
373
+ docker_absent = any(
374
+ r.name == "docker CLI on PATH" and r.status == "fail"
375
+ for r in startup_problems
376
+ )
377
+ # next_step="" because the trailer below IS the next step, and it is
378
+ # chosen by tier. Leaving format_results' default in place printed
379
+ # "re-run `flashnode doctor`" to a host the doctor cannot certify.
380
+ print(
381
+ f"flashnode work: this machine cannot run tasks with "
382
+ f"--runner {opts.runner}.\n"
383
+ + format_results(startup_problems, next_step="")
384
+ + "\n\n" + _next_step(opts.runner, docker_absent=docker_absent),
385
+ file=sys.stderr,
386
+ )
387
+ return 2
388
+
389
+ from flashnode.identity.credentials import load_token, resolve_coordinator
390
+
391
+ try:
392
+ opts.coordinator = resolve_coordinator(opts.coordinator)
393
+ except ValueError as exc:
394
+ print(f"flashnode work: {exc}", file=sys.stderr)
395
+ return 1
396
+
397
+ node_id = load_or_create_node_id()
398
+ client = CoordinatorClient(
399
+ opts.coordinator,
400
+ join_code=os.environ.get("FLASHNODE_JOIN_CODE") or None,
401
+ token=load_token(opts.coordinator),
402
+ )
403
+ registration = discover(
404
+ node_id, kubernetes_node="", node_meta=None,
405
+ argv_capable=(opts.runner == "argv"),
406
+ # An argv-only volunteer has no module runner behind it: advertise
407
+ # module_capable=False so the coordinator's placement gate stops
408
+ # routing "python -m <module>" tasks here (F1) — otherwise those
409
+ # tasks burn every attempt against ArgvDockerRunner's payload
410
+ # rejection before the job ever fails for real.
411
+ # `trusted` runs argv through TrustedArgvRunner, which refuses
412
+ # `python -m <module>` payloads exactly as the argv tier does. Both
413
+ # must advertise module_capable=False, or this machine is eligible
414
+ # for public MODULE tasks it fails every attempt of — burning the
415
+ # volunteer's electricity and the job's retries on a mismatch the
416
+ # registration already knew about.
417
+ module_capable=(opts.runner not in ("argv", "trusted")),
418
+ # Set true ONLY by the explicit --runner trusted opt-in, never
419
+ # inferred — the scheduler's placement gate treats this field as
420
+ # fail-closed (flashruntime.scheduler), so a node that merely has a
421
+ # shell available must not advertise it.
422
+ unsandboxed_argv_capable=(opts.runner == "trusted"),
423
+ # `trusted` ONLY, and deliberately narrower than "the tiers that
424
+ # could technically run pip".
425
+ #
426
+ # The feature exists for hosts that cannot run a container — a
427
+ # RunPod pod, a Colab notebook — and must execute the submitter's
428
+ # own argv; that is the trusted tier, and only there does the job's
429
+ # venv hold everything the job runs.
430
+ #
431
+ # `subprocess` is excluded even though it also runs on the host: it
432
+ # executes the AGENT's allowlisted `flashml_workloads.*` modules,
433
+ # which live in the agent's environment, not the job's. Sending it
434
+ # into a job venv built from the job's declared dependencies alone
435
+ # produces ModuleNotFoundError — the exact failure this feature
436
+ # exists to eliminate. Giving that venv access to the agent's own
437
+ # environment is not the fix either: a declared dependency would
438
+ # then shadow the agent's.
439
+ #
440
+ # The container tiers cannot install at all — --network none and a
441
+ # read-only rootfs. Physics, not policy.
442
+ #
443
+ # Fail closed and narrow. Widening this to another tier is a
444
+ # deliberate change that must arrive with its own tests.
445
+ can_install_dependencies=(opts.runner == "trusted"),
446
+ )
447
+ client.register(registration)
448
+ loop = ExecutorLoop(
449
+ client, node_id, runner=runner,
450
+ poll_seconds=opts.poll_seconds, workdir_base=workdir_base,
451
+ registration=registration, # survives coordinator restarts
452
+ health_check=_blocking_problems,
453
+ max_consecutive_failures=opts.max_consecutive_failures,
454
+ )
455
+
456
+ def _stop(signum, frame): # noqa: ARG001
457
+ loop.stop_event.set()
458
+
459
+ signal.signal(signal.SIGTERM, _stop)
460
+ signal.signal(signal.SIGINT, _stop)
461
+ view = None
462
+ if sys.stdout.isatty() and not opts.log_json:
463
+ # Two writers redrawing one terminal is unreadable, so the JSON
464
+ # handler goes when the view arrives. --log-json keeps it, and a
465
+ # non-TTY never gets here — ANSI cursor movement into a pipe or a
466
+ # systemd journal is corruption, not output.
467
+ from flashnode.status import StatusView
468
+
469
+ logging.getLogger().handlers.clear()
470
+ view = StatusView(loop, coordinator=opts.coordinator,
471
+ version=__version__, stream=sys.stdout)
472
+ view.start()
473
+ try:
474
+ accepted = loop.run(max_tasks=opts.max_tasks)
475
+ finally:
476
+ if view is not None:
477
+ view.stop()
478
+
479
+ if getattr(loop, "quarantined", False):
480
+ from flashnode.doctor import format_results
481
+
482
+ print(
483
+ "\nflashnode work: stopping — this machine can no longer run "
484
+ "tasks.\n"
485
+ + format_results(loop.health_report or [], next_step="")
486
+ + "\n\n" + _next_step(opts.runner, restarting=True),
487
+ file=sys.stderr,
488
+ )
489
+ return 2
490
+ print(f"flashnode work: {accepted} task(s) accepted", file=sys.stderr)
491
+ return 0
492
+
493
+
494
+ def _human_bytes(n: int) -> str:
495
+ """`n` bytes, rendered the way a volunteer reads disk usage — whole B,
496
+ one decimal place from KB up."""
497
+ size = float(n)
498
+ for unit in ("B", "KB", "MB", "GB"):
499
+ if size < 1024 or unit == "GB":
500
+ return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
501
+ size /= 1024
502
+ return f"{size:.1f} GB" # pragma: no cover - unreachable, GB always returns above
503
+
504
+
505
+ def _env(args: list[str]) -> int:
506
+ """`flashnode env list|purge` — the disk cached dependency environments
507
+ hold, and a way to give it back without `rm -rf ~/.flashnode`.
508
+
509
+ `purge` deletes only what `EnvironmentCache.entries()` reports — the
510
+ contents of `$FLASHNODE_STATE_DIR/envs/`. It never touches the state
511
+ directory itself, so `node-id` and `credentials.json`, its siblings, are
512
+ untouched: that is the entire difference between this command and
513
+ un-enrolling the machine.
514
+ """
515
+ import argparse
516
+ import shutil
517
+ from datetime import datetime
518
+
519
+ from flashnode.executor.environments import EnvironmentCache
520
+
521
+ parser = argparse.ArgumentParser(prog="flashnode env")
522
+ sub = parser.add_subparsers(dest="subcommand", required=True)
523
+ sub.add_parser("list", help="show cached environments, their size, and last use")
524
+ sub.add_parser(
525
+ "purge",
526
+ help="delete all cached environments (identity and credentials "
527
+ "survive; stop `flashnode work` first — this can break a task "
528
+ "in flight)",
529
+ )
530
+ opts = parser.parse_args(args)
531
+
532
+ cache = EnvironmentCache()
533
+ entries = cache.entries() # most recently used first
534
+
535
+ # Scratch directories from builds that were KILLED — SIGKILL, the OOM
536
+ # killer, a power cut. `entries()` skips dotted names, so without this
537
+ # the two commands whose whole job is to account for disk were the two
538
+ # that could not see a multi-GB directory.
539
+ stale_bytes = cache.stale_build_bytes()
540
+
541
+ if opts.subcommand == "list":
542
+ if not entries and not stale_bytes:
543
+ print("flashnode env: no environments cached — nothing is using disk yet.")
544
+ return 0
545
+ for entry in entries:
546
+ when = datetime.fromtimestamp(entry.last_used).strftime("%Y-%m-%d %H:%M")
547
+ print(f" {entry.key} {_human_bytes(entry.size_bytes):>8} last used {when}")
548
+ total = sum(e.size_bytes for e in entries) + stale_bytes
549
+ if stale_bytes:
550
+ print(
551
+ f" leftover build directories {_human_bytes(stale_bytes):>8} "
552
+ "from interrupted builds"
553
+ )
554
+ print(
555
+ f"total: {_human_bytes(total)} across {len(entries)} environment(s) "
556
+ f"at {cache.root}"
557
+ )
558
+ if stale_bytes:
559
+ print(
560
+ " the leftover build directories are swept by the next build, "
561
+ "or now by `flashnode env purge`"
562
+ )
563
+ return 0
564
+
565
+ # purge
566
+ #
567
+ # This runs in a SEPARATE process from the agent. The in-process
568
+ # eviction path is safe only because ExecutorLoop holds one lease at a
569
+ # time and `_evict` protects the entry it just handed out; purge
570
+ # sidesteps both and can delete the venv a running task's interpreter
571
+ # lives in. Refusing would beat warning, but nothing in the state
572
+ # directory records a running agent — no pidfile, no lock — and a
573
+ # guessed signal (a stale pid, an mtime heuristic) would refuse on a
574
+ # machine with no agent at all. So: say it, plainly, every time, and
575
+ # before the deletion rather than after it.
576
+ print(
577
+ "flashnode env purge: warning — if `flashnode work` is running on "
578
+ "this machine, this can delete the environment a task is running in "
579
+ "and fail that task. This command cannot tell whether an agent is "
580
+ "running. Stop the agent first if you are not sure."
581
+ )
582
+ # Counted AFTER each removal, never before. The old version totalled the
583
+ # entries up front and deleted with `ignore_errors=True`, so a
584
+ # permission error printed the full figure as reclaimed — on the one
585
+ # command whose entire value is that number, and which a volunteer has
586
+ # no other way to check.
587
+ freed = 0
588
+ removed = 0
589
+ survivors = []
590
+ for entry in entries:
591
+ shutil.rmtree(entry.path, ignore_errors=True)
592
+ if entry.path.exists():
593
+ survivors.append(entry.path)
594
+ else:
595
+ freed += entry.size_bytes
596
+ removed += 1
597
+ stale_freed = cache.sweep_stale_builds() # counts only what actually went
598
+ freed += stale_freed
599
+ survivors.extend(cache.stale_build_dirs())
600
+ leftovers = (
601
+ f" (including {_human_bytes(stale_freed)} of leftover build "
602
+ "directories from interrupted builds)"
603
+ if stale_freed
604
+ else ""
605
+ )
606
+ print(
607
+ f"flashnode env purge: freed {_human_bytes(freed)} across "
608
+ f"{removed} environment(s){leftovers}. node-id and credentials.json "
609
+ "were not touched."
610
+ )
611
+ if survivors:
612
+ noun = "directory" if len(survivors) == 1 else "directories"
613
+ print(
614
+ f"flashnode env purge: could not remove {len(survivors)} {noun} "
615
+ "— the disk it holds was NOT reclaimed:"
616
+ )
617
+ for path in survivors:
618
+ print(f" {path}")
619
+ print(
620
+ " check the permissions on those paths, or whether another "
621
+ "process is holding them open, and run purge again."
622
+ )
623
+ return 1
624
+ return 0
625
+
626
+
627
+ def main(argv: list[str] | None = None) -> int:
628
+ args = sys.argv[1:] if argv is None else argv
629
+ if args and args[0] == "agent":
630
+ from flashnode.agent.daemon import main as agent_main
631
+
632
+ return agent_main()
633
+ if args and args[0] == "work":
634
+ return _work(args[1:])
635
+ if args and args[0] == "doctor":
636
+ from flashnode.doctor import doctor_main
637
+
638
+ return doctor_main(args[1:])
639
+ if args and args[0] == "login":
640
+ return _login(args[1:])
641
+ if args and args[0] == "logout":
642
+ return _logout(args[1:])
643
+ if args and args[0] == "env":
644
+ return _env(args[1:])
645
+ print(USAGE.format(version=__version__), end="")
646
+ if args and args[0] in {"join", "status", "leave"}:
647
+ print(f"\nerror: '{args[0]}' is not implemented yet in this scaffold.", file=sys.stderr)
648
+ return 1
649
+ return 0
650
+
651
+
652
+ if __name__ == "__main__":
653
+ raise SystemExit(main())