flashnode 0.3.2__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. {flashnode-0.3.2 → flashnode-0.3.3}/PKG-INFO +2 -2
  2. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/argv_runner.py +10 -0
  3. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/client.py +24 -5
  4. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/docker_runner.py +12 -0
  5. flashnode-0.3.3/flashnode/executor/evidence.py +258 -0
  6. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/loop.py +35 -2
  7. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/runner.py +17 -0
  8. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/inventory/gpu.py +68 -0
  9. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/PKG-INFO +2 -2
  10. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/SOURCES.txt +2 -0
  11. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/requires.txt +1 -1
  12. {flashnode-0.3.2 → flashnode-0.3.3}/pyproject.toml +13 -2
  13. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_argv_runner.py +7 -3
  14. flashnode-0.3.3/tests/test_evidence.py +564 -0
  15. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_gpu_probe.py +100 -0
  16. {flashnode-0.3.2 → flashnode-0.3.3}/LICENSE +0 -0
  17. {flashnode-0.3.2 → flashnode-0.3.3}/README.md +0 -0
  18. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/__init__.py +0 -0
  19. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/agent/__init__.py +0 -0
  20. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/agent/cli.py +0 -0
  21. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/agent/daemon.py +0 -0
  22. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/agent/kube.py +0 -0
  23. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/artifacts/__init__.py +0 -0
  24. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/benchmark/__init__.py +0 -0
  25. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/config/__init__.py +0 -0
  26. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/config/local_data.py +0 -0
  27. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/doctor.py +0 -0
  28. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/__init__.py +0 -0
  29. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/archives.py +0 -0
  30. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/hardening.py +0 -0
  31. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/images.py +0 -0
  32. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/identity/__init__.py +0 -0
  33. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/identity/credentials.py +0 -0
  34. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/identity/enrol.py +0 -0
  35. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/identity/store.py +0 -0
  36. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/inventory/__init__.py +0 -0
  37. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/inventory/capabilities.py +0 -0
  38. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/status.py +0 -0
  39. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/telemetry/__init__.py +0 -0
  40. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/dependency_links.txt +0 -0
  41. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/entry_points.txt +0 -0
  42. {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/top_level.txt +0 -0
  43. {flashnode-0.3.2 → flashnode-0.3.3}/setup.cfg +0 -0
  44. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_agent.py +0 -0
  45. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_allowlist_drift.py +0 -0
  46. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_archives.py +0 -0
  47. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_capabilities.py +0 -0
  48. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_checkpoint_relay.py +0 -0
  49. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_credentials.py +0 -0
  50. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_docker_runner.py +0 -0
  51. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_doctor.py +0 -0
  52. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_enrol.py +0 -0
  53. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_executor.py +0 -0
  54. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_hardening.py +0 -0
  55. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_hardening_platform.py +0 -0
  56. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_identity.py +0 -0
  57. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_images_allowlist.py +0 -0
  58. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_interfaces.py +0 -0
  59. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_local_data.py +0 -0
  60. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_loop_counters.py +0 -0
  61. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_status.py +0 -0
  62. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_work_gate.py +0 -0
  63. {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_work_status.py +0 -0
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flashnode
3
- Version: 0.3.2
3
+ Version: 0.3.3
4
4
  Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
7
7
  Requires-Python: >=3.10
8
8
  Description-Content-Type: text/markdown
9
9
  License-File: LICENSE
10
- Requires-Dist: flashruntime<0.5,>=0.4.1
10
+ Requires-Dist: flashruntime<0.5,>=0.4.2
11
11
  Requires-Dist: psutil>=5.9
12
12
  Requires-Dist: websockets>=12
13
13
  Requires-Dist: cryptography>=42
@@ -20,6 +20,7 @@ import re
20
20
  import subprocess
21
21
  from pathlib import Path
22
22
 
23
+ from flashnode.executor.evidence import image_digest
23
24
  from flashnode.executor.hardening import container_name, harden_args
24
25
  from flashnode.executor.images import DEFAULT_ALLOWED_IMAGE_PREFIXES, image_is_allowed
25
26
  from flashnode.executor.runner import TaskExecutionError
@@ -43,8 +44,14 @@ class ArgvDockerRunner:
43
44
  self.memory_gb = memory_gb
44
45
  self.timeout_seconds = timeout_seconds
45
46
  self.max_output_bytes = max_output_bytes
47
+ # What the LAST run measured, read by ExecutorLoop for
48
+ # ExecutionEvidence — see SubprocessRunner for why both reset below.
49
+ self.last_exit_code: int | None = None
50
+ self.last_image_digest: str = ""
46
51
 
47
52
  def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
53
+ self.last_exit_code = None
54
+ self.last_image_digest = ""
48
55
  argv = payload.get("argv")
49
56
  if not argv or not isinstance(argv, list) or not all(isinstance(t, str) for t in argv):
50
57
  raise TaskExecutionError("payload 'argv' must be a non-empty list of strings")
@@ -105,9 +112,12 @@ class ArgvDockerRunner:
105
112
  # (a subclass of OSError). Degrade to a failed task, not a dead
106
113
  # agent — execute_one only catches TaskExecutionError/LeaseLost.
107
114
  raise TaskExecutionError(f"docker is unavailable: {exc}") from exc
115
+ self.last_exit_code = proc.returncode
108
116
  if proc.returncode != 0:
109
117
  tail = proc.stderr.decode(errors="replace")[-800:]
110
118
  raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
119
+ # Asked of the daemon, not read off the payload — see DockerRunner.
120
+ self.last_image_digest = image_digest(image)
111
121
 
112
122
  # metrics.json is load-bearing, not a preference: CommandRecipe sets
113
123
  # commit_key to <prefix>/metrics.json and the coordinator validates
@@ -19,7 +19,12 @@ import urllib.error
19
19
  import urllib.request
20
20
  from pathlib import Path
21
21
 
22
- from flashruntime.protocol.v1alpha1 import Lease, NodeHeartbeat, NodeRegistration
22
+ from flashruntime.protocol.v1alpha1 import (
23
+ ExecutionEvidence,
24
+ Lease,
25
+ NodeHeartbeat,
26
+ NodeRegistration,
27
+ )
23
28
 
24
29
 
25
30
  class LeaseLost(Exception):
@@ -122,11 +127,25 @@ class CoordinatorClient:
122
127
  if status != 200:
123
128
  raise RuntimeError(f"attempt heartbeat failed ({status}): {body}")
124
129
 
125
- def complete(self, lease_id: str, output_sha256: str) -> bool:
130
+ def complete(
131
+ self,
132
+ lease_id: str,
133
+ output_sha256: str,
134
+ evidence: ExecutionEvidence | None = None,
135
+ ) -> bool:
136
+ """Commit this attempt, optionally saying what the run looked like.
137
+
138
+ `evidence` is a keyword with a default so every existing caller keeps
139
+ working, and the body is byte-for-byte the one previous releases sent
140
+ when there is nothing to report — this agent may be talking to a
141
+ coordinator that predates the field, and an empty block is not the
142
+ same statement as no block.
143
+ """
144
+ payload: dict = {"output_sha256": output_sha256}
145
+ if evidence is not None:
146
+ payload["evidence"] = evidence.model_dump(mode="json")
126
147
  status, body = self._json(
127
- "POST",
128
- f"/v1alpha1/attempts/{lease_id}/complete",
129
- {"output_sha256": output_sha256},
148
+ "POST", f"/v1alpha1/attempts/{lease_id}/complete", payload
130
149
  )
131
150
  if status != 200:
132
151
  raise RuntimeError(f"complete failed ({status}): {body}")
@@ -19,6 +19,7 @@ import json
19
19
  import subprocess
20
20
  from pathlib import Path
21
21
 
22
+ from flashnode.executor.evidence import image_digest
22
23
  from flashnode.executor.hardening import CONTAINER_WORKDIR, container_name, harden_args
23
24
  from flashnode.executor.images import DEFAULT_ALLOWED_IMAGE_PREFIXES, image_is_allowed
24
25
  from flashnode.executor.runner import DEFAULT_ALLOWED_MODULES, TaskExecutionError
@@ -44,8 +45,14 @@ class DockerRunner:
44
45
  self.cpus = cpus
45
46
  self.memory_gb = memory_gb
46
47
  self.timeout_seconds = timeout_seconds
48
+ # What the LAST run measured, read by ExecutorLoop for
49
+ # ExecutionEvidence — see SubprocessRunner for why both reset below.
50
+ self.last_exit_code: int | None = None
51
+ self.last_image_digest: str = ""
47
52
 
48
53
  def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
54
+ self.last_exit_code = None
55
+ self.last_image_digest = ""
49
56
  module = payload.get("module", "")
50
57
  if module not in self.allowed_modules:
51
58
  raise TaskExecutionError(f"module {module!r} is not allowlisted — refusing to run")
@@ -110,9 +117,14 @@ class DockerRunner:
110
117
  # (a subclass of OSError). Degrade to a failed task, not a dead
111
118
  # agent — execute_one only catches TaskExecutionError/LeaseLost.
112
119
  raise TaskExecutionError(f"docker is unavailable: {exc}") from exc
120
+ self.last_exit_code = proc.returncode
113
121
  if proc.returncode != 0:
114
122
  tail = proc.stderr.decode(errors="replace")[-800:]
115
123
  raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
124
+ # Asked of the daemon, not read off the payload: the point of the
125
+ # field is which bytes ran here, and the payload only says which
126
+ # bytes were asked for. "" when the daemon will not say.
127
+ self.last_image_digest = image_digest(image)
116
128
  if not (outdir / "metrics.json").is_file():
117
129
  raise TaskExecutionError("task produced no metrics.json — nothing to commit")
118
130
  return outdir
@@ -0,0 +1,258 @@
1
+ """What this agent can honestly say about a run it just finished.
2
+
3
+ Fills `flashruntime.protocol.v1alpha1.ExecutionEvidence`, the optional block
4
+ the executor attaches to `complete`. Three sources, all best-effort:
5
+
6
+ - **wall clock** — measured by the loop around the runner, always available.
7
+ - **CPU / GPU utilisation** — sampled on a background thread while the task
8
+ runs (`ResourceSampler`), averaged at the end.
9
+ - **image digest** — asked of the local docker daemon after the container
10
+ exits (`image_digest`).
11
+
12
+ THE ONE RULE. A value that could not be measured is reported as `None` (or
13
+ `""` for the digest), never as `0`. The temptation runs entirely one way:
14
+ `sum([]) / max(len([]), 1)` is 0.0, `psutil.cpu_percent()`'s first call in a
15
+ process is 0.0, an unparsed `docker` error string is truthy. Each of those is
16
+ a fabricated measurement that a verifier downstream cannot tell from a real
17
+ one — and "0% CPU on a completed task" is not a missing field, it is an
18
+ accusation. Every path below that could produce a zero by accident is written
19
+ so it produces absence instead, and each has a test.
20
+
21
+ NO NEW DEPENDENCIES. `psutil` is already an agent dependency (inventory/);
22
+ GPU utilisation goes through the same `nvidia-smi` subprocess as the
23
+ registration probe. This code runs on strangers' machines and every
24
+ dependency is attack surface (AGENTS.md).
25
+
26
+ Every probe takes its subprocess/sampler as a PARAMETER, resolved at call
27
+ time, so the whole test suite runs with no NVIDIA driver and no docker
28
+ daemon — the shape `inventory/gpu.py` and `doctor.py` established.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import math
34
+ import re
35
+ import subprocess
36
+ import threading
37
+ import time
38
+ from typing import Callable
39
+
40
+ from flashnode.inventory.gpu import probe_gpu_utilisation
41
+
42
+ __all__ = ["ResourceSampler", "default_cpu_probe", "image_digest"]
43
+
44
+ #: A probe answers "what is it now?" — a percentage, or None for "could not
45
+ #: read it". Never raises for the caller; the sampler defends anyway.
46
+ Probe = Callable[[], "float | None"]
47
+
48
+ CommandRunner = Callable[..., subprocess.CompletedProcess]
49
+
50
+ #: `docker image inspect` reads metadata the daemon already holds. Short on
51
+ #: purpose: this runs on the commit path, and a wedged daemon must cost the
52
+ #: field, not the task's result.
53
+ DIGEST_TIMEOUT_S = 10
54
+
55
+ #: What a content-addressable image identity looks like. Anything else the
56
+ #: daemon prints — a prose error, `<no value>` from a template miss, an empty
57
+ #: line — is NOT a digest and must not be forwarded as one.
58
+ _DIGEST_RE = re.compile(r"^[a-z0-9]+:[0-9a-f]{32,}$")
59
+
60
+
61
+ def _text(raw: object) -> str:
62
+ """Subprocess output as a string, or `""` for anything that is not one.
63
+
64
+ Total on purpose, and not quite the shape `inventory/gpu.py` uses: the
65
+ runner is a parameter, so `stdout` is whatever a caller's stub put there.
66
+ A duck-typed object with a `.decode` that returns another object would
67
+ otherwise walk a non-string all the way to the regex below and raise
68
+ TypeError from inside a best-effort probe.
69
+ """
70
+ if isinstance(raw, str):
71
+ return raw
72
+ if isinstance(raw, (bytes, bytearray)):
73
+ return bytes(raw).decode(errors="replace")
74
+ return ""
75
+
76
+
77
+ def _reading(value: object) -> float | None:
78
+ """One probe answer as a number the mean can survive, or None.
79
+
80
+ NaN is the reason this exists: a single NaN sample turns a whole run's
81
+ mean into NaN, which serialises to something no verifier can read. `bool`
82
+ is excluded explicitly — it is a subclass of `int`, so `True` would
83
+ otherwise average in as 1.0%.
84
+ """
85
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
86
+ return None
87
+ number = float(value)
88
+ return number if math.isfinite(number) else None
89
+
90
+
91
+ def _mean(values: list[float]) -> float | None:
92
+ """The average, or None for no samples at all.
93
+
94
+ NOT 0.0. This one line is the whole honesty contract of this module: an
95
+ empty list means nothing was measured, and a verifier reading 0.0 would
96
+ see a machine that did nothing on a task it completed.
97
+ """
98
+ if not values:
99
+ return None
100
+ return sum(values) / len(values)
101
+
102
+
103
+ def default_cpu_probe() -> Probe:
104
+ """A host-wide CPU probe, with psutil's meaningless first reading burned.
105
+
106
+ `psutil.cpu_percent(interval=None)` reports usage since the LAST call, so
107
+ its first answer in a process is 0.0 by construction — a fabricated zero
108
+ that looks exactly like an idle machine. That call happens here, at
109
+ construction, before the task starts; every reading the sampler takes
110
+ afterwards covers a real window.
111
+
112
+ Host-wide, not task-scoped: a volunteer's machine has other things
113
+ running on it. That makes a HIGH reading weak evidence and a LOW one the
114
+ interesting direction — a completed training task on a machine that never
115
+ got busy. `ExecutionEvidence` says so in the field's own docstring.
116
+
117
+ No psutil (a stripped install) ⇒ a probe that always says None.
118
+ """
119
+ try:
120
+ import psutil
121
+ except Exception: # noqa: BLE001 - an optional field must never break the run
122
+ return lambda: None
123
+
124
+ try:
125
+ psutil.cpu_percent(interval=None) # prime; the answer is meaningless
126
+ except Exception: # noqa: BLE001
127
+ return lambda: None
128
+
129
+ def probe() -> float | None:
130
+ return psutil.cpu_percent(interval=None)
131
+
132
+ return probe
133
+
134
+
135
+ class ResourceSampler(threading.Thread):
136
+ """Averages utilisation across one task's run.
137
+
138
+ Daemon thread, like `_AttemptHeartbeat` and `_CheckpointRelay`: telemetry
139
+ must never be the reason an agent will not exit. Started before the
140
+ runner and stopped after it; `stop()` returns
141
+ `(cpu_percent_mean, gpu_util_percent_mean)`, either of which may be None.
142
+
143
+ Two cadences, because the two probes cost wildly different amounts.
144
+ Reading CPU is a memory access; reading GPU utilisation spawns
145
+ `nvidia-smi`. Sampling both at one interval means either a useless CPU
146
+ trace or thousands of subprocesses over a long job on a machine somebody
147
+ lent us.
148
+ """
149
+
150
+ def __init__(
151
+ self,
152
+ cpu_probe: Probe | None = None,
153
+ gpu_probe: Probe | None = None,
154
+ cpu_interval_s: float = 1.0,
155
+ gpu_interval_s: float = 5.0,
156
+ ):
157
+ super().__init__(daemon=True)
158
+ # Built now, not on the thread: `default_cpu_probe` burns psutil's
159
+ # priming reading, and that has to happen before the run starts.
160
+ self._cpu_probe = cpu_probe if cpu_probe is not None else default_cpu_probe()
161
+ self._gpu_probe = gpu_probe if gpu_probe is not None else probe_gpu_utilisation
162
+ self._cpu_interval_s = cpu_interval_s
163
+ self._gpu_interval_s = gpu_interval_s
164
+ self._halt = threading.Event()
165
+ self._cpu: list[float] = []
166
+ self._gpu: list[float] = []
167
+ #: Set False the first time the GPU probe answers "nothing here", so
168
+ #: a host with no driver — the overwhelmingly common case — is asked
169
+ #: once instead of every few seconds for the length of the job.
170
+ self._gpu_worth_asking = True
171
+
172
+ def run(self) -> None:
173
+ next_gpu = 0.0 # sample the GPU on the first tick
174
+ while not self._halt.wait(self._cpu_interval_s):
175
+ self._sample_cpu()
176
+ now = time.monotonic()
177
+ if now >= next_gpu:
178
+ self._sample_gpu()
179
+ next_gpu = now + self._gpu_interval_s
180
+
181
+ def stop(self) -> tuple[float | None, float | None]:
182
+ """Halt, take one last reading, and report the means.
183
+
184
+ The final reading is not a courtesy: a task shorter than one tick
185
+ would otherwise produce zero samples, and "no samples" on every fast
186
+ task is how a fabricated default gets added later. It measures the
187
+ window that genuinely just elapsed.
188
+ """
189
+ self._halt.set()
190
+ if self.is_alive():
191
+ self.join(timeout=self._cpu_interval_s + 5.0)
192
+ self._sample_cpu()
193
+ if not self._gpu: # only to rescue a run too short for a single tick
194
+ self._sample_gpu()
195
+ return _mean(self._cpu), _mean(self._gpu)
196
+
197
+ # -- sampling ------------------------------------------------------------
198
+
199
+ def _ask(self, probe: Probe) -> float | None:
200
+ """One probe call that cannot take the task down with it."""
201
+ try:
202
+ return _reading(probe())
203
+ except Exception: # noqa: BLE001 - a field is never worth a run
204
+ return None
205
+
206
+ def _sample_cpu(self) -> None:
207
+ value = self._ask(self._cpu_probe)
208
+ if value is not None:
209
+ self._cpu.append(value)
210
+
211
+ def _sample_gpu(self) -> None:
212
+ if not self._gpu_worth_asking:
213
+ return
214
+ value = self._ask(self._gpu_probe)
215
+ if value is None:
216
+ # Only give up when nothing has EVER been read. Once a reading has
217
+ # succeeded the driver demonstrably exists, and one blip must not
218
+ # cost the rest of the run's GPU evidence.
219
+ if not self._gpu:
220
+ self._gpu_worth_asking = False
221
+ return
222
+ self._gpu.append(value)
223
+
224
+
225
+ def image_digest(reference: str, run: CommandRunner | None = None) -> str:
226
+ """Which image bytes actually executed, as this host resolved them.
227
+
228
+ `docker image inspect --format {{.Id}}` — the sha256 of the image config,
229
+ i.e. the local content-addressable identity of the thing that ran.
230
+
231
+ NOT `RepoDigests[0]`, the registry manifest digest, though that reads like
232
+ the more obvious answer. A locally built or `docker load`-ed image has no
233
+ `RepoDigests` entry at all, so that field is empty on exactly the hosts
234
+ most worth checking. `.Id` is always present and is what two honest nodes
235
+ running the same pinned tag on the same architecture agree on, which is
236
+ the comparison the verifier actually wants to make.
237
+
238
+ `""` means unknown: no daemon, an image it will not describe, or output
239
+ that is not a digest. Never the payload's image reference — echoing back
240
+ what the coordinator already told us is not evidence, it is a rumour with
241
+ a hash-shaped name.
242
+ """
243
+ if not reference or not isinstance(reference, str):
244
+ return ""
245
+ run = run or subprocess.run
246
+ try:
247
+ proc = run(
248
+ ["docker", "image", "inspect", reference, "--format", "{{.Id}}"],
249
+ capture_output=True,
250
+ timeout=DIGEST_TIMEOUT_S,
251
+ check=False,
252
+ )
253
+ if getattr(proc, "returncode", 1) != 0:
254
+ return ""
255
+ value = _text(getattr(proc, "stdout", None)).strip()
256
+ except Exception: # noqa: BLE001 - best effort, like every field here
257
+ return ""
258
+ return value if _DIGEST_RE.match(value) else ""
@@ -29,7 +29,7 @@ import threading
29
29
  import time
30
30
  from pathlib import Path
31
31
 
32
- from flashruntime.protocol.v1alpha1 import Lease
32
+ from flashruntime.protocol.v1alpha1 import ExecutionEvidence, Lease
33
33
 
34
34
  from flashnode.executor.archives import (
35
35
  DEFAULT_MAX_BYTES,
@@ -38,6 +38,7 @@ from flashnode.executor.archives import (
38
38
  extract_archive_safely,
39
39
  )
40
40
  from flashnode.executor.client import CoordinatorClient, LeaseLost
41
+ from flashnode.executor.evidence import ResourceSampler
41
42
  from flashnode.executor.runner import SubprocessRunner, TaskExecutionError
42
43
 
43
44
  log = logging.getLogger("flashnode.executor")
@@ -143,6 +144,7 @@ class ExecutorLoop:
143
144
  max_unpacked_members: int = DEFAULT_MAX_MEMBERS,
144
145
  health_check=None,
145
146
  max_consecutive_failures: int = 0,
147
+ sampler_factory=None,
146
148
  ):
147
149
  self.client = client
148
150
  self.node_id = node_id
@@ -185,6 +187,11 @@ class ExecutorLoop:
185
187
  # on their third unlucky job.
186
188
  self.health_check = health_check
187
189
  self.max_consecutive_failures = max_consecutive_failures
190
+ # Builds the per-task utilisation sampler. A factory, not an
191
+ # instance: a Thread runs once, so each task needs its own — and a
192
+ # factory is the seam that lets the suite drive the probes instead of
193
+ # the host's real psutil and nvidia-smi.
194
+ self.sampler_factory = sampler_factory or ResourceSampler
188
195
 
189
196
  # -- inputs --------------------------------------------------------------
190
197
 
@@ -336,9 +343,21 @@ class ExecutorLoop:
336
343
  relay = _CheckpointRelay(self.client, lease, workdir / "out" / "ckpt", prefix)
337
344
  relay.start()
338
345
 
346
+ # Execution evidence is measured around the RUNNER and
347
+ # nothing else. Folding the artifact upload into the wall
348
+ # clock would inflate every reading by however slow the
349
+ # coordinator was that minute — and inflation is the
350
+ # direction a liar wants, since the coordinator's own
351
+ # claim-to-commit elapsed is the number this is cross-checked
352
+ # against.
353
+ sampler = self.sampler_factory()
354
+ sampler.start()
355
+ started = time.monotonic()
339
356
  try:
340
357
  outdir = self.runner.run(payload, workdir, inputs)
341
358
  finally:
359
+ wall_seconds = time.monotonic() - started
360
+ cpu_mean, gpu_mean = sampler.stop()
342
361
  if relay is not None:
343
362
  relay.finish() # ship the dying attempt's last checkpoint too
344
363
 
@@ -363,7 +382,21 @@ class ExecutorLoop:
363
382
  # nested in a subdirectory.
364
383
  if rel == Path("metrics.json"):
365
384
  metrics_sha = sha
366
- accepted = self.client.complete(lease.lease_id, metrics_sha or "0" * 64)
385
+ # Read off the runner rather than inferred: only the runner
386
+ # knows whether a container ran at all. `getattr` with an
387
+ # absent default because a runner is an interface anyone may
388
+ # implement — one that measures nothing reports absence, and
389
+ # absence is a fine answer. Guessing would not be.
390
+ evidence = ExecutionEvidence(
391
+ wall_seconds=wall_seconds,
392
+ cpu_percent_mean=cpu_mean,
393
+ gpu_util_percent_mean=gpu_mean,
394
+ image_digest=getattr(self.runner, "last_image_digest", "") or "",
395
+ exit_code=getattr(self.runner, "last_exit_code", None),
396
+ )
397
+ accepted = self.client.complete(
398
+ lease.lease_id, metrics_sha or "0" * 64, evidence=evidence
399
+ )
367
400
  if accepted:
368
401
  self.tasks_accepted += 1
369
402
  log.info(_jlog("task finished", task=lease.task_id,
@@ -54,6 +54,17 @@ class SubprocessRunner:
54
54
  ):
55
55
  self.allowed_modules = allowed_modules
56
56
  self.timeout_seconds = timeout_seconds
57
+ # What the LAST run measured, read by ExecutorLoop for
58
+ # ExecutionEvidence. Reset at the top of every run(): a value left
59
+ # over from the previous task is a real measurement of a DIFFERENT
60
+ # run wearing this one's name, which is fabricated evidence by
61
+ # another route. A runner that sets neither reports absence, which
62
+ # is the honest answer for a tier that measures nothing.
63
+ self.last_exit_code: int | None = None
64
+ #: Always "" here — tier 1 runs `python -m` on the host, so no image
65
+ #: bytes executed. Echoing the payload's image reference would claim
66
+ #: a container ran when none did.
67
+ self.last_image_digest: str = ""
57
68
 
58
69
  def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
59
70
  """Execute one task payload; return the output directory.
@@ -63,6 +74,8 @@ class SubprocessRunner:
63
74
  argv ← python -m <module> --spec spec.json --out out/
64
75
  outputs → files written under out/ (metrics.json required)
65
76
  """
77
+ self.last_exit_code = None
78
+ self.last_image_digest = ""
66
79
  # Tier 1 has no isolation, so it must never execute a caller-supplied
67
80
  # command line. Argv workloads are container-only (ArgvDockerRunner);
68
81
  # refusing here keeps a misrouted payload from silently running
@@ -100,7 +113,11 @@ class SubprocessRunner:
100
113
  env=task_env(),
101
114
  )
102
115
  except subprocess.TimeoutExpired:
116
+ # Deliberately leaves last_exit_code as None: a killed process has
117
+ # no exit status we observed, and 0 or -9 here would both be a
118
+ # guess dressed as a reading.
103
119
  raise TaskExecutionError(f"task exceeded {self.timeout_seconds}s wall clock")
120
+ self.last_exit_code = proc.returncode
104
121
  if proc.returncode != 0:
105
122
  tail = proc.stderr.decode(errors="replace")[-800:]
106
123
  raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
@@ -82,6 +82,74 @@ def probe_gpus(run: CommandRunner | None = None) -> list[GpuInfo]:
82
82
  return []
83
83
 
84
84
 
85
+ #: What "how busy is it right now" is called in nvidia-smi's vocabulary.
86
+ UTILISATION_FIELD = "utilization.gpu"
87
+
88
+
89
+ def probe_gpu_utilisation(run: CommandRunner | None = None) -> float | None:
90
+ """Mean GPU utilisation across this host's devices right now, or `None`.
91
+
92
+ Feeds `ExecutionEvidence.gpu_util_percent_mean`: sampled repeatedly while
93
+ a task runs, averaged, and reported at commit time. 0% on a task that
94
+ asked for a GPU is the strongest single signal the evidence slice carries
95
+ — and the one thing that partially covers GPU work, which redundant
96
+ re-execution cannot verify at all (CUDA is non-deterministic).
97
+
98
+ `None` AND `0.0` ARE BOTH REAL ANSWERS AND THEY ARE NOT THE SAME ONE.
99
+ `0.0` means the driver was asked and reported an idle card. `None` means
100
+ there was nothing to ask — no driver, no `nvidia-smi`, output this
101
+ runtime cannot parse. Collapsing them would report every CPU-only
102
+ volunteer as a GPU host that did nothing, which is the exact shape of an
103
+ accusation.
104
+
105
+ Same contract as `probe_gpus`: never raises, never guesses, `run` is a
106
+ parameter resolved at CALL time so the suite exercises it with no driver.
107
+ Bounded by `PROBE_TIMEOUT_S` because this one is called *during* a task,
108
+ not just at registration — a wedged probe must never wedge the run.
109
+ """
110
+ run = run or subprocess.run
111
+ try:
112
+ proc = run(
113
+ [
114
+ "nvidia-smi",
115
+ f"--query-gpu={UTILISATION_FIELD}",
116
+ "--format=csv,noheader,nounits",
117
+ ],
118
+ capture_output=True,
119
+ timeout=PROBE_TIMEOUT_S,
120
+ check=False,
121
+ )
122
+ if getattr(proc, "returncode", 1) != 0:
123
+ return None
124
+ readings = _utilisations(_text(getattr(proc, "stdout", None)))
125
+ except Exception: # noqa: BLE001 - see the module docstring
126
+ return None
127
+ if not readings:
128
+ return None
129
+ return sum(readings) / len(readings)
130
+
131
+
132
+ def _utilisations(text: str) -> list[float]:
133
+ """Every percentage the driver reported, dropping what it would not say.
134
+
135
+ One unreadable card costs that card, not the reading — the same rule
136
+ `_parse` applies to devices. Values outside 0–100 are a misparse, not a
137
+ measurement, and are dropped rather than averaged in.
138
+ """
139
+ readings: list[float] = []
140
+ for line in text.splitlines():
141
+ cell = line.strip()
142
+ if not cell or cell.lower() in _PLACEHOLDERS:
143
+ continue
144
+ try:
145
+ value = float(cell.split()[0]) # tolerate a stray "%" or unit
146
+ except (ValueError, IndexError):
147
+ continue
148
+ if 0.0 <= value <= 100.0:
149
+ readings.append(value)
150
+ return readings
151
+
152
+
85
153
  def _query(run: CommandRunner, fields: Sequence[str]) -> list[GpuInfo]:
86
154
  proc = run(
87
155
  [
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flashnode
3
- Version: 0.3.2
3
+ Version: 0.3.3
4
4
  Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
7
7
  Requires-Python: >=3.10
8
8
  Description-Content-Type: text/markdown
9
9
  License-File: LICENSE
10
- Requires-Dist: flashruntime<0.5,>=0.4.1
10
+ Requires-Dist: flashruntime<0.5,>=0.4.2
11
11
  Requires-Dist: psutil>=5.9
12
12
  Requires-Dist: websockets>=12
13
13
  Requires-Dist: cryptography>=42
@@ -23,6 +23,7 @@ flashnode/executor/archives.py
23
23
  flashnode/executor/argv_runner.py
24
24
  flashnode/executor/client.py
25
25
  flashnode/executor/docker_runner.py
26
+ flashnode/executor/evidence.py
26
27
  flashnode/executor/hardening.py
27
28
  flashnode/executor/images.py
28
29
  flashnode/executor/loop.py
@@ -45,6 +46,7 @@ tests/test_credentials.py
45
46
  tests/test_docker_runner.py
46
47
  tests/test_doctor.py
47
48
  tests/test_enrol.py
49
+ tests/test_evidence.py
48
50
  tests/test_executor.py
49
51
  tests/test_gpu_probe.py
50
52
  tests/test_hardening.py
@@ -1,4 +1,4 @@
1
- flashruntime<0.5,>=0.4.1
1
+ flashruntime<0.5,>=0.4.2
2
2
  psutil>=5.9
3
3
  websockets>=12
4
4
  cryptography>=42
@@ -10,7 +10,12 @@ name = "flashnode"
10
10
  # hardcoded __version__ that made every agent register as 0.1.0.) Released from
11
11
  # the Zolli-Labs/flashml monorepo by pushing a `flashnode-vX.Y.Z` tag, and only
12
12
  # AFTER the flashruntime floor below is on PyPI.
13
- version = "0.3.2"
13
+ # 0.3.3 reports execution evidence at commit time (wall clock, mean CPU/GPU
14
+ # utilisation, the digest of the image that ran, exit code). It needs
15
+ # flashruntime 0.4.2's ExecutionEvidence — see the floor below. Bumped even
16
+ # though nothing breaks: an agent whose wire behaviour differs must not share
17
+ # a version string with one that predates it.
18
+ version = "0.3.3"
14
19
  description = "Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits."
15
20
  readme = "README.md"
16
21
  license = { text = "Apache-2.0" }
@@ -38,7 +43,13 @@ dependencies = [
38
43
  # workflow's `resolvable` job runs `flashnode --help` against PyPI alone
39
44
  # precisely to catch this, so a wrong floor here fails the release rather
40
45
  # than a volunteer's machine.
41
- "flashruntime>=0.4.1,<0.5",
46
+ # Raised again to >=0.4.2 for execution evidence, and for exactly the same
47
+ # import-time reason as the GPU probe above: `executor/client.py` and
48
+ # `executor/loop.py` import ExecutionEvidence at MODULE scope, and it does
49
+ # not exist in the published 0.4.1 wheel. The agent raises ImportError and
50
+ # never registers at all. The release workflow's `resolvable` job runs
51
+ # `flashnode --help` against PyPI alone precisely to catch this.
52
+ "flashruntime>=0.4.2,<0.5",
42
53
  "psutil>=5.9",
43
54
  "websockets>=12",
44
55
  "cryptography>=42",