flashnode 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flashnode-0.3.2 → flashnode-0.3.3}/PKG-INFO +2 -2
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/argv_runner.py +10 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/client.py +24 -5
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/docker_runner.py +12 -0
- flashnode-0.3.3/flashnode/executor/evidence.py +258 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/loop.py +35 -2
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/runner.py +17 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/inventory/gpu.py +68 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/PKG-INFO +2 -2
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/SOURCES.txt +2 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/requires.txt +1 -1
- {flashnode-0.3.2 → flashnode-0.3.3}/pyproject.toml +13 -2
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_argv_runner.py +7 -3
- flashnode-0.3.3/tests/test_evidence.py +564 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_gpu_probe.py +100 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/LICENSE +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/README.md +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/agent/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/agent/cli.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/agent/daemon.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/agent/kube.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/artifacts/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/benchmark/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/config/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/config/local_data.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/doctor.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/archives.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/hardening.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/executor/images.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/identity/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/identity/credentials.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/identity/enrol.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/identity/store.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/inventory/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/inventory/capabilities.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/status.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode/telemetry/__init__.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/dependency_links.txt +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/entry_points.txt +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/flashnode.egg-info/top_level.txt +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/setup.cfg +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_agent.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_allowlist_drift.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_archives.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_capabilities.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_checkpoint_relay.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_credentials.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_docker_runner.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_doctor.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_enrol.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_executor.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_hardening.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_hardening_platform.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_identity.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_images_allowlist.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_interfaces.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_local_data.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_loop_counters.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_status.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_work_gate.py +0 -0
- {flashnode-0.3.2 → flashnode-0.3.3}/tests/test_work_status.py +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flashnode
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
|
|
7
7
|
Requires-Python: >=3.10
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
License-File: LICENSE
|
|
10
|
-
Requires-Dist: flashruntime<0.5,>=0.4.
|
|
10
|
+
Requires-Dist: flashruntime<0.5,>=0.4.2
|
|
11
11
|
Requires-Dist: psutil>=5.9
|
|
12
12
|
Requires-Dist: websockets>=12
|
|
13
13
|
Requires-Dist: cryptography>=42
|
|
@@ -20,6 +20,7 @@ import re
|
|
|
20
20
|
import subprocess
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
|
|
23
|
+
from flashnode.executor.evidence import image_digest
|
|
23
24
|
from flashnode.executor.hardening import container_name, harden_args
|
|
24
25
|
from flashnode.executor.images import DEFAULT_ALLOWED_IMAGE_PREFIXES, image_is_allowed
|
|
25
26
|
from flashnode.executor.runner import TaskExecutionError
|
|
@@ -43,8 +44,14 @@ class ArgvDockerRunner:
|
|
|
43
44
|
self.memory_gb = memory_gb
|
|
44
45
|
self.timeout_seconds = timeout_seconds
|
|
45
46
|
self.max_output_bytes = max_output_bytes
|
|
47
|
+
# What the LAST run measured, read by ExecutorLoop for
|
|
48
|
+
# ExecutionEvidence — see SubprocessRunner for why both reset below.
|
|
49
|
+
self.last_exit_code: int | None = None
|
|
50
|
+
self.last_image_digest: str = ""
|
|
46
51
|
|
|
47
52
|
def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
|
|
53
|
+
self.last_exit_code = None
|
|
54
|
+
self.last_image_digest = ""
|
|
48
55
|
argv = payload.get("argv")
|
|
49
56
|
if not argv or not isinstance(argv, list) or not all(isinstance(t, str) for t in argv):
|
|
50
57
|
raise TaskExecutionError("payload 'argv' must be a non-empty list of strings")
|
|
@@ -105,9 +112,12 @@ class ArgvDockerRunner:
|
|
|
105
112
|
# (a subclass of OSError). Degrade to a failed task, not a dead
|
|
106
113
|
# agent — execute_one only catches TaskExecutionError/LeaseLost.
|
|
107
114
|
raise TaskExecutionError(f"docker is unavailable: {exc}") from exc
|
|
115
|
+
self.last_exit_code = proc.returncode
|
|
108
116
|
if proc.returncode != 0:
|
|
109
117
|
tail = proc.stderr.decode(errors="replace")[-800:]
|
|
110
118
|
raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
|
|
119
|
+
# Asked of the daemon, not read off the payload — see DockerRunner.
|
|
120
|
+
self.last_image_digest = image_digest(image)
|
|
111
121
|
|
|
112
122
|
# metrics.json is load-bearing, not a preference: CommandRecipe sets
|
|
113
123
|
# commit_key to <prefix>/metrics.json and the coordinator validates
|
|
@@ -19,7 +19,12 @@ import urllib.error
|
|
|
19
19
|
import urllib.request
|
|
20
20
|
from pathlib import Path
|
|
21
21
|
|
|
22
|
-
from flashruntime.protocol.v1alpha1 import
|
|
22
|
+
from flashruntime.protocol.v1alpha1 import (
|
|
23
|
+
ExecutionEvidence,
|
|
24
|
+
Lease,
|
|
25
|
+
NodeHeartbeat,
|
|
26
|
+
NodeRegistration,
|
|
27
|
+
)
|
|
23
28
|
|
|
24
29
|
|
|
25
30
|
class LeaseLost(Exception):
|
|
@@ -122,11 +127,25 @@ class CoordinatorClient:
|
|
|
122
127
|
if status != 200:
|
|
123
128
|
raise RuntimeError(f"attempt heartbeat failed ({status}): {body}")
|
|
124
129
|
|
|
125
|
-
def complete(
|
|
130
|
+
def complete(
|
|
131
|
+
self,
|
|
132
|
+
lease_id: str,
|
|
133
|
+
output_sha256: str,
|
|
134
|
+
evidence: ExecutionEvidence | None = None,
|
|
135
|
+
) -> bool:
|
|
136
|
+
"""Commit this attempt, optionally saying what the run looked like.
|
|
137
|
+
|
|
138
|
+
`evidence` is a keyword with a default so every existing caller keeps
|
|
139
|
+
working, and the body is byte-for-byte the one previous releases sent
|
|
140
|
+
when there is nothing to report — this agent may be talking to a
|
|
141
|
+
coordinator that predates the field, and an empty block is not the
|
|
142
|
+
same statement as no block.
|
|
143
|
+
"""
|
|
144
|
+
payload: dict = {"output_sha256": output_sha256}
|
|
145
|
+
if evidence is not None:
|
|
146
|
+
payload["evidence"] = evidence.model_dump(mode="json")
|
|
126
147
|
status, body = self._json(
|
|
127
|
-
"POST",
|
|
128
|
-
f"/v1alpha1/attempts/{lease_id}/complete",
|
|
129
|
-
{"output_sha256": output_sha256},
|
|
148
|
+
"POST", f"/v1alpha1/attempts/{lease_id}/complete", payload
|
|
130
149
|
)
|
|
131
150
|
if status != 200:
|
|
132
151
|
raise RuntimeError(f"complete failed ({status}): {body}")
|
|
@@ -19,6 +19,7 @@ import json
|
|
|
19
19
|
import subprocess
|
|
20
20
|
from pathlib import Path
|
|
21
21
|
|
|
22
|
+
from flashnode.executor.evidence import image_digest
|
|
22
23
|
from flashnode.executor.hardening import CONTAINER_WORKDIR, container_name, harden_args
|
|
23
24
|
from flashnode.executor.images import DEFAULT_ALLOWED_IMAGE_PREFIXES, image_is_allowed
|
|
24
25
|
from flashnode.executor.runner import DEFAULT_ALLOWED_MODULES, TaskExecutionError
|
|
@@ -44,8 +45,14 @@ class DockerRunner:
|
|
|
44
45
|
self.cpus = cpus
|
|
45
46
|
self.memory_gb = memory_gb
|
|
46
47
|
self.timeout_seconds = timeout_seconds
|
|
48
|
+
# What the LAST run measured, read by ExecutorLoop for
|
|
49
|
+
# ExecutionEvidence — see SubprocessRunner for why both reset below.
|
|
50
|
+
self.last_exit_code: int | None = None
|
|
51
|
+
self.last_image_digest: str = ""
|
|
47
52
|
|
|
48
53
|
def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
|
|
54
|
+
self.last_exit_code = None
|
|
55
|
+
self.last_image_digest = ""
|
|
49
56
|
module = payload.get("module", "")
|
|
50
57
|
if module not in self.allowed_modules:
|
|
51
58
|
raise TaskExecutionError(f"module {module!r} is not allowlisted — refusing to run")
|
|
@@ -110,9 +117,14 @@ class DockerRunner:
|
|
|
110
117
|
# (a subclass of OSError). Degrade to a failed task, not a dead
|
|
111
118
|
# agent — execute_one only catches TaskExecutionError/LeaseLost.
|
|
112
119
|
raise TaskExecutionError(f"docker is unavailable: {exc}") from exc
|
|
120
|
+
self.last_exit_code = proc.returncode
|
|
113
121
|
if proc.returncode != 0:
|
|
114
122
|
tail = proc.stderr.decode(errors="replace")[-800:]
|
|
115
123
|
raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
|
|
124
|
+
# Asked of the daemon, not read off the payload: the point of the
|
|
125
|
+
# field is which bytes ran here, and the payload only says which
|
|
126
|
+
# bytes were asked for. "" when the daemon will not say.
|
|
127
|
+
self.last_image_digest = image_digest(image)
|
|
116
128
|
if not (outdir / "metrics.json").is_file():
|
|
117
129
|
raise TaskExecutionError("task produced no metrics.json — nothing to commit")
|
|
118
130
|
return outdir
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""What this agent can honestly say about a run it just finished.
|
|
2
|
+
|
|
3
|
+
Fills `flashruntime.protocol.v1alpha1.ExecutionEvidence`, the optional block
|
|
4
|
+
the executor attaches to `complete`. Three sources, all best-effort:
|
|
5
|
+
|
|
6
|
+
- **wall clock** — measured by the loop around the runner, always available.
|
|
7
|
+
- **CPU / GPU utilisation** — sampled on a background thread while the task
|
|
8
|
+
runs (`ResourceSampler`), averaged at the end.
|
|
9
|
+
- **image digest** — asked of the local docker daemon after the container
|
|
10
|
+
exits (`image_digest`).
|
|
11
|
+
|
|
12
|
+
THE ONE RULE. A value that could not be measured is reported as `None` (or
|
|
13
|
+
`""` for the digest), never as `0`. The temptation runs entirely one way:
|
|
14
|
+
`sum([]) / max(len([]), 1)` is 0.0, `psutil.cpu_percent()`'s first call in a
|
|
15
|
+
process is 0.0, an unparsed `docker` error string is truthy. Each of those is
|
|
16
|
+
a fabricated measurement that a verifier downstream cannot tell from a real
|
|
17
|
+
one — and "0% CPU on a completed task" is not a missing field, it is an
|
|
18
|
+
accusation. Every path below that could produce a zero by accident is written
|
|
19
|
+
so it produces absence instead, and each has a test.
|
|
20
|
+
|
|
21
|
+
NO NEW DEPENDENCIES. `psutil` is already an agent dependency (inventory/);
|
|
22
|
+
GPU utilisation goes through the same `nvidia-smi` subprocess as the
|
|
23
|
+
registration probe. This code runs on strangers' machines and every
|
|
24
|
+
dependency is attack surface (AGENTS.md).
|
|
25
|
+
|
|
26
|
+
Every probe takes its subprocess/sampler as a PARAMETER, resolved at call
|
|
27
|
+
time, so the whole test suite runs with no NVIDIA driver and no docker
|
|
28
|
+
daemon — the shape `inventory/gpu.py` and `doctor.py` established.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import math
|
|
34
|
+
import re
|
|
35
|
+
import subprocess
|
|
36
|
+
import threading
|
|
37
|
+
import time
|
|
38
|
+
from typing import Callable
|
|
39
|
+
|
|
40
|
+
from flashnode.inventory.gpu import probe_gpu_utilisation
|
|
41
|
+
|
|
42
|
+
__all__ = ["ResourceSampler", "default_cpu_probe", "image_digest"]
|
|
43
|
+
|
|
44
|
+
#: A probe answers "what is it now?" — a percentage, or None for "could not
|
|
45
|
+
#: read it". Never raises for the caller; the sampler defends anyway.
|
|
46
|
+
Probe = Callable[[], "float | None"]
|
|
47
|
+
|
|
48
|
+
CommandRunner = Callable[..., subprocess.CompletedProcess]
|
|
49
|
+
|
|
50
|
+
#: `docker image inspect` reads metadata the daemon already holds. Short on
|
|
51
|
+
#: purpose: this runs on the commit path, and a wedged daemon must cost the
|
|
52
|
+
#: field, not the task's result.
|
|
53
|
+
DIGEST_TIMEOUT_S = 10
|
|
54
|
+
|
|
55
|
+
#: What a content-addressable image identity looks like. Anything else the
|
|
56
|
+
#: daemon prints — a prose error, `<no value>` from a template miss, an empty
|
|
57
|
+
#: line — is NOT a digest and must not be forwarded as one.
|
|
58
|
+
_DIGEST_RE = re.compile(r"^[a-z0-9]+:[0-9a-f]{32,}$")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _text(raw: object) -> str:
|
|
62
|
+
"""Subprocess output as a string, or `""` for anything that is not one.
|
|
63
|
+
|
|
64
|
+
Total on purpose, and not quite the shape `inventory/gpu.py` uses: the
|
|
65
|
+
runner is a parameter, so `stdout` is whatever a caller's stub put there.
|
|
66
|
+
A duck-typed object with a `.decode` that returns another object would
|
|
67
|
+
otherwise walk a non-string all the way to the regex below and raise
|
|
68
|
+
TypeError from inside a best-effort probe.
|
|
69
|
+
"""
|
|
70
|
+
if isinstance(raw, str):
|
|
71
|
+
return raw
|
|
72
|
+
if isinstance(raw, (bytes, bytearray)):
|
|
73
|
+
return bytes(raw).decode(errors="replace")
|
|
74
|
+
return ""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _reading(value: object) -> float | None:
|
|
78
|
+
"""One probe answer as a number the mean can survive, or None.
|
|
79
|
+
|
|
80
|
+
NaN is the reason this exists: a single NaN sample turns a whole run's
|
|
81
|
+
mean into NaN, which serialises to something no verifier can read. `bool`
|
|
82
|
+
is excluded explicitly — it is a subclass of `int`, so `True` would
|
|
83
|
+
otherwise average in as 1.0%.
|
|
84
|
+
"""
|
|
85
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
86
|
+
return None
|
|
87
|
+
number = float(value)
|
|
88
|
+
return number if math.isfinite(number) else None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _mean(values: list[float]) -> float | None:
|
|
92
|
+
"""The average, or None for no samples at all.
|
|
93
|
+
|
|
94
|
+
NOT 0.0. This one line is the whole honesty contract of this module: an
|
|
95
|
+
empty list means nothing was measured, and a verifier reading 0.0 would
|
|
96
|
+
see a machine that did nothing on a task it completed.
|
|
97
|
+
"""
|
|
98
|
+
if not values:
|
|
99
|
+
return None
|
|
100
|
+
return sum(values) / len(values)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def default_cpu_probe() -> Probe:
|
|
104
|
+
"""A host-wide CPU probe, with psutil's meaningless first reading burned.
|
|
105
|
+
|
|
106
|
+
`psutil.cpu_percent(interval=None)` reports usage since the LAST call, so
|
|
107
|
+
its first answer in a process is 0.0 by construction — a fabricated zero
|
|
108
|
+
that looks exactly like an idle machine. That call happens here, at
|
|
109
|
+
construction, before the task starts; every reading the sampler takes
|
|
110
|
+
afterwards covers a real window.
|
|
111
|
+
|
|
112
|
+
Host-wide, not task-scoped: a volunteer's machine has other things
|
|
113
|
+
running on it. That makes a HIGH reading weak evidence and a LOW one the
|
|
114
|
+
interesting direction — a completed training task on a machine that never
|
|
115
|
+
got busy. `ExecutionEvidence` says so in the field's own docstring.
|
|
116
|
+
|
|
117
|
+
No psutil (a stripped install) ⇒ a probe that always says None.
|
|
118
|
+
"""
|
|
119
|
+
try:
|
|
120
|
+
import psutil
|
|
121
|
+
except Exception: # noqa: BLE001 - an optional field must never break the run
|
|
122
|
+
return lambda: None
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
psutil.cpu_percent(interval=None) # prime; the answer is meaningless
|
|
126
|
+
except Exception: # noqa: BLE001
|
|
127
|
+
return lambda: None
|
|
128
|
+
|
|
129
|
+
def probe() -> float | None:
|
|
130
|
+
return psutil.cpu_percent(interval=None)
|
|
131
|
+
|
|
132
|
+
return probe
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class ResourceSampler(threading.Thread):
|
|
136
|
+
"""Averages utilisation across one task's run.
|
|
137
|
+
|
|
138
|
+
Daemon thread, like `_AttemptHeartbeat` and `_CheckpointRelay`: telemetry
|
|
139
|
+
must never be the reason an agent will not exit. Started before the
|
|
140
|
+
runner and stopped after it; `stop()` returns
|
|
141
|
+
`(cpu_percent_mean, gpu_util_percent_mean)`, either of which may be None.
|
|
142
|
+
|
|
143
|
+
Two cadences, because the two probes cost wildly different amounts.
|
|
144
|
+
Reading CPU is a memory access; reading GPU utilisation spawns
|
|
145
|
+
`nvidia-smi`. Sampling both at one interval means either a useless CPU
|
|
146
|
+
trace or thousands of subprocesses over a long job on a machine somebody
|
|
147
|
+
lent us.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
def __init__(
|
|
151
|
+
self,
|
|
152
|
+
cpu_probe: Probe | None = None,
|
|
153
|
+
gpu_probe: Probe | None = None,
|
|
154
|
+
cpu_interval_s: float = 1.0,
|
|
155
|
+
gpu_interval_s: float = 5.0,
|
|
156
|
+
):
|
|
157
|
+
super().__init__(daemon=True)
|
|
158
|
+
# Built now, not on the thread: `default_cpu_probe` burns psutil's
|
|
159
|
+
# priming reading, and that has to happen before the run starts.
|
|
160
|
+
self._cpu_probe = cpu_probe if cpu_probe is not None else default_cpu_probe()
|
|
161
|
+
self._gpu_probe = gpu_probe if gpu_probe is not None else probe_gpu_utilisation
|
|
162
|
+
self._cpu_interval_s = cpu_interval_s
|
|
163
|
+
self._gpu_interval_s = gpu_interval_s
|
|
164
|
+
self._halt = threading.Event()
|
|
165
|
+
self._cpu: list[float] = []
|
|
166
|
+
self._gpu: list[float] = []
|
|
167
|
+
#: Set False the first time the GPU probe answers "nothing here", so
|
|
168
|
+
#: a host with no driver — the overwhelmingly common case — is asked
|
|
169
|
+
#: once instead of every few seconds for the length of the job.
|
|
170
|
+
self._gpu_worth_asking = True
|
|
171
|
+
|
|
172
|
+
def run(self) -> None:
|
|
173
|
+
next_gpu = 0.0 # sample the GPU on the first tick
|
|
174
|
+
while not self._halt.wait(self._cpu_interval_s):
|
|
175
|
+
self._sample_cpu()
|
|
176
|
+
now = time.monotonic()
|
|
177
|
+
if now >= next_gpu:
|
|
178
|
+
self._sample_gpu()
|
|
179
|
+
next_gpu = now + self._gpu_interval_s
|
|
180
|
+
|
|
181
|
+
def stop(self) -> tuple[float | None, float | None]:
|
|
182
|
+
"""Halt, take one last reading, and report the means.
|
|
183
|
+
|
|
184
|
+
The final reading is not a courtesy: a task shorter than one tick
|
|
185
|
+
would otherwise produce zero samples, and "no samples" on every fast
|
|
186
|
+
task is how a fabricated default gets added later. It measures the
|
|
187
|
+
window that genuinely just elapsed.
|
|
188
|
+
"""
|
|
189
|
+
self._halt.set()
|
|
190
|
+
if self.is_alive():
|
|
191
|
+
self.join(timeout=self._cpu_interval_s + 5.0)
|
|
192
|
+
self._sample_cpu()
|
|
193
|
+
if not self._gpu: # only to rescue a run too short for a single tick
|
|
194
|
+
self._sample_gpu()
|
|
195
|
+
return _mean(self._cpu), _mean(self._gpu)
|
|
196
|
+
|
|
197
|
+
# -- sampling ------------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
def _ask(self, probe: Probe) -> float | None:
|
|
200
|
+
"""One probe call that cannot take the task down with it."""
|
|
201
|
+
try:
|
|
202
|
+
return _reading(probe())
|
|
203
|
+
except Exception: # noqa: BLE001 - a field is never worth a run
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
def _sample_cpu(self) -> None:
|
|
207
|
+
value = self._ask(self._cpu_probe)
|
|
208
|
+
if value is not None:
|
|
209
|
+
self._cpu.append(value)
|
|
210
|
+
|
|
211
|
+
def _sample_gpu(self) -> None:
|
|
212
|
+
if not self._gpu_worth_asking:
|
|
213
|
+
return
|
|
214
|
+
value = self._ask(self._gpu_probe)
|
|
215
|
+
if value is None:
|
|
216
|
+
# Only give up when nothing has EVER been read. Once a reading has
|
|
217
|
+
# succeeded the driver demonstrably exists, and one blip must not
|
|
218
|
+
# cost the rest of the run's GPU evidence.
|
|
219
|
+
if not self._gpu:
|
|
220
|
+
self._gpu_worth_asking = False
|
|
221
|
+
return
|
|
222
|
+
self._gpu.append(value)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def image_digest(reference: str, run: CommandRunner | None = None) -> str:
|
|
226
|
+
"""Which image bytes actually executed, as this host resolved them.
|
|
227
|
+
|
|
228
|
+
`docker image inspect --format {{.Id}}` — the sha256 of the image config,
|
|
229
|
+
i.e. the local content-addressable identity of the thing that ran.
|
|
230
|
+
|
|
231
|
+
NOT `RepoDigests[0]`, the registry manifest digest, though that reads like
|
|
232
|
+
the more obvious answer. A locally built or `docker load`-ed image has no
|
|
233
|
+
`RepoDigests` entry at all, so that field is empty on exactly the hosts
|
|
234
|
+
most worth checking. `.Id` is always present and is what two honest nodes
|
|
235
|
+
running the same pinned tag on the same architecture agree on, which is
|
|
236
|
+
the comparison the verifier actually wants to make.
|
|
237
|
+
|
|
238
|
+
`""` means unknown: no daemon, an image it will not describe, or output
|
|
239
|
+
that is not a digest. Never the payload's image reference — echoing back
|
|
240
|
+
what the coordinator already told us is not evidence, it is a rumour with
|
|
241
|
+
a hash-shaped name.
|
|
242
|
+
"""
|
|
243
|
+
if not reference or not isinstance(reference, str):
|
|
244
|
+
return ""
|
|
245
|
+
run = run or subprocess.run
|
|
246
|
+
try:
|
|
247
|
+
proc = run(
|
|
248
|
+
["docker", "image", "inspect", reference, "--format", "{{.Id}}"],
|
|
249
|
+
capture_output=True,
|
|
250
|
+
timeout=DIGEST_TIMEOUT_S,
|
|
251
|
+
check=False,
|
|
252
|
+
)
|
|
253
|
+
if getattr(proc, "returncode", 1) != 0:
|
|
254
|
+
return ""
|
|
255
|
+
value = _text(getattr(proc, "stdout", None)).strip()
|
|
256
|
+
except Exception: # noqa: BLE001 - best effort, like every field here
|
|
257
|
+
return ""
|
|
258
|
+
return value if _DIGEST_RE.match(value) else ""
|
|
@@ -29,7 +29,7 @@ import threading
|
|
|
29
29
|
import time
|
|
30
30
|
from pathlib import Path
|
|
31
31
|
|
|
32
|
-
from flashruntime.protocol.v1alpha1 import Lease
|
|
32
|
+
from flashruntime.protocol.v1alpha1 import ExecutionEvidence, Lease
|
|
33
33
|
|
|
34
34
|
from flashnode.executor.archives import (
|
|
35
35
|
DEFAULT_MAX_BYTES,
|
|
@@ -38,6 +38,7 @@ from flashnode.executor.archives import (
|
|
|
38
38
|
extract_archive_safely,
|
|
39
39
|
)
|
|
40
40
|
from flashnode.executor.client import CoordinatorClient, LeaseLost
|
|
41
|
+
from flashnode.executor.evidence import ResourceSampler
|
|
41
42
|
from flashnode.executor.runner import SubprocessRunner, TaskExecutionError
|
|
42
43
|
|
|
43
44
|
log = logging.getLogger("flashnode.executor")
|
|
@@ -143,6 +144,7 @@ class ExecutorLoop:
|
|
|
143
144
|
max_unpacked_members: int = DEFAULT_MAX_MEMBERS,
|
|
144
145
|
health_check=None,
|
|
145
146
|
max_consecutive_failures: int = 0,
|
|
147
|
+
sampler_factory=None,
|
|
146
148
|
):
|
|
147
149
|
self.client = client
|
|
148
150
|
self.node_id = node_id
|
|
@@ -185,6 +187,11 @@ class ExecutorLoop:
|
|
|
185
187
|
# on their third unlucky job.
|
|
186
188
|
self.health_check = health_check
|
|
187
189
|
self.max_consecutive_failures = max_consecutive_failures
|
|
190
|
+
# Builds the per-task utilisation sampler. A factory, not an
|
|
191
|
+
# instance: a Thread runs once, so each task needs its own — and a
|
|
192
|
+
# factory is the seam that lets the suite drive the probes instead of
|
|
193
|
+
# the host's real psutil and nvidia-smi.
|
|
194
|
+
self.sampler_factory = sampler_factory or ResourceSampler
|
|
188
195
|
|
|
189
196
|
# -- inputs --------------------------------------------------------------
|
|
190
197
|
|
|
@@ -336,9 +343,21 @@ class ExecutorLoop:
|
|
|
336
343
|
relay = _CheckpointRelay(self.client, lease, workdir / "out" / "ckpt", prefix)
|
|
337
344
|
relay.start()
|
|
338
345
|
|
|
346
|
+
# Execution evidence is measured around the RUNNER and
|
|
347
|
+
# nothing else. Folding the artifact upload into the wall
|
|
348
|
+
# clock would inflate every reading by however slow the
|
|
349
|
+
# coordinator was that minute — and inflation is the
|
|
350
|
+
# direction a liar wants, since the coordinator's own
|
|
351
|
+
# claim-to-commit elapsed is the number this is cross-checked
|
|
352
|
+
# against.
|
|
353
|
+
sampler = self.sampler_factory()
|
|
354
|
+
sampler.start()
|
|
355
|
+
started = time.monotonic()
|
|
339
356
|
try:
|
|
340
357
|
outdir = self.runner.run(payload, workdir, inputs)
|
|
341
358
|
finally:
|
|
359
|
+
wall_seconds = time.monotonic() - started
|
|
360
|
+
cpu_mean, gpu_mean = sampler.stop()
|
|
342
361
|
if relay is not None:
|
|
343
362
|
relay.finish() # ship the dying attempt's last checkpoint too
|
|
344
363
|
|
|
@@ -363,7 +382,21 @@ class ExecutorLoop:
|
|
|
363
382
|
# nested in a subdirectory.
|
|
364
383
|
if rel == Path("metrics.json"):
|
|
365
384
|
metrics_sha = sha
|
|
366
|
-
|
|
385
|
+
# Read off the runner rather than inferred: only the runner
|
|
386
|
+
# knows whether a container ran at all. `getattr` with an
|
|
387
|
+
# absent default because a runner is an interface anyone may
|
|
388
|
+
# implement — one that measures nothing reports absence, and
|
|
389
|
+
# absence is a fine answer. Guessing would not be.
|
|
390
|
+
evidence = ExecutionEvidence(
|
|
391
|
+
wall_seconds=wall_seconds,
|
|
392
|
+
cpu_percent_mean=cpu_mean,
|
|
393
|
+
gpu_util_percent_mean=gpu_mean,
|
|
394
|
+
image_digest=getattr(self.runner, "last_image_digest", "") or "",
|
|
395
|
+
exit_code=getattr(self.runner, "last_exit_code", None),
|
|
396
|
+
)
|
|
397
|
+
accepted = self.client.complete(
|
|
398
|
+
lease.lease_id, metrics_sha or "0" * 64, evidence=evidence
|
|
399
|
+
)
|
|
367
400
|
if accepted:
|
|
368
401
|
self.tasks_accepted += 1
|
|
369
402
|
log.info(_jlog("task finished", task=lease.task_id,
|
|
@@ -54,6 +54,17 @@ class SubprocessRunner:
|
|
|
54
54
|
):
|
|
55
55
|
self.allowed_modules = allowed_modules
|
|
56
56
|
self.timeout_seconds = timeout_seconds
|
|
57
|
+
# What the LAST run measured, read by ExecutorLoop for
|
|
58
|
+
# ExecutionEvidence. Reset at the top of every run(): a value left
|
|
59
|
+
# over from the previous task is a real measurement of a DIFFERENT
|
|
60
|
+
# run wearing this one's name, which is fabricated evidence by
|
|
61
|
+
# another route. A runner that sets neither reports absence, which
|
|
62
|
+
# is the honest answer for a tier that measures nothing.
|
|
63
|
+
self.last_exit_code: int | None = None
|
|
64
|
+
#: Always "" here — tier 1 runs `python -m` on the host, so no image
|
|
65
|
+
#: bytes executed. Echoing the payload's image reference would claim
|
|
66
|
+
#: a container ran when none did.
|
|
67
|
+
self.last_image_digest: str = ""
|
|
57
68
|
|
|
58
69
|
def run(self, payload: dict, workdir: Path, inputs: dict[str, Path]) -> Path:
|
|
59
70
|
"""Execute one task payload; return the output directory.
|
|
@@ -63,6 +74,8 @@ class SubprocessRunner:
|
|
|
63
74
|
argv ← python -m <module> --spec spec.json --out out/
|
|
64
75
|
outputs → files written under out/ (metrics.json required)
|
|
65
76
|
"""
|
|
77
|
+
self.last_exit_code = None
|
|
78
|
+
self.last_image_digest = ""
|
|
66
79
|
# Tier 1 has no isolation, so it must never execute a caller-supplied
|
|
67
80
|
# command line. Argv workloads are container-only (ArgvDockerRunner);
|
|
68
81
|
# refusing here keeps a misrouted payload from silently running
|
|
@@ -100,7 +113,11 @@ class SubprocessRunner:
|
|
|
100
113
|
env=task_env(),
|
|
101
114
|
)
|
|
102
115
|
except subprocess.TimeoutExpired:
|
|
116
|
+
# Deliberately leaves last_exit_code as None: a killed process has
|
|
117
|
+
# no exit status we observed, and 0 or -9 here would both be a
|
|
118
|
+
# guess dressed as a reading.
|
|
103
119
|
raise TaskExecutionError(f"task exceeded {self.timeout_seconds}s wall clock")
|
|
120
|
+
self.last_exit_code = proc.returncode
|
|
104
121
|
if proc.returncode != 0:
|
|
105
122
|
tail = proc.stderr.decode(errors="replace")[-800:]
|
|
106
123
|
raise TaskExecutionError(f"task exited {proc.returncode}: {tail}")
|
|
@@ -82,6 +82,74 @@ def probe_gpus(run: CommandRunner | None = None) -> list[GpuInfo]:
|
|
|
82
82
|
return []
|
|
83
83
|
|
|
84
84
|
|
|
85
|
+
#: What "how busy is it right now" is called in nvidia-smi's vocabulary.
|
|
86
|
+
UTILISATION_FIELD = "utilization.gpu"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def probe_gpu_utilisation(run: CommandRunner | None = None) -> float | None:
|
|
90
|
+
"""Mean GPU utilisation across this host's devices right now, or `None`.
|
|
91
|
+
|
|
92
|
+
Feeds `ExecutionEvidence.gpu_util_percent_mean`: sampled repeatedly while
|
|
93
|
+
a task runs, averaged, and reported at commit time. 0% on a task that
|
|
94
|
+
asked for a GPU is the strongest single signal the evidence slice carries
|
|
95
|
+
— and the one thing that partially covers GPU work, which redundant
|
|
96
|
+
re-execution cannot verify at all (CUDA is non-deterministic).
|
|
97
|
+
|
|
98
|
+
`None` AND `0.0` ARE BOTH REAL ANSWERS AND THEY ARE NOT THE SAME ONE.
|
|
99
|
+
`0.0` means the driver was asked and reported an idle card. `None` means
|
|
100
|
+
there was nothing to ask — no driver, no `nvidia-smi`, output this
|
|
101
|
+
runtime cannot parse. Collapsing them would report every CPU-only
|
|
102
|
+
volunteer as a GPU host that did nothing, which is the exact shape of an
|
|
103
|
+
accusation.
|
|
104
|
+
|
|
105
|
+
Same contract as `probe_gpus`: never raises, never guesses, `run` is a
|
|
106
|
+
parameter resolved at CALL time so the suite exercises it with no driver.
|
|
107
|
+
Bounded by `PROBE_TIMEOUT_S` because this one is called *during* a task,
|
|
108
|
+
not just at registration — a wedged probe must never wedge the run.
|
|
109
|
+
"""
|
|
110
|
+
run = run or subprocess.run
|
|
111
|
+
try:
|
|
112
|
+
proc = run(
|
|
113
|
+
[
|
|
114
|
+
"nvidia-smi",
|
|
115
|
+
f"--query-gpu={UTILISATION_FIELD}",
|
|
116
|
+
"--format=csv,noheader,nounits",
|
|
117
|
+
],
|
|
118
|
+
capture_output=True,
|
|
119
|
+
timeout=PROBE_TIMEOUT_S,
|
|
120
|
+
check=False,
|
|
121
|
+
)
|
|
122
|
+
if getattr(proc, "returncode", 1) != 0:
|
|
123
|
+
return None
|
|
124
|
+
readings = _utilisations(_text(getattr(proc, "stdout", None)))
|
|
125
|
+
except Exception: # noqa: BLE001 - see the module docstring
|
|
126
|
+
return None
|
|
127
|
+
if not readings:
|
|
128
|
+
return None
|
|
129
|
+
return sum(readings) / len(readings)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _utilisations(text: str) -> list[float]:
|
|
133
|
+
"""Every percentage the driver reported, dropping what it would not say.
|
|
134
|
+
|
|
135
|
+
One unreadable card costs that card, not the reading — the same rule
|
|
136
|
+
`_parse` applies to devices. Values outside 0–100 are a misparse, not a
|
|
137
|
+
measurement, and are dropped rather than averaged in.
|
|
138
|
+
"""
|
|
139
|
+
readings: list[float] = []
|
|
140
|
+
for line in text.splitlines():
|
|
141
|
+
cell = line.strip()
|
|
142
|
+
if not cell or cell.lower() in _PLACEHOLDERS:
|
|
143
|
+
continue
|
|
144
|
+
try:
|
|
145
|
+
value = float(cell.split()[0]) # tolerate a stray "%" or unit
|
|
146
|
+
except (ValueError, IndexError):
|
|
147
|
+
continue
|
|
148
|
+
if 0.0 <= value <= 100.0:
|
|
149
|
+
readings.append(value)
|
|
150
|
+
return readings
|
|
151
|
+
|
|
152
|
+
|
|
85
153
|
def _query(run: CommandRunner, fields: Sequence[str]) -> list[GpuInfo]:
|
|
86
154
|
proc = run(
|
|
87
155
|
[
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flashnode
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/Zolli-Labs/flashnode
|
|
7
7
|
Requires-Python: >=3.10
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
License-File: LICENSE
|
|
10
|
-
Requires-Dist: flashruntime<0.5,>=0.4.
|
|
10
|
+
Requires-Dist: flashruntime<0.5,>=0.4.2
|
|
11
11
|
Requires-Dist: psutil>=5.9
|
|
12
12
|
Requires-Dist: websockets>=12
|
|
13
13
|
Requires-Dist: cryptography>=42
|
|
@@ -23,6 +23,7 @@ flashnode/executor/archives.py
|
|
|
23
23
|
flashnode/executor/argv_runner.py
|
|
24
24
|
flashnode/executor/client.py
|
|
25
25
|
flashnode/executor/docker_runner.py
|
|
26
|
+
flashnode/executor/evidence.py
|
|
26
27
|
flashnode/executor/hardening.py
|
|
27
28
|
flashnode/executor/images.py
|
|
28
29
|
flashnode/executor/loop.py
|
|
@@ -45,6 +46,7 @@ tests/test_credentials.py
|
|
|
45
46
|
tests/test_docker_runner.py
|
|
46
47
|
tests/test_doctor.py
|
|
47
48
|
tests/test_enrol.py
|
|
49
|
+
tests/test_evidence.py
|
|
48
50
|
tests/test_executor.py
|
|
49
51
|
tests/test_gpu_probe.py
|
|
50
52
|
tests/test_hardening.py
|
|
@@ -10,7 +10,12 @@ name = "flashnode"
|
|
|
10
10
|
# hardcoded __version__ that made every agent register as 0.1.0.) Released from
|
|
11
11
|
# the Zolli-Labs/flashml monorepo by pushing a `flashnode-vX.Y.Z` tag, and only
|
|
12
12
|
# AFTER the flashruntime floor below is on PyPI.
|
|
13
|
-
|
|
13
|
+
# 0.3.3 reports execution evidence at commit time (wall clock, mean CPU/GPU
|
|
14
|
+
# utilisation, the digest of the image that ran, exit code). It needs
|
|
15
|
+
# flashruntime 0.4.2's ExecutionEvidence — see the floor below. Bumped even
|
|
16
|
+
# though nothing breaks: an agent whose wire behaviour differs must not share
|
|
17
|
+
# a version string with one that predates it.
|
|
18
|
+
version = "0.3.3"
|
|
14
19
|
description = "Open host agent for the FlashML fragmented-compute network: join, benchmark, execute sandboxed ML tasks, earn contribution credits."
|
|
15
20
|
readme = "README.md"
|
|
16
21
|
license = { text = "Apache-2.0" }
|
|
@@ -38,7 +43,13 @@ dependencies = [
|
|
|
38
43
|
# workflow's `resolvable` job runs `flashnode --help` against PyPI alone
|
|
39
44
|
# precisely to catch this, so a wrong floor here fails the release rather
|
|
40
45
|
# than a volunteer's machine.
|
|
41
|
-
|
|
46
|
+
# Raised again to >=0.4.2 for execution evidence, and for exactly the same
|
|
47
|
+
# import-time reason as the GPU probe above: `executor/client.py` and
|
|
48
|
+
# `executor/loop.py` import ExecutionEvidence at MODULE scope, and it does
|
|
49
|
+
# not exist in the published 0.4.1 wheel. The agent raises ImportError and
|
|
50
|
+
# never registers at all. The release workflow's `resolvable` job runs
|
|
51
|
+
# `flashnode --help` against PyPI alone precisely to catch this.
|
|
52
|
+
"flashruntime>=0.4.2,<0.5",
|
|
42
53
|
"psutil>=5.9",
|
|
43
54
|
"websockets>=12",
|
|
44
55
|
"cryptography>=42",
|