flashnode 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flashnode/__init__.py +7 -0
- flashnode/agent/__init__.py +6 -0
- flashnode/agent/cli.py +285 -0
- flashnode/agent/daemon.py +108 -0
- flashnode/agent/kube.py +41 -0
- flashnode/artifacts/__init__.py +5 -0
- flashnode/benchmark/__init__.py +100 -0
- flashnode/config/__init__.py +103 -0
- flashnode/executor/__init__.py +27 -0
- flashnode/executor/archives.py +290 -0
- flashnode/executor/argv_runner.py +114 -0
- flashnode/executor/client.py +201 -0
- flashnode/executor/docker_runner.py +108 -0
- flashnode/executor/hardening.py +141 -0
- flashnode/executor/images.py +161 -0
- flashnode/executor/loop.py +363 -0
- flashnode/executor/runner.py +109 -0
- flashnode/identity/__init__.py +5 -0
- flashnode/identity/credentials.py +70 -0
- flashnode/identity/enrol.py +179 -0
- flashnode/identity/store.py +48 -0
- flashnode/inventory/__init__.py +5 -0
- flashnode/inventory/capabilities.py +117 -0
- flashnode/telemetry/__init__.py +79 -0
- flashnode-0.2.0.dist-info/METADATA +160 -0
- flashnode-0.2.0.dist-info/RECORD +30 -0
- flashnode-0.2.0.dist-info/WHEEL +5 -0
- flashnode-0.2.0.dist-info/entry_points.txt +2 -0
- flashnode-0.2.0.dist-info/licenses/LICENSE +202 -0
- flashnode-0.2.0.dist-info/top_level.txt +1 -0
flashnode/__init__.py
ADDED
flashnode/agent/cli.py
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""Command-line entry point for the FlashNode agent.
|
|
2
|
+
|
|
3
|
+
Target surface (see docs/SYSTEM_OVERVIEW.md §10):
|
|
4
|
+
|
|
5
|
+
flashnode join --code <one-time-code>
|
|
6
|
+
flashnode status
|
|
7
|
+
flashnode leave
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
from flashnode import __version__
|
|
15
|
+
|
|
16
|
+
USAGE = """\
|
|
17
|
+
flashnode {version} — FlashML open host agent (pre-release scaffold)
|
|
18
|
+
|
|
19
|
+
usage: flashnode <command>
|
|
20
|
+
|
|
21
|
+
commands:
|
|
22
|
+
agent run the node agent loop (register with FlashML Cloud + heartbeat)
|
|
23
|
+
work register with a FlashRuntime coordinator and execute leased tasks
|
|
24
|
+
(--coordinator URL | FLASHNODE_COORDINATOR_URL; --max-tasks N)
|
|
25
|
+
login enrol this machine — prints a code to approve in a browser
|
|
26
|
+
(--coordinator URL; --token TOKEN to skip the browser step)
|
|
27
|
+
logout remove the saved bearer token for a FlashRuntime coordinator
|
|
28
|
+
(--coordinator URL)
|
|
29
|
+
join connect this machine to a FlashML control plane (not yet implemented)
|
|
30
|
+
status show node identity, capabilities, and active leases (not yet implemented)
|
|
31
|
+
leave drain and disconnect (not yet implemented)
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _login(args: list[str]) -> int:
|
|
36
|
+
"""Enrol this machine.
|
|
37
|
+
|
|
38
|
+
The default path is device-code: print a short code, wait for a
|
|
39
|
+
signed-in human to approve it in a browser, save the token that comes
|
|
40
|
+
back. That is what the console tells volunteers to run, and until now it
|
|
41
|
+
could not work — `--token` was REQUIRED, and the enrolment flow issues
|
|
42
|
+
no token for anyone to paste. The API half (/v1alpha1/device/code,
|
|
43
|
+
/v1alpha1/device/token) had been built and simply had no client.
|
|
44
|
+
|
|
45
|
+
`--token` stays supported for a credential you already hold: CI, a
|
|
46
|
+
self-hosted coordinator, or re-pointing a machine with no browser to
|
|
47
|
+
hand.
|
|
48
|
+
"""
|
|
49
|
+
import argparse
|
|
50
|
+
|
|
51
|
+
from flashnode.identity.credentials import save_token
|
|
52
|
+
|
|
53
|
+
parser = argparse.ArgumentParser(
|
|
54
|
+
prog="flashnode login",
|
|
55
|
+
description="Enrol this machine with FlashML.",
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
"--coordinator",
|
|
59
|
+
required=True,
|
|
60
|
+
help="FlashML Cloud API base URL (e.g. https://flashml-api.onrender.com)",
|
|
61
|
+
)
|
|
62
|
+
parser.add_argument(
|
|
63
|
+
"--token",
|
|
64
|
+
help="skip the browser step and save a token you already have",
|
|
65
|
+
)
|
|
66
|
+
opts = parser.parse_args(args)
|
|
67
|
+
|
|
68
|
+
if opts.token:
|
|
69
|
+
path = save_token(opts.coordinator, opts.token)
|
|
70
|
+
print(f"flashnode login: credential saved to {path}", file=sys.stderr)
|
|
71
|
+
return 0
|
|
72
|
+
|
|
73
|
+
from flashnode.identity.enrol import (
|
|
74
|
+
EnrolmentError,
|
|
75
|
+
describe_this_machine,
|
|
76
|
+
poll_for_token,
|
|
77
|
+
request_device_code,
|
|
78
|
+
)
|
|
79
|
+
from flashnode.identity.store import load_or_create_node_id
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
node_id = load_or_create_node_id()
|
|
83
|
+
except OSError as exc:
|
|
84
|
+
print(
|
|
85
|
+
f"flashnode login: cannot write this machine's identity: {exc}\n"
|
|
86
|
+
"Set FLASHNODE_STATE_DIR to a directory you can write to.",
|
|
87
|
+
file=sys.stderr,
|
|
88
|
+
)
|
|
89
|
+
return 1
|
|
90
|
+
|
|
91
|
+
hostname, platform_name = describe_this_machine()
|
|
92
|
+
|
|
93
|
+
try:
|
|
94
|
+
start = request_device_code(
|
|
95
|
+
opts.coordinator, node_id, hostname, platform_name
|
|
96
|
+
)
|
|
97
|
+
except EnrolmentError as exc:
|
|
98
|
+
print(f"flashnode login: {exc}", file=sys.stderr)
|
|
99
|
+
return 1
|
|
100
|
+
|
|
101
|
+
# stdout, not stderr: this is the output the person is here for, and it
|
|
102
|
+
# should survive being piped.
|
|
103
|
+
#
|
|
104
|
+
# flush=True is load-bearing. Python block-buffers stdout when it is not
|
|
105
|
+
# a terminal, so piping `flashnode login` anywhere — tee, a log, a setup
|
|
106
|
+
# script — showed nothing at all while the process sat waiting for an
|
|
107
|
+
# approval of a code it had never displayed.
|
|
108
|
+
print(flush=True)
|
|
109
|
+
print(f" Your code: {start.user_code}", flush=True)
|
|
110
|
+
print(f" Approve at: {start.verification_uri}", flush=True)
|
|
111
|
+
print(flush=True)
|
|
112
|
+
print(
|
|
113
|
+
"Open that on any device you're signed in on — your phone is fine.",
|
|
114
|
+
flush=True,
|
|
115
|
+
)
|
|
116
|
+
print("Waiting for approval… (Ctrl-C to cancel)", flush=True)
|
|
117
|
+
|
|
118
|
+
try:
|
|
119
|
+
token = poll_for_token(
|
|
120
|
+
opts.coordinator, start.device_code, interval=start.interval
|
|
121
|
+
)
|
|
122
|
+
except EnrolmentError as exc:
|
|
123
|
+
print(f"\nflashnode login: {exc}", file=sys.stderr)
|
|
124
|
+
return 1
|
|
125
|
+
except KeyboardInterrupt:
|
|
126
|
+
# Cancelling is a normal act, not a crash. The code expires unused.
|
|
127
|
+
print("\nflashnode login: cancelled.", file=sys.stderr)
|
|
128
|
+
return 130
|
|
129
|
+
|
|
130
|
+
path = save_token(opts.coordinator, token)
|
|
131
|
+
print(f"\nApproved. This machine is enrolled — credential saved to {path}.")
|
|
132
|
+
print("Start contributing with: flashnode work --runner docker")
|
|
133
|
+
return 0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _logout(args: list[str]) -> int:
|
|
137
|
+
import argparse
|
|
138
|
+
|
|
139
|
+
from flashnode.identity.credentials import clear_token, credentials_path
|
|
140
|
+
|
|
141
|
+
parser = argparse.ArgumentParser(prog="flashnode logout")
|
|
142
|
+
parser.add_argument("--coordinator", required=True, help="FlashRuntime coordinator base URL")
|
|
143
|
+
opts = parser.parse_args(args)
|
|
144
|
+
|
|
145
|
+
removed = clear_token(opts.coordinator)
|
|
146
|
+
if removed:
|
|
147
|
+
print(f"flashnode logout: credential removed from {credentials_path()}", file=sys.stderr)
|
|
148
|
+
else:
|
|
149
|
+
print(f"flashnode logout: no saved credential for {opts.coordinator}", file=sys.stderr)
|
|
150
|
+
return 0
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _work(args: list[str]) -> int:
|
|
154
|
+
import argparse
|
|
155
|
+
import logging
|
|
156
|
+
import os
|
|
157
|
+
import shutil
|
|
158
|
+
import signal
|
|
159
|
+
|
|
160
|
+
from flashnode.executor import CoordinatorClient, ExecutorLoop
|
|
161
|
+
from flashnode.identity.store import load_or_create_node_id
|
|
162
|
+
from flashnode.inventory.capabilities import discover
|
|
163
|
+
|
|
164
|
+
logging.basicConfig(
|
|
165
|
+
level=logging.INFO,
|
|
166
|
+
format='{"ts":"%(asctime)s","level":"%(levelname)s","service":"flashnode","msg":%(message)s}',
|
|
167
|
+
)
|
|
168
|
+
parser = argparse.ArgumentParser(prog="flashnode work")
|
|
169
|
+
parser.add_argument(
|
|
170
|
+
"--coordinator",
|
|
171
|
+
default=os.environ.get("FLASHNODE_COORDINATOR_URL", "http://localhost:8100"),
|
|
172
|
+
help="FlashRuntime coordinator base URL",
|
|
173
|
+
)
|
|
174
|
+
parser.add_argument(
|
|
175
|
+
"--runner",
|
|
176
|
+
choices=["subprocess", "docker", "argv"],
|
|
177
|
+
default=os.environ.get("FLASHNODE_RUNNER", "subprocess"),
|
|
178
|
+
help="task execution tier (docker/argv need the docker CLI on PATH)",
|
|
179
|
+
)
|
|
180
|
+
parser.add_argument("--max-tasks", type=int, default=None)
|
|
181
|
+
parser.add_argument("--poll-seconds", type=float, default=1.0)
|
|
182
|
+
opts = parser.parse_args(args)
|
|
183
|
+
|
|
184
|
+
runner = None
|
|
185
|
+
if opts.runner in ("docker", "argv"):
|
|
186
|
+
# OPTIONAL, and additive. The built-in namespace allowlist
|
|
187
|
+
# (executor/images.py DEFAULT_ALLOWED_IMAGE_PREFIXES) is what a
|
|
188
|
+
# volunteer runs on, so this env var is no longer required and an
|
|
189
|
+
# empty value is no longer a refusal to start.
|
|
190
|
+
#
|
|
191
|
+
# It used to be mandatory, which quietly capped the project at the
|
|
192
|
+
# number of machines whose owners would hand-maintain a list of image
|
|
193
|
+
# references: every image we published stranded every host until its
|
|
194
|
+
# owner edited the variable, so security fixes would reach a fraction
|
|
195
|
+
# of the fleet. Setting it now means "also allow these", for
|
|
196
|
+
# self-hosting the stack or for integration tests.
|
|
197
|
+
images = frozenset(
|
|
198
|
+
i.strip() for i in os.environ.get("FLASHNODE_ALLOWED_IMAGES", "").split(",") if i.strip()
|
|
199
|
+
)
|
|
200
|
+
# Both sandboxed tiers shell out to the `docker` binary directly
|
|
201
|
+
# (subprocess.run(["docker", ...])); if it isn't installed that call
|
|
202
|
+
# raises FileNotFoundError deep inside a task attempt. Check for it
|
|
203
|
+
# here, at startup, rather than let the agent die on the first task.
|
|
204
|
+
if shutil.which("docker") is None:
|
|
205
|
+
print(
|
|
206
|
+
f"flashnode work: --runner {opts.runner} requires the `docker` CLI "
|
|
207
|
+
"on PATH — refusing to start without it",
|
|
208
|
+
file=sys.stderr,
|
|
209
|
+
)
|
|
210
|
+
return 2
|
|
211
|
+
if opts.runner == "docker":
|
|
212
|
+
from flashnode.executor.docker_runner import DockerRunner
|
|
213
|
+
|
|
214
|
+
runner = DockerRunner(allowed_images=images)
|
|
215
|
+
else:
|
|
216
|
+
from flashnode.executor.argv_runner import ArgvDockerRunner
|
|
217
|
+
|
|
218
|
+
runner = ArgvDockerRunner(
|
|
219
|
+
allowed_images=images,
|
|
220
|
+
cpus=float(os.environ.get("FLASHNODE_MAX_CPUS", "2.0")),
|
|
221
|
+
memory_gb=float(os.environ.get("FLASHNODE_MAX_MEMORY_GB", "2.0")),
|
|
222
|
+
timeout_seconds=float(os.environ.get("FLASHNODE_TASK_TIMEOUT_S", "3600")),
|
|
223
|
+
max_output_bytes=int(
|
|
224
|
+
os.environ.get("FLASHNODE_MAX_OUTPUT_BYTES", str(2 * 1024**3))
|
|
225
|
+
),
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
workdir_base = os.environ.get("FLASHNODE_WORKDIR") or None
|
|
229
|
+
|
|
230
|
+
from flashnode.identity.credentials import load_token
|
|
231
|
+
|
|
232
|
+
node_id = load_or_create_node_id()
|
|
233
|
+
client = CoordinatorClient(
|
|
234
|
+
opts.coordinator,
|
|
235
|
+
join_code=os.environ.get("FLASHNODE_JOIN_CODE") or None,
|
|
236
|
+
token=load_token(opts.coordinator),
|
|
237
|
+
)
|
|
238
|
+
registration = discover(
|
|
239
|
+
node_id, kubernetes_node="", node_meta=None,
|
|
240
|
+
argv_capable=(opts.runner == "argv"),
|
|
241
|
+
# An argv-only volunteer has no module runner behind it: advertise
|
|
242
|
+
# module_capable=False so the coordinator's placement gate stops
|
|
243
|
+
# routing "python -m <module>" tasks here (F1) — otherwise those
|
|
244
|
+
# tasks burn every attempt against ArgvDockerRunner's payload
|
|
245
|
+
# rejection before the job ever fails for real.
|
|
246
|
+
module_capable=(opts.runner != "argv"),
|
|
247
|
+
)
|
|
248
|
+
client.register(registration)
|
|
249
|
+
loop = ExecutorLoop(
|
|
250
|
+
client, node_id, runner=runner,
|
|
251
|
+
poll_seconds=opts.poll_seconds, workdir_base=workdir_base,
|
|
252
|
+
registration=registration, # survives coordinator restarts
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
def _stop(signum, frame): # noqa: ARG001
|
|
256
|
+
loop.stop_event.set()
|
|
257
|
+
|
|
258
|
+
signal.signal(signal.SIGTERM, _stop)
|
|
259
|
+
signal.signal(signal.SIGINT, _stop)
|
|
260
|
+
accepted = loop.run(max_tasks=opts.max_tasks)
|
|
261
|
+
print(f"flashnode work: {accepted} task(s) accepted", file=sys.stderr)
|
|
262
|
+
return 0
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def main(argv: list[str] | None = None) -> int:
|
|
266
|
+
args = sys.argv[1:] if argv is None else argv
|
|
267
|
+
if args and args[0] == "agent":
|
|
268
|
+
from flashnode.agent.daemon import main as agent_main
|
|
269
|
+
|
|
270
|
+
return agent_main()
|
|
271
|
+
if args and args[0] == "work":
|
|
272
|
+
return _work(args[1:])
|
|
273
|
+
if args and args[0] == "login":
|
|
274
|
+
return _login(args[1:])
|
|
275
|
+
if args and args[0] == "logout":
|
|
276
|
+
return _logout(args[1:])
|
|
277
|
+
print(USAGE.format(version=__version__), end="")
|
|
278
|
+
if args and args[0] in {"join", "status", "leave"}:
|
|
279
|
+
print(f"\nerror: '{args[0]}' is not implemented yet in this scaffold.", file=sys.stderr)
|
|
280
|
+
return 1
|
|
281
|
+
return 0
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
if __name__ == "__main__":
|
|
285
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""FlashNode agent loop for the Kubernetes profile.
|
|
2
|
+
|
|
3
|
+
One agent per node (DaemonSet). It never launches workload containers, never
|
|
4
|
+
touches a container runtime socket, and only makes *outbound* HTTP calls to
|
|
5
|
+
FlashML Cloud: register once, then heartbeat. On SIGTERM it reports
|
|
6
|
+
`terminating` and exits.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import signal
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
import urllib.error
|
|
18
|
+
import urllib.request
|
|
19
|
+
|
|
20
|
+
from flashruntime.protocol.v1alpha1 import NodeHeartbeat, NodeRegistration
|
|
21
|
+
|
|
22
|
+
from flashnode.agent.kube import get_own_node
|
|
23
|
+
from flashnode.identity.store import load_or_create_node_id
|
|
24
|
+
from flashnode.inventory.capabilities import discover
|
|
25
|
+
|
|
26
|
+
log = logging.getLogger("flashnode")
|
|
27
|
+
logging.basicConfig(
|
|
28
|
+
level=logging.INFO,
|
|
29
|
+
format='{"ts":"%(asctime)s","level":"%(levelname)s","service":"flashnode","msg":%(message)s}',
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _jlog(msg: str, **kv) -> str:
|
|
34
|
+
return json.dumps({"text": msg, **kv})
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _post(url: str, payload: str, timeout: float = 10) -> None:
|
|
38
|
+
req = urllib.request.Request(
|
|
39
|
+
url, data=payload.encode(), headers={"Content-Type": "application/json"}
|
|
40
|
+
)
|
|
41
|
+
with urllib.request.urlopen(req, timeout=timeout):
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Agent:
|
|
46
|
+
def __init__(self) -> None:
|
|
47
|
+
self.cloud_url = os.environ.get("FLASHNODE_CLOUD_URL", "http://localhost:8000").rstrip("/")
|
|
48
|
+
self.heartbeat_seconds = float(os.environ.get("FLASHNODE_HEARTBEAT_SECONDS", "10"))
|
|
49
|
+
self.node_id = load_or_create_node_id()
|
|
50
|
+
self.kubernetes_node = os.environ.get("K8S_NODE_NAME", "")
|
|
51
|
+
self._stop = False
|
|
52
|
+
|
|
53
|
+
def registration(self) -> NodeRegistration:
|
|
54
|
+
node_meta = get_own_node(self.kubernetes_node)
|
|
55
|
+
return discover(self.node_id, self.kubernetes_node, node_meta)
|
|
56
|
+
|
|
57
|
+
def register(self) -> None:
|
|
58
|
+
reg = self.registration()
|
|
59
|
+
backoff = 1.0
|
|
60
|
+
while not self._stop:
|
|
61
|
+
try:
|
|
62
|
+
_post(f"{self.cloud_url}/v1alpha1/nodes/register", reg.model_dump_json())
|
|
63
|
+
log.info(_jlog("registered", node_id=self.node_id,
|
|
64
|
+
k8s_node=self.kubernetes_node,
|
|
65
|
+
pool=reg.pool, arch=reg.capabilities.architecture))
|
|
66
|
+
return
|
|
67
|
+
except (urllib.error.URLError, OSError) as exc:
|
|
68
|
+
log.warning(_jlog("registration failed; retrying",
|
|
69
|
+
error=str(exc), backoff_s=backoff))
|
|
70
|
+
time.sleep(backoff)
|
|
71
|
+
backoff = min(backoff * 2, 30)
|
|
72
|
+
|
|
73
|
+
def heartbeat_once(self, status: str = "online") -> bool:
|
|
74
|
+
hb = NodeHeartbeat(node_id=self.node_id, status=status)
|
|
75
|
+
try:
|
|
76
|
+
_post(f"{self.cloud_url}/v1alpha1/nodes/{self.node_id}/heartbeat",
|
|
77
|
+
hb.model_dump_json())
|
|
78
|
+
return True
|
|
79
|
+
except (urllib.error.URLError, OSError) as exc:
|
|
80
|
+
log.warning(_jlog("heartbeat failed", error=str(exc)))
|
|
81
|
+
return False
|
|
82
|
+
|
|
83
|
+
def _handle_term(self, signum, frame) -> None: # noqa: ARG002
|
|
84
|
+
self._stop = True
|
|
85
|
+
|
|
86
|
+
def run(self) -> int:
|
|
87
|
+
signal.signal(signal.SIGTERM, self._handle_term)
|
|
88
|
+
signal.signal(signal.SIGINT, self._handle_term)
|
|
89
|
+
log.info(_jlog("flashnode agent starting", node_id=self.node_id,
|
|
90
|
+
cloud=self.cloud_url, k8s_node=self.kubernetes_node))
|
|
91
|
+
self.register()
|
|
92
|
+
while not self._stop:
|
|
93
|
+
self.heartbeat_once()
|
|
94
|
+
deadline = time.monotonic() + self.heartbeat_seconds
|
|
95
|
+
while not self._stop and time.monotonic() < deadline:
|
|
96
|
+
time.sleep(0.25)
|
|
97
|
+
# Graceful goodbye — best effort, one attempt.
|
|
98
|
+
self.heartbeat_once(status="terminating")
|
|
99
|
+
log.info(_jlog("flashnode agent stopped", node_id=self.node_id))
|
|
100
|
+
return 0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def main() -> int:
|
|
104
|
+
return Agent().run()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
if __name__ == "__main__":
|
|
108
|
+
sys.exit(main())
|
flashnode/agent/kube.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Minimal in-cluster Kubernetes API access — stdlib only.
|
|
2
|
+
|
|
3
|
+
The agent needs exactly one call (GET its own Node object) so it can report
|
|
4
|
+
allocatable resources and allow-listed labels. A full kubernetes client
|
|
5
|
+
would be attack surface and megabytes of dependencies on a contributor's
|
|
6
|
+
machine; a 30-line REST call is inspectable.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import ssl
|
|
14
|
+
import urllib.request
|
|
15
|
+
|
|
16
|
+
SA_DIR = "/var/run/secrets/kubernetes.io/serviceaccount"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def in_cluster() -> bool:
|
|
20
|
+
return os.path.exists(f"{SA_DIR}/token")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def get_own_node(node_name: str) -> dict | None:
|
|
24
|
+
"""Fetch this agent's Node object; None when not in-cluster or on any
|
|
25
|
+
failure (the agent then degrades to host-level probes)."""
|
|
26
|
+
if not in_cluster() or not node_name:
|
|
27
|
+
return None
|
|
28
|
+
try:
|
|
29
|
+
with open(f"{SA_DIR}/token") as f:
|
|
30
|
+
token = f.read().strip()
|
|
31
|
+
host = os.environ["KUBERNETES_SERVICE_HOST"]
|
|
32
|
+
port = os.environ.get("KUBERNETES_SERVICE_PORT", "443")
|
|
33
|
+
ctx = ssl.create_default_context(cafile=f"{SA_DIR}/ca.crt")
|
|
34
|
+
req = urllib.request.Request(
|
|
35
|
+
f"https://{host}:{port}/api/v1/nodes/{node_name}",
|
|
36
|
+
headers={"Authorization": f"Bearer {token}"},
|
|
37
|
+
)
|
|
38
|
+
with urllib.request.urlopen(req, timeout=10, context=ctx) as resp:
|
|
39
|
+
return json.load(resp)
|
|
40
|
+
except Exception:
|
|
41
|
+
return None
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Admission and workload probes: measured capability, never self-reported.
|
|
2
|
+
|
|
3
|
+
Short benchmarks run at join time (and periodically after) so the
|
|
4
|
+
coordinator's placement/capability decisions rest on measurements, not on
|
|
5
|
+
what a host *claims* (the Vast.ai lesson — master report §9.1: hosts earn
|
|
6
|
+
eligibility through measurement). Results ride the existing
|
|
7
|
+
`NodeRegistration.capabilities`/`labels` — raw numbers each, NEVER a
|
|
8
|
+
single synthetic score (§9.2: a composite hides exactly the bottleneck
|
|
9
|
+
that matters for a given workload).
|
|
10
|
+
|
|
11
|
+
Standard probe names (implement as separate AdmissionProbe subclasses so
|
|
12
|
+
each stays independently budgeted and independently failable):
|
|
13
|
+
cpu_hash_mbps — sustained single-core throughput (sha256 over RAM)
|
|
14
|
+
mem_bandwidth_mbps — large-buffer copy rate
|
|
15
|
+
disk_write_mbps — workdir sequential write (the checkpoint path!)
|
|
16
|
+
net_down_mbps — artifact-download rate from the coordinator
|
|
17
|
+
gpu_* — deferred to the GPU tier (nvml presence first)
|
|
18
|
+
|
|
19
|
+
Budget discipline: probes must respect their `budget_seconds` — admission
|
|
20
|
+
runs while a human waits on `flashnode work` startup; the whole suite gets
|
|
21
|
+
a few seconds, not minutes. Re-probing cadence (daily? on capability
|
|
22
|
+
change?) is a coordinator policy, not the agent's.
|
|
23
|
+
|
|
24
|
+
Status: interface + orchestration complete; concrete probes land with the
|
|
25
|
+
GPU/community tier (HANDBOOK missing-list; flashnode AGENTS §profiles).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import time
|
|
31
|
+
from abc import ABC, abstractmethod
|
|
32
|
+
from typing import ClassVar
|
|
33
|
+
|
|
34
|
+
from pydantic import BaseModel, Field
|
|
35
|
+
|
|
36
|
+
__all__ = ["ProbeResult", "AdmissionProbe", "run_admission"]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ProbeResult(BaseModel):
|
|
40
|
+
"""One raw measurement. `value=None` means the probe failed — recorded
|
|
41
|
+
honestly (with `detail`), never fabricated and never fatal to the
|
|
42
|
+
node's admission (a laptop with a broken disk probe can still serve
|
|
43
|
+
CPU tasks; the *coordinator* decides what a missing number implies)."""
|
|
44
|
+
|
|
45
|
+
name: str
|
|
46
|
+
value: float | None = None
|
|
47
|
+
unit: str = ""
|
|
48
|
+
duration_s: float = Field(ge=0, default=0.0)
|
|
49
|
+
detail: str = ""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class AdmissionProbe(ABC):
|
|
53
|
+
"""One benchmark, one number.
|
|
54
|
+
|
|
55
|
+
Contract: `run` must (a) finish within `budget_seconds` or return its
|
|
56
|
+
best partial measurement, (b) touch only its own scratch space, (c) be
|
|
57
|
+
safe to re-run at any time on a live node — probes may execute while
|
|
58
|
+
tasks run, so they must be gentle (no cache-thrashing full-RAM sweeps).
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
#: Stable name from the standard list above — it becomes the
|
|
62
|
+
#: capability key the planner/scheduler reads; renaming one is a
|
|
63
|
+
#: protocol-level event, treat with the same care as a schema field.
|
|
64
|
+
name: ClassVar[str]
|
|
65
|
+
|
|
66
|
+
@abstractmethod
|
|
67
|
+
def run(self, budget_seconds: float) -> ProbeResult:
|
|
68
|
+
"""Measure and return. Should not raise — but `run_admission`
|
|
69
|
+
guards anyway, because one broken probe must never sink a node's
|
|
70
|
+
whole admission."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def run_admission(
|
|
74
|
+
probes: list[AdmissionProbe], total_budget_seconds: float = 10.0
|
|
75
|
+
) -> dict[str, ProbeResult]:
|
|
76
|
+
"""Run a probe suite under a shared budget; return name → result.
|
|
77
|
+
|
|
78
|
+
Budget is split evenly across remaining probes and re-balanced as
|
|
79
|
+
probes finish early. Failures are captured as `value=None` results
|
|
80
|
+
(isolation rule) so callers always get one entry per probe — the
|
|
81
|
+
difference between "not measured" and "measured badly" stays visible.
|
|
82
|
+
"""
|
|
83
|
+
results: dict[str, ProbeResult] = {}
|
|
84
|
+
remaining = list(probes)
|
|
85
|
+
deadline = time.monotonic() + total_budget_seconds
|
|
86
|
+
while remaining:
|
|
87
|
+
probe = remaining.pop(0)
|
|
88
|
+
per_probe = max(0.1, (deadline - time.monotonic()) / (len(remaining) + 1))
|
|
89
|
+
started = time.monotonic()
|
|
90
|
+
try:
|
|
91
|
+
result = probe.run(budget_seconds=per_probe)
|
|
92
|
+
except Exception as exc: # isolation: a broken probe is a data point
|
|
93
|
+
result = ProbeResult(
|
|
94
|
+
name=probe.name,
|
|
95
|
+
value=None,
|
|
96
|
+
duration_s=time.monotonic() - started,
|
|
97
|
+
detail=f"probe failed: {exc}",
|
|
98
|
+
)
|
|
99
|
+
results[probe.name] = result
|
|
100
|
+
return results
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Host-owner policy: the machine's owner has the last word.
|
|
2
|
+
|
|
3
|
+
`HostPolicy` is the contract between FlashNode and the person whose
|
|
4
|
+
machine it runs on: which resources tasks may use, which code may run,
|
|
5
|
+
and how much of the machine FlashML may occupy. It loads from
|
|
6
|
+
`<state_dir>/policy.json` (the same state dir that holds the node id) so
|
|
7
|
+
the owner edits one file, and `flashnode status` *(planned)* can show
|
|
8
|
+
exactly which limits apply — AGENTS hard rule 4: the owner sees what a
|
|
9
|
+
workload may do before it runs.
|
|
10
|
+
|
|
11
|
+
Safety rules baked into this module:
|
|
12
|
+
- **Conservative defaults**: a policy-less install is the *most*
|
|
13
|
+
restrictive configuration, not the least (1 task at a time, no network
|
|
14
|
+
for tasks, modest quotas). Generosity is always an explicit owner act.
|
|
15
|
+
- **Fail closed on garbage**: an unparseable/invalid policy.json raises —
|
|
16
|
+
the agent refuses to start rather than silently running with defaults
|
|
17
|
+
the owner believes they overrode.
|
|
18
|
+
- **Owner overrides only narrow, never widen, safety**: `allowed_modules`
|
|
19
|
+
/`allowed_images` = None defers to the built-in allowlists; a non-None
|
|
20
|
+
value INTERSECTS with them at enforcement time (an owner can forbid
|
|
21
|
+
more, never permit code the build doesn't trust).
|
|
22
|
+
|
|
23
|
+
Wiring points for the implementer (each a small, separate slice):
|
|
24
|
+
- `agent/cli.py _work`: load_host_policy(state_dir) → cap ExecutorLoop
|
|
25
|
+
concurrency, pass quotas to runners;
|
|
26
|
+
- runners: enforce `max_task_cpus`/`max_task_memory_gb` (docker flags
|
|
27
|
+
already exist; the subprocess tier gains rlimits);
|
|
28
|
+
- executor workdir: enforce `workdir_quota_gb` before download (fixes the
|
|
29
|
+
"no disk-space guard" debt in HANDBOOK §6).
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import json
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
|
|
37
|
+
from pydantic import BaseModel, Field, ValidationError
|
|
38
|
+
|
|
39
|
+
__all__ = ["HostPolicy", "load_host_policy"]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class HostPolicy(BaseModel):
|
|
43
|
+
"""Everything a host owner may restrict. All defaults are deliberately
|
|
44
|
+
tight — see module notes."""
|
|
45
|
+
|
|
46
|
+
max_concurrent_tasks: int = Field(
|
|
47
|
+
default=1, ge=1, description="Parallel task slots this machine offers"
|
|
48
|
+
)
|
|
49
|
+
max_task_cpus: float = Field(
|
|
50
|
+
default=2.0, gt=0, description="CPU cores any single task may use"
|
|
51
|
+
)
|
|
52
|
+
max_task_memory_gb: float = Field(
|
|
53
|
+
default=2.0, gt=0, description="RAM any single task may use"
|
|
54
|
+
)
|
|
55
|
+
workdir_quota_gb: float = Field(
|
|
56
|
+
default=5.0,
|
|
57
|
+
gt=0,
|
|
58
|
+
description="Max disk per task workdir (inputs + outputs + checkpoints)",
|
|
59
|
+
)
|
|
60
|
+
allow_network_tasks: bool = Field(
|
|
61
|
+
default=False,
|
|
62
|
+
description="Whether tasks may have network access. False keeps the "
|
|
63
|
+
"docker tier's --network none and (once enforced) sandboxes the "
|
|
64
|
+
"subprocess tier; flipping this is a real trust decision",
|
|
65
|
+
)
|
|
66
|
+
allowed_modules: list[str] | None = Field(
|
|
67
|
+
default=None,
|
|
68
|
+
description="None = defer to the built-in module allowlist; a list "
|
|
69
|
+
"INTERSECTS with it (owner can only narrow)",
|
|
70
|
+
)
|
|
71
|
+
allowed_images: list[str] | None = Field(
|
|
72
|
+
default=None,
|
|
73
|
+
description="Same narrowing rule as allowed_modules, for the docker tier",
|
|
74
|
+
)
|
|
75
|
+
active_hours: tuple[int, int] | None = Field(
|
|
76
|
+
default=None,
|
|
77
|
+
description="Optional (start_hour, end_hour) local-time window in "
|
|
78
|
+
"which the agent claims work — outside it the loop idles but keeps "
|
|
79
|
+
"heartbeating so the node shows as present-but-unavailable",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def load_host_policy(state_dir: str | Path) -> HostPolicy:
|
|
84
|
+
"""Load `<state_dir>/policy.json`, defaults when absent, loud when broken.
|
|
85
|
+
|
|
86
|
+
Returns: the owner's HostPolicy, or `HostPolicy()` if no file exists
|
|
87
|
+
(defaults ARE the no-file behavior — documented, conservative).
|
|
88
|
+
Raises: ValueError naming policy.json for unparseable JSON or invalid
|
|
89
|
+
fields — never silently substitutes defaults over an owner's broken
|
|
90
|
+
edit (they believed they had set limits; honoring that belief matters
|
|
91
|
+
more than starting up).
|
|
92
|
+
"""
|
|
93
|
+
path = Path(state_dir) / "policy.json"
|
|
94
|
+
if not path.exists():
|
|
95
|
+
return HostPolicy()
|
|
96
|
+
try:
|
|
97
|
+
raw = json.loads(path.read_text())
|
|
98
|
+
return HostPolicy.model_validate(raw)
|
|
99
|
+
except (json.JSONDecodeError, ValidationError) as exc:
|
|
100
|
+
raise ValueError(
|
|
101
|
+
f"invalid host policy at {path}: {exc} — fix or delete the file "
|
|
102
|
+
"(deleting restores the conservative defaults)"
|
|
103
|
+
) from exc
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Task execution: the device profile's pull-based work cycle.
|
|
2
|
+
|
|
3
|
+
`ExecutorLoop` claims leases from a FlashRuntime coordinator (outbound
|
|
4
|
+
only), downloads shared input artifacts, runs the task through a runner,
|
|
5
|
+
uploads outputs, and commits — with attempt heartbeats renewing the lease
|
|
6
|
+
throughout.
|
|
7
|
+
|
|
8
|
+
Runner tiers behind one interface:
|
|
9
|
+
- Tier 1 `SubprocessRunner` (implemented): allowlisted Python modules in a
|
|
10
|
+
fresh subprocess with a wall-clock timeout. Dev/trusted-pool profile —
|
|
11
|
+
not a security boundary.
|
|
12
|
+
- Tier 2 Docker (next): allowlisted images, cpu/memory limits,
|
|
13
|
+
`--network none`, non-root, read-only rootfs. No host Docker socket
|
|
14
|
+
exposure to workloads, no privileged mode. gVisor/Kata tiers later.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from flashnode.executor.client import CoordinatorClient, LeaseLost
|
|
18
|
+
from flashnode.executor.loop import ExecutorLoop
|
|
19
|
+
from flashnode.executor.runner import SubprocessRunner, TaskExecutionError
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"CoordinatorClient",
|
|
23
|
+
"ExecutorLoop",
|
|
24
|
+
"LeaseLost",
|
|
25
|
+
"SubprocessRunner",
|
|
26
|
+
"TaskExecutionError",
|
|
27
|
+
]
|