nvdc 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nvdc-0.2.0 → nvdc-0.2.2}/PKG-INFO +11 -1
- {nvdc-0.2.0 → nvdc-0.2.2}/README.md +10 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/pyproject.toml +1 -1
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/__init__.py +1 -1
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/agent.py +30 -1
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/app.py +23 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/cli.py +14 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/coordinator.py +81 -12
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/keys.py +16 -3
- nvdc-0.2.2/src/nvdc/models.py +100 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/protocol.py +1 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/runtime.py +62 -1
- nvdc-0.2.2/src/nvdc/update.py +53 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/web/index.html +13 -4
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc.egg-info/PKG-INFO +11 -1
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc.egg-info/SOURCES.txt +4 -0
- nvdc-0.2.2/tests/test_model_attestation.py +195 -0
- nvdc-0.2.2/tests/test_self_update.py +79 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/setup.cfg +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/accounts.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/attestation.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/catalog.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/config.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/gpu.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/hardware.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/house.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/inference.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/payments.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/storage.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc/wallet.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc.egg-info/dependency_links.txt +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc.egg-info/entry_points.txt +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc.egg-info/requires.txt +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/src/nvdc.egg-info/top_level.txt +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_accounts_auth.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_coordinator_security.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_house_node.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_persistence.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_self_serve.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_split_pricing.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_storage_money.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_stripe_webhook.py +0 -0
- {nvdc-0.2.0 → nvdc-0.2.2}/tests/test_version_gate.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nvdc
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Bring your GPU onto the network: one command turns a GPU into a verifiable, OpenAI-compatible inference node.
|
|
5
5
|
Author: NVDC
|
|
6
6
|
License: Apache-2.0
|
|
@@ -150,6 +150,16 @@ Attestation is a first-class, pluggable component (`nvdc/attestation.py`):
|
|
|
150
150
|
A coordinator can enforce policy with `--require-attested` to only route work to
|
|
151
151
|
nodes whose attestation verifies.
|
|
152
152
|
|
|
153
|
+
**Model identity** is attested separately from hardware. Every node publishes a
|
|
154
|
+
content-addressed commitment of the artifact it serves (the Ollama manifest
|
|
155
|
+
digest) and signs it into each work receipt (`nvdc-work-v2`). The coordinator
|
|
156
|
+
independently checks the claim against `registry.ollama.ai` and surfaces a
|
|
157
|
+
verdict (`verified` / `mismatch` / `unknown`) per node and per receipt;
|
|
158
|
+
`--require-verified-models` restricts routing to verified artifacts. Without a
|
|
159
|
+
TEE this cannot *prove* which bytes ran — what it guarantees is that a node
|
|
160
|
+
claiming one model while serving another has signed a non-repudiable false
|
|
161
|
+
statement that contradicts public ground truth.
|
|
162
|
+
|
|
153
163
|
> Note: the DGX Spark / GB10 cannot produce hardware attestation (NVIDIA disabled
|
|
154
164
|
> CC on this SKU). It serves inference fine; it just joins as an unattested node.
|
|
155
165
|
|
|
@@ -128,6 +128,16 @@ Attestation is a first-class, pluggable component (`nvdc/attestation.py`):
|
|
|
128
128
|
A coordinator can enforce policy with `--require-attested` to only route work to
|
|
129
129
|
nodes whose attestation verifies.
|
|
130
130
|
|
|
131
|
+
**Model identity** is attested separately from hardware. Every node publishes a
|
|
132
|
+
content-addressed commitment of the artifact it serves (the Ollama manifest
|
|
133
|
+
digest) and signs it into each work receipt (`nvdc-work-v2`). The coordinator
|
|
134
|
+
independently checks the claim against `registry.ollama.ai` and surfaces a
|
|
135
|
+
verdict (`verified` / `mismatch` / `unknown`) per node and per receipt;
|
|
136
|
+
`--require-verified-models` restricts routing to verified artifacts. Without a
|
|
137
|
+
TEE this cannot *prove* which bytes ran — what it guarantees is that a node
|
|
138
|
+
claiming one model while serving another has signed a non-repudiable false
|
|
139
|
+
statement that contradicts public ground truth.
|
|
140
|
+
|
|
131
141
|
> Note: the DGX Spark / GB10 cannot produce hardware attestation (NVIDIA disabled
|
|
132
142
|
> CC on this SKU). It serves inference fine; it just joins as an unattested node.
|
|
133
143
|
|
|
@@ -21,6 +21,7 @@ from .gpu import detect_gpu, detect_gpus, detect_interconnect
|
|
|
21
21
|
from .hardware import detect_hardware, machine_id as hw_machine_id
|
|
22
22
|
from .inference import Backend, make_backend
|
|
23
23
|
from .keys import Identity
|
|
24
|
+
from .models import local_ollama_commit
|
|
24
25
|
|
|
25
26
|
log = logging.getLogger("nvdc.agent")
|
|
26
27
|
|
|
@@ -68,6 +69,7 @@ class NodeAgent:
|
|
|
68
69
|
machine_id: str = "",
|
|
69
70
|
cluster: str = "",
|
|
70
71
|
private: bool = False,
|
|
72
|
+
auto_update: bool = False,
|
|
71
73
|
):
|
|
72
74
|
# A node commits to exactly ONE hot-loaded model at a time — the
|
|
73
75
|
# "mining algorithm" it has chosen. It advertises and serves only this
|
|
@@ -85,6 +87,8 @@ class NodeAgent:
|
|
|
85
87
|
self.machine_id = machine_id or hw_machine_id()
|
|
86
88
|
self.cluster = cluster
|
|
87
89
|
self.private = private
|
|
90
|
+
self.auto_update = auto_update
|
|
91
|
+
self.model_commit = "" # set at registration from the backend's store
|
|
88
92
|
self.token = token
|
|
89
93
|
self.require_attestation = require_attestation
|
|
90
94
|
self.status_cb = status_cb
|
|
@@ -120,6 +124,16 @@ class NodeAgent:
|
|
|
120
124
|
if code == protocol.CLOSE_UPGRADE_REQUIRED:
|
|
121
125
|
reason = getattr(e.rcvd, "reason", "") \
|
|
122
126
|
or "coordinator requires a newer agent; run: pipx upgrade nvdc"
|
|
127
|
+
if self.auto_update:
|
|
128
|
+
# Self-heal: install the new release from PyPI and
|
|
129
|
+
# restart this process; on success exec never returns.
|
|
130
|
+
log.warning("upgrade required — auto-updating: %s", reason)
|
|
131
|
+
self._emit("upgrade_required", detail=reason + " (auto-updating)")
|
|
132
|
+
from .update import restart_process, self_update
|
|
133
|
+
ok, detail = await asyncio.to_thread(self_update)
|
|
134
|
+
if ok:
|
|
135
|
+
restart_process()
|
|
136
|
+
log.error("auto-update failed: %s", detail[-300:])
|
|
123
137
|
log.error("not joining network — %s", reason)
|
|
124
138
|
self._emit("upgrade_required", detail=reason)
|
|
125
139
|
self._stopped = True
|
|
@@ -199,6 +213,17 @@ class NodeAgent:
|
|
|
199
213
|
raise RuntimeError(
|
|
200
214
|
f"attestation required but not verified: {att.reason or att.mode}"
|
|
201
215
|
)
|
|
216
|
+
# Model identity commitment: the content digest of the artifact this
|
|
217
|
+
# node will serve. Signed into every work receipt (nvdc-work-v2) and
|
|
218
|
+
# checked by the coordinator against the public registry.
|
|
219
|
+
if getattr(self.backend, "name", "") == "ollama":
|
|
220
|
+
self.model_commit = await local_ollama_commit(
|
|
221
|
+
self.model, getattr(self.backend, "base_url", "http://localhost:11434"))
|
|
222
|
+
if self.model_commit:
|
|
223
|
+
log.info("model commitment: %s -> %s", self.model, self.model_commit[:23] + "…")
|
|
224
|
+
else:
|
|
225
|
+
log.warning("could not derive a model commitment for '%s'; "
|
|
226
|
+
"receipts will not carry a model identity", self.model)
|
|
202
227
|
profile = protocol.NodeProfile(
|
|
203
228
|
name=self.name,
|
|
204
229
|
models=[self.model],
|
|
@@ -217,6 +242,7 @@ class NodeAgent:
|
|
|
217
242
|
machine_id=self.machine_id,
|
|
218
243
|
cluster=self.cluster,
|
|
219
244
|
private=self.private,
|
|
245
|
+
model_commit=self.model_commit,
|
|
220
246
|
agent_version=__version__,
|
|
221
247
|
)
|
|
222
248
|
await self._send(protocol.MSG_REGISTER, profile=protocol.node_profile_to_dict(profile))
|
|
@@ -312,7 +338,8 @@ class NodeAgent:
|
|
|
312
338
|
|
|
313
339
|
def _sign_work(self, req_id, prompt_commit, tokens, response_commit, delivery) -> str:
|
|
314
340
|
payload = keys.work_payload(req_id, self.model, prompt_commit, tokens,
|
|
315
|
-
response_commit, delivery
|
|
341
|
+
response_commit, delivery,
|
|
342
|
+
model_commit=self.model_commit)
|
|
316
343
|
return self.identity.sign(payload)
|
|
317
344
|
|
|
318
345
|
async def _send(self, msg_type: str, **fields: Any):
|
|
@@ -336,6 +363,7 @@ async def serve(
|
|
|
336
363
|
owner_account: str = "",
|
|
337
364
|
cluster: str = "",
|
|
338
365
|
private: bool = False,
|
|
366
|
+
auto_update: bool = False,
|
|
339
367
|
):
|
|
340
368
|
if not model:
|
|
341
369
|
raise ValueError("a single --model must be specified; a node serves exactly one model")
|
|
@@ -362,5 +390,6 @@ async def serve(
|
|
|
362
390
|
owner_account=owner_account,
|
|
363
391
|
cluster=cluster,
|
|
364
392
|
private=private,
|
|
393
|
+
auto_update=auto_update,
|
|
365
394
|
)
|
|
366
395
|
await agent.run_forever()
|
|
@@ -7,6 +7,7 @@ load one into memory (hot), and only then enables "Go Live" on the network.
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
import asyncio
|
|
10
11
|
import hashlib
|
|
11
12
|
import json as _json
|
|
12
13
|
import logging
|
|
@@ -85,6 +86,28 @@ def create_app(runtime: NodeRuntime) -> FastAPI:
|
|
|
85
86
|
ok, msg = runtime.go_offline()
|
|
86
87
|
return {"ok": ok, "message": msg}
|
|
87
88
|
|
|
89
|
+
def _same_origin(request: Request) -> bool:
|
|
90
|
+
"""The update endpoint is for the local UI only. CORS here is wide
|
|
91
|
+
open (so nvdc.ai can read state), which means without this check any
|
|
92
|
+
website could trigger an update/restart of this client. Browsers send
|
|
93
|
+
an Origin header on POSTs; require it to match this server."""
|
|
94
|
+
origin = request.headers.get("origin", "")
|
|
95
|
+
if not origin:
|
|
96
|
+
return True # curl / local scripts
|
|
97
|
+
return origin.split("://", 1)[-1] == request.headers.get("host", "")
|
|
98
|
+
|
|
99
|
+
@app.post("/api/update")
|
|
100
|
+
async def update(request: Request):
|
|
101
|
+
"""One-click self-update: drain, install from PyPI, restart in place."""
|
|
102
|
+
if not _same_origin(request):
|
|
103
|
+
return JSONResponse({"ok": False, "error": "forbidden"}, status_code=403)
|
|
104
|
+
ok, msg = await runtime.apply_update()
|
|
105
|
+
return {"ok": ok, "message": msg}
|
|
106
|
+
|
|
107
|
+
@app.on_event("startup")
|
|
108
|
+
async def _start_auto_update_loop():
|
|
109
|
+
asyncio.create_task(runtime.auto_update_loop())
|
|
110
|
+
|
|
88
111
|
# ---- consumer proxies to the coordinator --------------------------------
|
|
89
112
|
# Keep the UI same-origin (no CORS) by proxying the coordinator's public API
|
|
90
113
|
# through this local app. These power the Home / Chat / Network tabs.
|
|
@@ -67,6 +67,7 @@ def cmd_serve(args: argparse.Namespace) -> int:
|
|
|
67
67
|
owner_account=args.owner or os.environ.get("NVDC_OWNER", ""),
|
|
68
68
|
cluster=args.cluster,
|
|
69
69
|
private=args.private,
|
|
70
|
+
auto_update=args.auto_update or bool(os.environ.get("NVDC_AUTO_UPDATE")),
|
|
70
71
|
)
|
|
71
72
|
)
|
|
72
73
|
except KeyboardInterrupt:
|
|
@@ -91,6 +92,7 @@ def cmd_coordinator(args: argparse.Namespace) -> int:
|
|
|
91
92
|
require_payment=not args.no_payment,
|
|
92
93
|
min_agent_version=args.min_agent_version
|
|
93
94
|
or os.environ.get("NVDC_MIN_AGENT_VERSION", ""),
|
|
95
|
+
require_verified_models=args.require_verified_models or None,
|
|
94
96
|
)
|
|
95
97
|
print(f"nvdc coordinator on http://{args.host}:{args.port}")
|
|
96
98
|
print(f" OpenAI endpoint : http://{args.host}:{args.port}/v1/chat/completions")
|
|
@@ -116,6 +118,7 @@ def cmd_app(args: argparse.Namespace) -> int:
|
|
|
116
118
|
drain_timeout=args.drain_timeout,
|
|
117
119
|
owner_account=args.owner or os.environ.get("NVDC_OWNER", ""),
|
|
118
120
|
cluster=args.cluster,
|
|
121
|
+
auto_update=args.auto_update or bool(os.environ.get("NVDC_AUTO_UPDATE")),
|
|
119
122
|
)
|
|
120
123
|
app = create_app(runtime)
|
|
121
124
|
url = f"http://{args.host}:{args.port}"
|
|
@@ -178,6 +181,10 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
178
181
|
s.add_argument("--drain-timeout", type=float, default=120.0,
|
|
179
182
|
help="max seconds to wait for in-flight requests when leaving "
|
|
180
183
|
"before force-failing stuck ones (default: 120)")
|
|
184
|
+
s.add_argument("--auto-update", action="store_true",
|
|
185
|
+
help="when the coordinator requires a newer agent, install "
|
|
186
|
+
"the update from PyPI and restart automatically "
|
|
187
|
+
"(or NVDC_AUTO_UPDATE=1)")
|
|
181
188
|
s.set_defaults(func=cmd_serve)
|
|
182
189
|
|
|
183
190
|
c = sub.add_parser("coordinator", help="run the public network hub")
|
|
@@ -193,6 +200,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
193
200
|
c.add_argument("--min-agent-version", default="",
|
|
194
201
|
help="refuse agents older than this version at registration "
|
|
195
202
|
"(or NVDC_MIN_AGENT_VERSION); empty = no gate")
|
|
203
|
+
c.add_argument("--require-verified-models", action="store_true",
|
|
204
|
+
help="only route to nodes whose claimed model digest matches "
|
|
205
|
+
"the public registry (or NVDC_REQUIRE_VERIFIED_MODELS)")
|
|
196
206
|
c.set_defaults(func=cmd_coordinator)
|
|
197
207
|
|
|
198
208
|
a = sub.add_parser("app", help="launch the visual node client (web UI)")
|
|
@@ -213,6 +223,10 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
213
223
|
help="max seconds to wait for in-flight requests when leaving "
|
|
214
224
|
"before force-failing stuck ones (default: 120)")
|
|
215
225
|
a.add_argument("--no-browser", action="store_true", help="don't auto-open the browser")
|
|
226
|
+
a.add_argument("--auto-update", action="store_true",
|
|
227
|
+
help="install new releases from PyPI and restart automatically "
|
|
228
|
+
"when the network announces one (or NVDC_AUTO_UPDATE=1); "
|
|
229
|
+
"default is a one-click prompt in the UI")
|
|
216
230
|
a.set_defaults(func=cmd_app)
|
|
217
231
|
|
|
218
232
|
st = sub.add_parser("status", help="show local GPU + attestation status")
|
|
@@ -30,6 +30,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
|
|
30
30
|
from starlette.concurrency import run_in_threadpool
|
|
31
31
|
|
|
32
32
|
from . import __version__, accounts, keys, payments, protocol
|
|
33
|
+
from . import models as model_identity
|
|
33
34
|
from .keys import Identity
|
|
34
35
|
from .protocol import version_tuple
|
|
35
36
|
from .storage import Account, Storage, make_storage, to_micro, to_usd
|
|
@@ -140,10 +141,14 @@ def _verify_node(r: Dict[str, Any]) -> bool:
|
|
|
140
141
|
sig = r.get("node_sig", "")
|
|
141
142
|
if not sig:
|
|
142
143
|
return False
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
144
|
+
args = (r.get("req_id", ""), r.get("model", ""), r.get("prompt_commit", ""),
|
|
145
|
+
r.get("tokens", 0), r.get("response_commit", ""), r.get("delivery", ""))
|
|
146
|
+
account = r.get("node_account", "") or r.get("miner_account", "")
|
|
147
|
+
# v2 (signature covers the model artifact digest), then legacy v1.
|
|
148
|
+
if r.get("model_commit"):
|
|
149
|
+
if keys.verify(account, keys.work_payload(*args, model_commit=r["model_commit"]), sig):
|
|
150
|
+
return True
|
|
151
|
+
return keys.verify(account, keys.work_payload(*args), sig)
|
|
147
152
|
|
|
148
153
|
|
|
149
154
|
def build_clusters(machines: List[Dict[str, Any]], live_ids: set) -> List[Dict[str, Any]]:
|
|
@@ -181,6 +186,11 @@ class Node:
|
|
|
181
186
|
self.owner_account = profile.get("owner_account", "") or self.account_id
|
|
182
187
|
self.machine_id = profile.get("machine_id", "") or self.id
|
|
183
188
|
self.cluster = profile.get("cluster", "") or ""
|
|
189
|
+
# Model identity: the node's claimed artifact digest, and the verdict
|
|
190
|
+
# from checking that claim against the public registry at registration
|
|
191
|
+
# ("verified" | "mismatch" | "unknown").
|
|
192
|
+
self.model_commit = str(profile.get("model_commit", "") or "")
|
|
193
|
+
self.model_verdict = "unknown"
|
|
184
194
|
self.draining = False
|
|
185
195
|
self.pending: Dict[str, PendingRequest] = {}
|
|
186
196
|
self._send_lock = asyncio.Lock()
|
|
@@ -255,6 +265,8 @@ class Node:
|
|
|
255
265
|
"price_per_mtok": round(self.price_per_mtok, 6),
|
|
256
266
|
"price_per_mtok_input": round(self.price_per_mtok_input, 6),
|
|
257
267
|
"private": self.private,
|
|
268
|
+
"model_commit": self.model_commit,
|
|
269
|
+
"model_verdict": self.model_verdict,
|
|
258
270
|
"attested": self.attested,
|
|
259
271
|
"attestation_mode": self.attestation.get("mode", "none"),
|
|
260
272
|
"agent_version": self.profile.get("agent_version", ""),
|
|
@@ -339,7 +351,8 @@ class Registry:
|
|
|
339
351
|
|
|
340
352
|
def pick(self, model: str = "", models: Optional[List[str]] = None,
|
|
341
353
|
require_attested: bool = False, pin: str = "", min_rating: int = 0,
|
|
342
|
-
max_price: Optional[float] = None, consumer: str = ""
|
|
354
|
+
max_price: Optional[float] = None, consumer: str = "",
|
|
355
|
+
require_verified: bool = False) -> Optional[Node]:
|
|
343
356
|
def model_ok(n: Node) -> bool:
|
|
344
357
|
if model:
|
|
345
358
|
return model in n.models
|
|
@@ -357,6 +370,8 @@ class Registry:
|
|
|
357
370
|
# Private nodes serve only their owner (personal AI mode).
|
|
358
371
|
and (not n.private or own(n))
|
|
359
372
|
and (not require_attested or n.attested)
|
|
373
|
+
# Strict networks only route to registry-verified model artifacts.
|
|
374
|
+
and (not require_verified or n.model_verdict == "verified")
|
|
360
375
|
and n.rating() >= min_rating
|
|
361
376
|
# A max price never excludes your own node — it's free for you.
|
|
362
377
|
and (max_price is None or own(n)
|
|
@@ -455,11 +470,18 @@ class Registry:
|
|
|
455
470
|
|
|
456
471
|
def create_app(auth_token: str = "", require_attested: bool = False,
|
|
457
472
|
require_payment: bool = True,
|
|
458
|
-
min_agent_version: Optional[str] = None
|
|
473
|
+
min_agent_version: Optional[str] = None,
|
|
474
|
+
require_verified_models: Optional[bool] = None) -> FastAPI:
|
|
459
475
|
# Minimum agent version allowed to JOIN the network (older agents are
|
|
460
476
|
# refused at the door with a clear upgrade message). Empty = no gate.
|
|
461
477
|
if min_agent_version is None:
|
|
462
478
|
min_agent_version = os.environ.get("NVDC_MIN_AGENT_VERSION", "").strip()
|
|
479
|
+
# Strict model-identity policy: only route to nodes whose claimed model
|
|
480
|
+
# digest matches the public registry. Default off — verdicts are always
|
|
481
|
+
# computed and surfaced either way.
|
|
482
|
+
if require_verified_models is None:
|
|
483
|
+
require_verified_models = bool(
|
|
484
|
+
os.environ.get("NVDC_REQUIRE_VERIFIED_MODELS", "").strip())
|
|
463
485
|
app = FastAPI(title="nvdc coordinator")
|
|
464
486
|
# Allow browser clients (e.g. the Vercel static site) to read the network
|
|
465
487
|
# and stream chat. Inference billing identifies accounts by header, not
|
|
@@ -489,6 +511,34 @@ def create_app(auth_token: str = "", require_attested: bool = False,
|
|
|
489
511
|
node.restore_stats(rec)
|
|
490
512
|
return
|
|
491
513
|
|
|
514
|
+
# Registry ground truth for model digests, cached so registration doesn't
|
|
515
|
+
# hammer registry.ollama.ai (tags are mutable, hence the TTL refresh).
|
|
516
|
+
_model_digest_cache: Dict[str, tuple] = {}
|
|
517
|
+
|
|
518
|
+
async def expected_model_digest(model: str) -> str:
|
|
519
|
+
hit = _model_digest_cache.get(model)
|
|
520
|
+
if hit and time.time() - hit[1] < 6 * 3600:
|
|
521
|
+
return hit[0]
|
|
522
|
+
digest = await model_identity.fetch_registry_digest(model)
|
|
523
|
+
_model_digest_cache[model] = (digest, time.time())
|
|
524
|
+
return digest
|
|
525
|
+
|
|
526
|
+
async def _verify_node_model(node: Node) -> None:
|
|
527
|
+
"""Check the node's claimed artifact digest against the registry."""
|
|
528
|
+
if not node.model_commit:
|
|
529
|
+
return # no claim made -> verdict stays "unknown"
|
|
530
|
+
try:
|
|
531
|
+
expected = await expected_model_digest(node.model)
|
|
532
|
+
except Exception:
|
|
533
|
+
expected = ""
|
|
534
|
+
node.model_verdict = model_identity.model_verdict(node.model_commit, expected)
|
|
535
|
+
if node.model_verdict == "mismatch":
|
|
536
|
+
log.warning(
|
|
537
|
+
"model commitment MISMATCH: node '%s' claims %s for '%s' but the "
|
|
538
|
+
"registry says %s", node.profile.get("name"),
|
|
539
|
+
node.model_commit[:19] + "…", node.model,
|
|
540
|
+
(expected[:19] + "…") if expected else "?")
|
|
541
|
+
|
|
492
542
|
# House node: a coordinator-hosted miner backed by the Cerebras API, so
|
|
493
543
|
# the network always has at least one live node. Created only here from
|
|
494
544
|
# the coordinator's own environment — the public node protocol has no way
|
|
@@ -543,6 +593,7 @@ def create_app(auth_token: str = "", require_attested: bool = False,
|
|
|
543
593
|
node_id = uuid.uuid4().hex[:12]
|
|
544
594
|
node = Node(node_id, ws, first.get("profile", {}))
|
|
545
595
|
_rehydrate_stats(node) # a reconnect is not a brand-new miner
|
|
596
|
+
await _verify_node_model(node) # registry-anchored model identity
|
|
546
597
|
registry.add(node)
|
|
547
598
|
# Persist this machine under its owner so the account can list its
|
|
548
599
|
# clusters/machines even after the node disconnects.
|
|
@@ -691,7 +742,8 @@ def create_app(auth_token: str = "", require_attested: bool = False,
|
|
|
691
742
|
node = registry.pick(model, models=models_filter or None,
|
|
692
743
|
require_attested=require_attested, pin=pin,
|
|
693
744
|
min_rating=min_rating, max_price=max_price,
|
|
694
|
-
consumer=consumer_id
|
|
745
|
+
consumer=consumer_id,
|
|
746
|
+
require_verified=require_verified_models)
|
|
695
747
|
if node is None:
|
|
696
748
|
why = f"no available miner for model '{model or ','.join(models_filter)}'"
|
|
697
749
|
if pin:
|
|
@@ -774,6 +826,7 @@ def create_app(auth_token: str = "", require_attested: bool = False,
|
|
|
774
826
|
"max_price": round(max_price, 8) if max_price is not None else -1,
|
|
775
827
|
"nonce": nonce, "auth_ts": auth_ts, "hold_u": hold_u,
|
|
776
828
|
"price_in": price_in, "self_serve": self_serve,
|
|
829
|
+
"model_commit": node.model_commit, "model_verdict": node.model_verdict,
|
|
777
830
|
"consumer_sig": consumer_sig, "consumer_sig_valid": consumer_sig_valid,
|
|
778
831
|
}
|
|
779
832
|
base_headers = {
|
|
@@ -801,13 +854,26 @@ def create_app(auth_token: str = "", require_attested: bool = False,
|
|
|
801
854
|
# Refresh the persisted machine record so lifetime stats (TPS, rating
|
|
802
855
|
# inputs, earnings) survive reconnects and coordinator restarts.
|
|
803
856
|
ledger.record_machine(node.owner_account, node.machine_record())
|
|
804
|
-
# Verify the node's completion signature over what it delivered.
|
|
857
|
+
# Verify the node's completion signature over what it delivered. A
|
|
858
|
+
# node that registered a model commitment signs nvdc-work-v2 (the
|
|
859
|
+
# signature also covers the artifact digest); older agents sign v1.
|
|
860
|
+
# The v1 fallback keeps legacy receipts verifiable but records that
|
|
861
|
+
# the model identity was NOT covered by the signature.
|
|
805
862
|
node_sig_valid = False
|
|
863
|
+
model_commit_signed = False
|
|
806
864
|
if node_sig:
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
865
|
+
mc = ctx.get("model_commit", "")
|
|
866
|
+
if mc:
|
|
867
|
+
work2 = keys.work_payload(req_id, ctx["model"], ctx["prompt_commit"],
|
|
868
|
+
tokens, response_commit, delivery,
|
|
869
|
+
model_commit=mc)
|
|
870
|
+
# The machine signs with its own key; earnings credit the owner.
|
|
871
|
+
node_sig_valid = keys.verify(ctx["node_account"], work2, node_sig)
|
|
872
|
+
model_commit_signed = node_sig_valid
|
|
873
|
+
if not node_sig_valid:
|
|
874
|
+
work = keys.work_payload(req_id, ctx["model"], ctx["prompt_commit"],
|
|
875
|
+
tokens, response_commit, delivery)
|
|
876
|
+
node_sig_valid = keys.verify(ctx["node_account"], work, node_sig)
|
|
811
877
|
receipt = ledger.settle(ctx["consumer_id"], ctx["miner_account"], cost, {
|
|
812
878
|
"req_id": req_id, "model": ctx["model"], "req_model": ctx["req_model"],
|
|
813
879
|
"miner_node": ctx["node_id"], "node_account": ctx["node_account"], "tokens": tokens,
|
|
@@ -815,6 +881,9 @@ def create_app(auth_token: str = "", require_attested: bool = False,
|
|
|
815
881
|
"price_per_mtok": round(price, 6),
|
|
816
882
|
"price_per_mtok_input": round(ctx["price_in"], 6),
|
|
817
883
|
"self_serve": ctx["self_serve"],
|
|
884
|
+
"model_commit": ctx.get("model_commit", ""),
|
|
885
|
+
"model_commit_signed": model_commit_signed,
|
|
886
|
+
"model_verdict": ctx.get("model_verdict", "unknown"),
|
|
818
887
|
"delivery": delivery,
|
|
819
888
|
"attested": ctx["attested"], "prompt_commit": ctx["prompt_commit"],
|
|
820
889
|
"response_commit": response_commit,
|
|
@@ -44,13 +44,26 @@ def auth_payload(consumer: str, model: str, prompt_commit: str,
|
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
def work_payload(req_id: str, model: str, prompt_commit: str, tokens: int,
|
|
47
|
-
response_commit: str, delivery: str
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
response_commit: str, delivery: str,
|
|
48
|
+
model_commit: str = "") -> Dict[str, Any]:
|
|
49
|
+
"""What the NODE signs at the end: attestation of work delivered.
|
|
50
|
+
|
|
51
|
+
When the node knows the content digest of the artifact it served
|
|
52
|
+
(``model_commit``), the payload upgrades to v2 and the signature also
|
|
53
|
+
covers the model's identity — claiming one model while serving another
|
|
54
|
+
becomes a signed, non-repudiable false statement. With no commitment the
|
|
55
|
+
payload is byte-identical to v1, so existing nodes and receipts verify
|
|
56
|
+
unchanged.
|
|
57
|
+
"""
|
|
58
|
+
payload = {
|
|
50
59
|
"kind": "nvdc-work-v1", "req_id": req_id, "model": model,
|
|
51
60
|
"prompt_commit": prompt_commit, "tokens": int(tokens),
|
|
52
61
|
"response_commit": response_commit, "delivery": delivery,
|
|
53
62
|
}
|
|
63
|
+
if model_commit:
|
|
64
|
+
payload["kind"] = "nvdc-work-v2"
|
|
65
|
+
payload["model_commit"] = model_commit
|
|
66
|
+
return payload
|
|
54
67
|
|
|
55
68
|
|
|
56
69
|
def account_from_pubkey(pub: Ed25519PublicKey) -> str:
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Model identity: content-addressed commitments for served models.
|
|
2
|
+
|
|
3
|
+
Hardware attestation says WHERE work ran; this module is about WHAT was run.
|
|
4
|
+
Without a TEE we cannot cryptographically prove which bytes sat in GPU memory,
|
|
5
|
+
so the goal is different and honest: every node publishes a content-addressed
|
|
6
|
+
commitment (digest) of the artifact it serves, signs it into each work receipt
|
|
7
|
+
(``nvdc-work-v2`` in keys.py), and the coordinator independently checks the
|
|
8
|
+
claim against the same public registry nodes pull from. A node can still lie —
|
|
9
|
+
but the lie is now a signed, non-repudiable statement that contradicts public
|
|
10
|
+
ground truth, so it is detectable, attributable and punishable.
|
|
11
|
+
|
|
12
|
+
Ollama's model store is already content-addressed: every model resolves to an
|
|
13
|
+
OCI-style manifest whose sha256 digest uniquely identifies weights + config.
|
|
14
|
+
``registry.ollama.ai`` serves those manifests publicly, which gives the
|
|
15
|
+
coordinator an authoritative expected digest for every catalog tag.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import hashlib
|
|
21
|
+
import logging
|
|
22
|
+
from typing import Any, Dict
|
|
23
|
+
|
|
24
|
+
import httpx
|
|
25
|
+
|
|
26
|
+
log = logging.getLogger("nvdc.models")
|
|
27
|
+
|
|
28
|
+
OLLAMA_REGISTRY = "https://registry.ollama.ai"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _norm_digest(d: str) -> str:
|
|
32
|
+
d = str(d or "").strip()
|
|
33
|
+
if not d:
|
|
34
|
+
return ""
|
|
35
|
+
return d if d.startswith("sha256:") else "sha256:" + d
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _pick_digest(tags_json: Dict[str, Any], model: str) -> str:
|
|
39
|
+
"""Pull the digest of ``model`` out of an Ollama ``/api/tags`` response."""
|
|
40
|
+
want = model if ":" in model else model + ":latest"
|
|
41
|
+
for m in tags_json.get("models", []):
|
|
42
|
+
if m.get("name") == want or m.get("model") == want:
|
|
43
|
+
return _norm_digest(m.get("digest", ""))
|
|
44
|
+
return ""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def local_ollama_commit(model: str, ollama_url: str) -> str:
|
|
48
|
+
"""The content digest of the locally installed model, or "" if unknown.
|
|
49
|
+
|
|
50
|
+
Reported by the same daemon that will execute the inference — i.e. the
|
|
51
|
+
runtime's own claim of identity, which the node then signs.
|
|
52
|
+
"""
|
|
53
|
+
try:
|
|
54
|
+
async with httpx.AsyncClient(timeout=10) as c:
|
|
55
|
+
r = await c.get(f"{ollama_url.rstrip('/')}/api/tags")
|
|
56
|
+
r.raise_for_status()
|
|
57
|
+
return _pick_digest(r.json(), model)
|
|
58
|
+
except Exception as e:
|
|
59
|
+
log.debug("could not read local model digest for %s: %s", model, e)
|
|
60
|
+
return ""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def fetch_registry_digest(model: str) -> str:
|
|
64
|
+
"""Authoritative digest for a library tag from registry.ollama.ai.
|
|
65
|
+
|
|
66
|
+
Returns "" when unavailable (custom/private models, network failure) —
|
|
67
|
+
callers must treat that as "unknown", never as a mismatch.
|
|
68
|
+
"""
|
|
69
|
+
if "/" in model: # only library/* tags have a public canonical digest
|
|
70
|
+
return ""
|
|
71
|
+
name, _, tag = model.partition(":")
|
|
72
|
+
if not name:
|
|
73
|
+
return ""
|
|
74
|
+
url = f"{OLLAMA_REGISTRY}/v2/library/{name}/manifests/{tag or 'latest'}"
|
|
75
|
+
headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"}
|
|
76
|
+
try:
|
|
77
|
+
async with httpx.AsyncClient(timeout=8, follow_redirects=True) as c:
|
|
78
|
+
r = await c.get(url, headers=headers)
|
|
79
|
+
if r.status_code != 200:
|
|
80
|
+
return ""
|
|
81
|
+
dcd = r.headers.get("docker-content-digest", "")
|
|
82
|
+
if dcd:
|
|
83
|
+
return _norm_digest(dcd)
|
|
84
|
+
# OCI content addressing: the digest IS the sha256 of the bytes.
|
|
85
|
+
return "sha256:" + hashlib.sha256(r.content).hexdigest()
|
|
86
|
+
except Exception as e:
|
|
87
|
+
log.debug("registry digest fetch failed for %s: %s", model, e)
|
|
88
|
+
return ""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def model_verdict(claimed: str, expected: str) -> str:
|
|
92
|
+
"""Compare a node's claimed digest with registry ground truth.
|
|
93
|
+
|
|
94
|
+
'unknown' whenever either side is missing — absence of evidence is not a
|
|
95
|
+
mismatch. 'mismatch' only when both digests exist and differ.
|
|
96
|
+
"""
|
|
97
|
+
claimed, expected = _norm_digest(claimed), _norm_digest(expected)
|
|
98
|
+
if not claimed or not expected:
|
|
99
|
+
return "unknown"
|
|
100
|
+
return "verified" if claimed == expected else "mismatch"
|
|
@@ -95,6 +95,7 @@ class NodeProfile:
|
|
|
95
95
|
machine_id: str = "" # stable per-machine id (survives reconnects)
|
|
96
96
|
cluster: str = "" # optional cluster label grouping machines
|
|
97
97
|
private: bool = False # personal mode: serve only the owner account
|
|
98
|
+
model_commit: str = "" # content digest of the served model artifact
|
|
98
99
|
agent_version: str = ""
|
|
99
100
|
node_id: Optional[str] = None
|
|
100
101
|
|
|
@@ -54,11 +54,18 @@ class NetworkState:
|
|
|
54
54
|
detail: str = ""
|
|
55
55
|
|
|
56
56
|
|
|
57
|
+
@dataclass
|
|
58
|
+
class UpdateState:
|
|
59
|
+
status: str = "idle" # idle | updating | restarting | error
|
|
60
|
+
detail: str = ""
|
|
61
|
+
|
|
62
|
+
|
|
57
63
|
class NodeRuntime:
|
|
58
64
|
def __init__(self, name: str, mock: bool = False,
|
|
59
65
|
coordinator: str = DEFAULT_COORDINATOR_WS,
|
|
60
66
|
token: str = "", drain_timeout: float = 120.0,
|
|
61
|
-
owner_account: str = "", cluster: str = ""
|
|
67
|
+
owner_account: str = "", cluster: str = "",
|
|
68
|
+
auto_update: bool = False):
|
|
62
69
|
self.name = name
|
|
63
70
|
self.mock = mock
|
|
64
71
|
self.drain_timeout = drain_timeout
|
|
@@ -73,6 +80,8 @@ class NodeRuntime:
|
|
|
73
80
|
self.active_model = ""
|
|
74
81
|
self.price_per_mtok = 0.0 # miner's minimum USD per 1M tokens
|
|
75
82
|
self.private = False # personal mode: serve only the owner account
|
|
83
|
+
self.auto_update = auto_update
|
|
84
|
+
self.update = UpdateState()
|
|
76
85
|
self.wallet = Wallet() # consumer-side USD wallet
|
|
77
86
|
# Earnings owner account: defaults to this machine's own wallet.
|
|
78
87
|
self.owner_account = owner_account or self.wallet.account_id
|
|
@@ -158,6 +167,8 @@ class NodeRuntime:
|
|
|
158
167
|
return {
|
|
159
168
|
"name": self.name,
|
|
160
169
|
"version": __version__,
|
|
170
|
+
"auto_update": self.auto_update,
|
|
171
|
+
"update": asdict(self.update),
|
|
161
172
|
"mock": self.mock,
|
|
162
173
|
"hardware": hw,
|
|
163
174
|
"attestation": asdict(self.attestation),
|
|
@@ -364,6 +375,7 @@ class NodeRuntime:
|
|
|
364
375
|
machine_id=self.machine_id,
|
|
365
376
|
cluster=self.cluster,
|
|
366
377
|
private=self.private,
|
|
378
|
+
auto_update=self.auto_update,
|
|
367
379
|
)
|
|
368
380
|
self._agent = agent
|
|
369
381
|
self._agent_task = asyncio.create_task(agent.run_forever())
|
|
@@ -390,6 +402,55 @@ class NodeRuntime:
|
|
|
390
402
|
asyncio.create_task(self._agent.drain())
|
|
391
403
|
return True, "draining"
|
|
392
404
|
|
|
405
|
+
# ---- self-update ---------------------------------------------------------
|
|
406
|
+
async def apply_update(self):
|
|
407
|
+
"""One-click upgrade: drain off the network, pip-upgrade from PyPI,
|
|
408
|
+
then restart this process in place."""
|
|
409
|
+
if self.update.status in ("updating", "restarting"):
|
|
410
|
+
return False, "update already in progress"
|
|
411
|
+
from .update import restart_process, self_update
|
|
412
|
+
self.update = UpdateState(status="updating", detail="installing from PyPI…")
|
|
413
|
+
if self._agent is not None: # leave gracefully; in-flight work finishes
|
|
414
|
+
try:
|
|
415
|
+
await asyncio.wait_for(self._agent.drain(),
|
|
416
|
+
timeout=self.drain_timeout + 10)
|
|
417
|
+
except Exception:
|
|
418
|
+
pass
|
|
419
|
+
ok, detail = await asyncio.to_thread(self_update)
|
|
420
|
+
if not ok:
|
|
421
|
+
log.error("self-update failed: %s", detail)
|
|
422
|
+
self.update = UpdateState(status="error", detail=detail[-300:])
|
|
423
|
+
return False, "update failed"
|
|
424
|
+
self.update = UpdateState(status="restarting", detail="restarting client…")
|
|
425
|
+
# Give the HTTP response a moment to flush before exec replaces us.
|
|
426
|
+
asyncio.get_running_loop().call_later(0.7, restart_process)
|
|
427
|
+
return True, "restarting"
|
|
428
|
+
|
|
429
|
+
async def auto_update_loop(self):
|
|
430
|
+
"""Opt-in (--auto-update): periodically compare our version with the
|
|
431
|
+
coordinator's announced release and self-update when behind. The
|
|
432
|
+
upgrade always comes from PyPI — the network only provides the hint."""
|
|
433
|
+
from .protocol import version_tuple
|
|
434
|
+
delay = 20.0 # first check shortly after boot, then every 6 hours
|
|
435
|
+
while True:
|
|
436
|
+
await asyncio.sleep(delay)
|
|
437
|
+
delay = 6 * 3600
|
|
438
|
+
if not self.auto_update or self.update.status != "idle":
|
|
439
|
+
continue
|
|
440
|
+
base = self.coordinator_http
|
|
441
|
+
if not base:
|
|
442
|
+
continue
|
|
443
|
+
try:
|
|
444
|
+
async with httpx.AsyncClient(timeout=10) as c:
|
|
445
|
+
info = (await c.get(base + "/api/version")).json()
|
|
446
|
+
except Exception:
|
|
447
|
+
continue
|
|
448
|
+
target = str(info.get("coordinator", "") or "")
|
|
449
|
+
if version_tuple(target) > version_tuple(__version__):
|
|
450
|
+
log.info("auto-update: %s -> %s announced by coordinator",
|
|
451
|
+
__version__, target)
|
|
452
|
+
await self.apply_update()
|
|
453
|
+
|
|
393
454
|
def _on_agent_status(self, status: str, info: dict):
|
|
394
455
|
self.network.status = status
|
|
395
456
|
if status == "live":
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Self-update: upgrade the nvdc package in place and restart the process.
|
|
2
|
+
|
|
3
|
+
Trust model — the network only ever signals THAT a newer version exists. The
|
|
4
|
+
upgrade itself always installs from PyPI through this environment's own
|
|
5
|
+
package manager (``python -m pip install --upgrade nvdc``), never from a URL
|
|
6
|
+
supplied by the coordinator, so enabling auto-update grants the network no
|
|
7
|
+
more power than running ``pipx upgrade nvdc`` yourself. Auto mode is opt-in
|
|
8
|
+
(``--auto-update`` / ``NVDC_AUTO_UPDATE=1``); the default experience is a
|
|
9
|
+
one-click "Update & restart" prompt in the client UI.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
import os
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
18
|
+
from typing import Tuple
|
|
19
|
+
|
|
20
|
+
log = logging.getLogger("nvdc.update")
|
|
21
|
+
|
|
22
|
+
PACKAGE = "nvdc"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def update_command() -> list:
|
|
26
|
+
"""The exact command used to upgrade this installation. Pinned to the
|
|
27
|
+
current interpreter's environment, which is the pipx venv for installer/
|
|
28
|
+
pipx installs and the active venv/site for pip installs."""
|
|
29
|
+
return [sys.executable, "-m", "pip", "install", "--upgrade", PACKAGE]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def self_update(timeout: float = 600) -> Tuple[bool, str]:
|
|
33
|
+
"""Upgrade the package inside the current environment. Returns (ok, detail)."""
|
|
34
|
+
cmd = update_command()
|
|
35
|
+
log.info("self-update: %s", " ".join(cmd))
|
|
36
|
+
try:
|
|
37
|
+
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
|
38
|
+
except Exception as e: # pip missing, timeout, etc.
|
|
39
|
+
return False, f"updater failed to run: {e}"
|
|
40
|
+
out = ((proc.stdout or "") + (proc.stderr or "")).strip()
|
|
41
|
+
if proc.returncode != 0:
|
|
42
|
+
return False, out[-1500:] or f"pip exited {proc.returncode}"
|
|
43
|
+
return True, out[-1500:]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def restart_process() -> None:
|
|
47
|
+
"""Replace this process with a fresh one running the same command line.
|
|
48
|
+
|
|
49
|
+
Python opens sockets close-on-exec, so the listening port is freed and
|
|
50
|
+
the restarted process can rebind it. Never returns on success.
|
|
51
|
+
"""
|
|
52
|
+
log.info("restarting: %s %s", sys.executable, " ".join(sys.argv))
|
|
53
|
+
os.execv(sys.executable, [sys.executable] + sys.argv)
|
|
@@ -708,11 +708,20 @@ let VERSION_INFO = null;
|
|
|
708
708
|
function vparts(v){ return String(v||"").split("+")[0].split(".").map(x=>parseInt(x,10)||0); }
|
|
709
709
|
function vOlder(a,b){ const x=vparts(a),y=vparts(b); for(let i=0;i<Math.max(x.length,y.length);i++){ const d=(x[i]||0)-(y[i]||0); if(d) return d<0; } return false; }
|
|
710
710
|
function renderUpdateBanner(s){
|
|
711
|
-
const b=$("updateBanner"); if(!b||!s
|
|
711
|
+
const b=$("updateBanner"); if(!b||!s) return;
|
|
712
|
+
const up=s.update||{};
|
|
713
|
+
if(up.status==="updating"){ b.style.display="block"; b.textContent="Updating client — "+(up.detail||"installing from PyPI…"); return; }
|
|
714
|
+
if(up.status==="restarting"){ b.style.display="block"; b.textContent="Update installed — restarting… this page will reconnect automatically."; return; }
|
|
715
|
+
if(!VERSION_INFO) return;
|
|
712
716
|
const mine=s.version||"", min=VERSION_INFO.min_agent_version||"", cur=VERSION_INFO.coordinator||"";
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
else
|
|
717
|
+
let msg="";
|
|
718
|
+
if(min && vOlder(mine,min)) msg="Upgrade required — this client (v"+mine+") can no longer join the network (minimum v"+min+").";
|
|
719
|
+
else if(cur && vOlder(mine,cur)) msg="Update available — v"+cur+" is out (you run v"+mine+").";
|
|
720
|
+
if(up.status==="error") msg=(msg||"Update")+" Last attempt failed: "+String(up.detail||"see logs").replace(/[<>]/g,"")+".";
|
|
721
|
+
if(!msg){ b.style.display="none"; return; }
|
|
722
|
+
b.style.display="block";
|
|
723
|
+
b.innerHTML=msg.replace(/[<>]/g,"")+' <button class="btn" id="updateBtn" style="margin-left:10px">Update & restart</button>'+(s.auto_update?' <span style="opacity:.7">(auto-update is on)</span>':'');
|
|
724
|
+
$("updateBtn").onclick=async()=>{ $("updateBtn").disabled=true; $("updateBtn").textContent="Updating…"; try{ await fetch("/api/update",{method:"POST"}); }catch(e){} };
|
|
716
725
|
}
|
|
717
726
|
|
|
718
727
|
async function tick(){
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nvdc
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Bring your GPU onto the network: one command turns a GPU into a verifiable, OpenAI-compatible inference node.
|
|
5
5
|
Author: NVDC
|
|
6
6
|
License: Apache-2.0
|
|
@@ -150,6 +150,16 @@ Attestation is a first-class, pluggable component (`nvdc/attestation.py`):
|
|
|
150
150
|
A coordinator can enforce policy with `--require-attested` to only route work to
|
|
151
151
|
nodes whose attestation verifies.
|
|
152
152
|
|
|
153
|
+
**Model identity** is attested separately from hardware. Every node publishes a
|
|
154
|
+
content-addressed commitment of the artifact it serves (the Ollama manifest
|
|
155
|
+
digest) and signs it into each work receipt (`nvdc-work-v2`). The coordinator
|
|
156
|
+
independently checks the claim against `registry.ollama.ai` and surfaces a
|
|
157
|
+
verdict (`verified` / `mismatch` / `unknown`) per node and per receipt;
|
|
158
|
+
`--require-verified-models` restricts routing to verified artifacts. Without a
|
|
159
|
+
TEE this cannot *prove* which bytes ran — what it guarantees is that a node
|
|
160
|
+
claiming one model while serving another has signed a non-repudiable false
|
|
161
|
+
statement that contradicts public ground truth.
|
|
162
|
+
|
|
153
163
|
> Note: the DGX Spark / GB10 cannot produce hardware attestation (NVIDIA disabled
|
|
154
164
|
> CC on this SKU). It serves inference fine; it just joins as an unattested node.
|
|
155
165
|
|
|
@@ -14,10 +14,12 @@ src/nvdc/hardware.py
|
|
|
14
14
|
src/nvdc/house.py
|
|
15
15
|
src/nvdc/inference.py
|
|
16
16
|
src/nvdc/keys.py
|
|
17
|
+
src/nvdc/models.py
|
|
17
18
|
src/nvdc/payments.py
|
|
18
19
|
src/nvdc/protocol.py
|
|
19
20
|
src/nvdc/runtime.py
|
|
20
21
|
src/nvdc/storage.py
|
|
22
|
+
src/nvdc/update.py
|
|
21
23
|
src/nvdc/wallet.py
|
|
22
24
|
src/nvdc.egg-info/PKG-INFO
|
|
23
25
|
src/nvdc.egg-info/SOURCES.txt
|
|
@@ -29,8 +31,10 @@ src/nvdc/web/index.html
|
|
|
29
31
|
tests/test_accounts_auth.py
|
|
30
32
|
tests/test_coordinator_security.py
|
|
31
33
|
tests/test_house_node.py
|
|
34
|
+
tests/test_model_attestation.py
|
|
32
35
|
tests/test_persistence.py
|
|
33
36
|
tests/test_self_serve.py
|
|
37
|
+
tests/test_self_update.py
|
|
34
38
|
tests/test_split_pricing.py
|
|
35
39
|
tests/test_storage_money.py
|
|
36
40
|
tests/test_stripe_webhook.py
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Model identity: signed commitments, registry verdicts, routing policy."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
from starlette.testclient import TestClient
|
|
8
|
+
|
|
9
|
+
from nvdc import keys, models, protocol
|
|
10
|
+
from nvdc.coordinator import Node, Registry, create_app
|
|
11
|
+
from nvdc.models import _pick_digest, model_verdict
|
|
12
|
+
|
|
13
|
+
DIG = "sha256:" + "a" * 64
|
|
14
|
+
OTHER = "sha256:" + "b" * 64
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# ---- pure pieces -------------------------------------------------------------
|
|
18
|
+
def test_work_payload_v1_is_byte_identical_without_commitment():
|
|
19
|
+
p = keys.work_payload("r1", "m1", "pc", 5, "rc", "complete")
|
|
20
|
+
assert p == {"kind": "nvdc-work-v1", "req_id": "r1", "model": "m1",
|
|
21
|
+
"prompt_commit": "pc", "tokens": 5, "response_commit": "rc",
|
|
22
|
+
"delivery": "complete"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_work_payload_v2_signature_covers_model_identity(identity):
|
|
26
|
+
v2 = keys.work_payload("r1", "m1", "pc", 5, "rc", "complete", model_commit=DIG)
|
|
27
|
+
assert v2["kind"] == "nvdc-work-v2" and v2["model_commit"] == DIG
|
|
28
|
+
sig = identity.sign(v2)
|
|
29
|
+
assert keys.verify(identity.account_id, v2, sig)
|
|
30
|
+
# The same signature must NOT verify a payload claiming a different model.
|
|
31
|
+
forged = keys.work_payload("r1", "m1", "pc", 5, "rc", "complete", model_commit=OTHER)
|
|
32
|
+
assert not keys.verify(identity.account_id, forged, sig)
|
|
33
|
+
# ...nor strip the commitment and pass as v1.
|
|
34
|
+
v1 = keys.work_payload("r1", "m1", "pc", 5, "rc", "complete")
|
|
35
|
+
assert not keys.verify(identity.account_id, v1, sig)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_verdict_logic():
|
|
39
|
+
assert model_verdict("", DIG) == "unknown" # no claim
|
|
40
|
+
assert model_verdict(DIG, "") == "unknown" # no ground truth
|
|
41
|
+
assert model_verdict(DIG, DIG) == "verified"
|
|
42
|
+
assert model_verdict("a" * 64, DIG) == "verified" # prefix normalization
|
|
43
|
+
assert model_verdict(DIG, OTHER) == "mismatch"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_pick_digest_from_ollama_tags():
|
|
47
|
+
tags = {"models": [
|
|
48
|
+
{"name": "llama3.1:8b", "digest": "a" * 64},
|
|
49
|
+
{"name": "mistral:latest", "digest": "sha256:" + "c" * 64},
|
|
50
|
+
]}
|
|
51
|
+
assert _pick_digest(tags, "llama3.1:8b") == DIG
|
|
52
|
+
assert _pick_digest(tags, "mistral") == "sha256:" + "c" * 64
|
|
53
|
+
assert _pick_digest(tags, "missing:1b") == ""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# ---- registration verdicts ---------------------------------------------------
|
|
57
|
+
def _register(client, commit, model="llama3.1:8b"):
|
|
58
|
+
ws = client.websocket_connect("/node/ws")
|
|
59
|
+
ws.__enter__()
|
|
60
|
+
ws.send_text(protocol.encode(protocol.MSG_REGISTER, profile={
|
|
61
|
+
"name": "gpu", "account_id": "nvdc_machinekey",
|
|
62
|
+
"owner_account": "nvdc_owner", "machine_id": "m1",
|
|
63
|
+
"models": [model], "price_per_mtok": 1.0,
|
|
64
|
+
"model_commit": commit, "agent_version": "0.2.2",
|
|
65
|
+
"attestation": {"verified": False}}))
|
|
66
|
+
ack = json.loads(ws.receive_text())
|
|
67
|
+
return ws, ack
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@pytest.mark.parametrize("claimed,expected,verdict", [
|
|
71
|
+
(DIG, DIG, "verified"),
|
|
72
|
+
(OTHER, DIG, "mismatch"),
|
|
73
|
+
("", DIG, "unknown"), # legacy agent: no claim, never penalized
|
|
74
|
+
])
|
|
75
|
+
def test_registration_checks_claim_against_registry(monkeypatch, claimed, expected, verdict):
|
|
76
|
+
async def fake_registry(model):
|
|
77
|
+
return expected
|
|
78
|
+
monkeypatch.setattr(models, "fetch_registry_digest", fake_registry)
|
|
79
|
+
app = create_app(require_payment=False)
|
|
80
|
+
client = TestClient(app)
|
|
81
|
+
ws, ack = _register(client, claimed)
|
|
82
|
+
node = app.state.registry.nodes[ack["node_id"]]
|
|
83
|
+
assert node.model_verdict == verdict
|
|
84
|
+
d = node.to_dict()
|
|
85
|
+
assert d["model_verdict"] == verdict and d["model_commit"] == claimed
|
|
86
|
+
ws.__exit__(None, None, None)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_registry_unreachable_is_unknown_not_mismatch(monkeypatch):
|
|
90
|
+
async def broken(model):
|
|
91
|
+
raise RuntimeError("registry down")
|
|
92
|
+
monkeypatch.setattr(models, "fetch_registry_digest", broken)
|
|
93
|
+
app = create_app(require_payment=False)
|
|
94
|
+
ws, ack = _register(TestClient(app), DIG)
|
|
95
|
+
assert app.state.registry.nodes[ack["node_id"]].model_verdict == "unknown"
|
|
96
|
+
ws.__exit__(None, None, None)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# ---- routing policy ----------------------------------------------------------
|
|
100
|
+
def _node(verdict, node_id="n1"):
|
|
101
|
+
n = Node(node_id, None, {"name": node_id, "models": ["m1"],
|
|
102
|
+
"price_per_mtok": 1.0})
|
|
103
|
+
n.model_verdict = verdict
|
|
104
|
+
return n
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_strict_mode_routes_only_to_verified():
|
|
108
|
+
reg = Registry()
|
|
109
|
+
reg.add(_node("mismatch", "bad"))
|
|
110
|
+
reg.add(_node("unknown", "meh"))
|
|
111
|
+
assert reg.pick("m1", require_verified=True) is None
|
|
112
|
+
ok = _node("verified", "good")
|
|
113
|
+
reg.add(ok)
|
|
114
|
+
assert reg.pick("m1", require_verified=True) is ok
|
|
115
|
+
# Default policy: verdicts are informational, everyone routable.
|
|
116
|
+
assert reg.pick("m1") is not None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# ---- receipts ----------------------------------------------------------------
|
|
120
|
+
class _SigningNode(Node):
|
|
121
|
+
"""In-process node that signs its work like a real v2 agent."""
|
|
122
|
+
|
|
123
|
+
def __init__(self, identity, commit):
|
|
124
|
+
super().__init__("s1", None, {
|
|
125
|
+
"name": "signer", "account_id": identity.account_id,
|
|
126
|
+
"owner_account": "nvdc_sowner", "machine_id": "sm1",
|
|
127
|
+
"models": ["m1"], "price_per_mtok": 1.0, "model_commit": commit,
|
|
128
|
+
"attestation": {"verified": False}})
|
|
129
|
+
self._identity = identity
|
|
130
|
+
self.model_verdict = "verified"
|
|
131
|
+
|
|
132
|
+
async def send(self, msg_type, **fields):
|
|
133
|
+
if msg_type != protocol.MSG_INFER:
|
|
134
|
+
return
|
|
135
|
+
req_id, body = fields["id"], fields["body"]
|
|
136
|
+
pc = hashlib.sha256(json.dumps(body.get("messages", []),
|
|
137
|
+
sort_keys=True).encode()).hexdigest()
|
|
138
|
+
rc = hashlib.sha256(b"ok").hexdigest()
|
|
139
|
+
sig = self._identity.sign(keys.work_payload(
|
|
140
|
+
req_id, "m1", pc, 7, rc, "complete", model_commit=self.model_commit))
|
|
141
|
+
self.pending[req_id].queue.put_nowait({
|
|
142
|
+
"t": protocol.MSG_RESULT, "id": req_id,
|
|
143
|
+
"body": {"choices": [{"message": {"role": "assistant", "content": "ok"}}],
|
|
144
|
+
"usage": {"prompt_tokens": 3, "completion_tokens": 7}},
|
|
145
|
+
"tokens": 7, "prompt_tokens": 3, "response_commit": rc, "sig": sig})
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def test_receipt_carries_signed_model_commitment(identity):
|
|
149
|
+
app = create_app(require_payment=False)
|
|
150
|
+
app.state.registry.add(_SigningNode(identity, DIG))
|
|
151
|
+
client = TestClient(app)
|
|
152
|
+
r = client.post("/v1/chat/completions",
|
|
153
|
+
headers={"X-NVDC-Account": "nvdc_consumer"},
|
|
154
|
+
json={"model": "m1", "messages": [{"role": "user", "content": "hi"}]})
|
|
155
|
+
assert r.status_code == 200
|
|
156
|
+
receipt = app.state.ledger.recent(1)[0]
|
|
157
|
+
assert receipt["model_commit"] == DIG
|
|
158
|
+
assert receipt["model_commit_signed"] is True
|
|
159
|
+
assert receipt["node_sig_valid"] is True
|
|
160
|
+
assert receipt["model_verdict"] == "verified"
|
|
161
|
+
# /api/verify independently re-verifies the v2 node signature.
|
|
162
|
+
verdict = client.post("/api/verify", json=receipt).json()
|
|
163
|
+
assert verdict["node"] is True
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_legacy_v1_nodes_still_verify_but_commitment_unsigned(identity):
|
|
167
|
+
class _LegacyNode(_SigningNode):
|
|
168
|
+
def __init__(self, ident):
|
|
169
|
+
super().__init__(ident, commit="") # pre-0.2.2: no commitment
|
|
170
|
+
|
|
171
|
+
async def send(self, msg_type, **fields):
|
|
172
|
+
if msg_type != protocol.MSG_INFER:
|
|
173
|
+
return
|
|
174
|
+
req_id, body = fields["id"], fields["body"]
|
|
175
|
+
pc = hashlib.sha256(json.dumps(body.get("messages", []),
|
|
176
|
+
sort_keys=True).encode()).hexdigest()
|
|
177
|
+
rc = hashlib.sha256(b"ok").hexdigest()
|
|
178
|
+
sig = self._identity.sign(keys.work_payload(req_id, "m1", pc, 7, rc, "complete"))
|
|
179
|
+
self.pending[req_id].queue.put_nowait({
|
|
180
|
+
"t": protocol.MSG_RESULT, "id": req_id,
|
|
181
|
+
"body": {"choices": [{"message": {"role": "assistant", "content": "ok"}}],
|
|
182
|
+
"usage": {"prompt_tokens": 3, "completion_tokens": 7}},
|
|
183
|
+
"tokens": 7, "prompt_tokens": 3, "response_commit": rc, "sig": sig})
|
|
184
|
+
|
|
185
|
+
app = create_app(require_payment=False)
|
|
186
|
+
app.state.registry.add(_LegacyNode(identity))
|
|
187
|
+
client = TestClient(app)
|
|
188
|
+
r = client.post("/v1/chat/completions",
|
|
189
|
+
headers={"X-NVDC-Account": "nvdc_consumer"},
|
|
190
|
+
json={"model": "m1", "messages": [{"role": "user", "content": "hi"}]})
|
|
191
|
+
assert r.status_code == 200
|
|
192
|
+
receipt = app.state.ledger.recent(1)[0]
|
|
193
|
+
assert receipt["node_sig_valid"] is True # legacy sigs still verify
|
|
194
|
+
assert receipt["model_commit_signed"] is False # but identity not covered
|
|
195
|
+
assert receipt["model_commit"] == ""
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Self-update plumbing: PyPI-anchored command, endpoint guard, states."""
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
from starlette.testclient import TestClient
|
|
8
|
+
|
|
9
|
+
from nvdc import update
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_update_command_is_pinned_to_own_environment():
|
|
13
|
+
cmd = update.update_command()
|
|
14
|
+
assert cmd[0] == sys.executable # the pipx/venv interpreter
|
|
15
|
+
assert cmd[1:] == ["-m", "pip", "install", "--upgrade", "nvdc"]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_self_update_success_and_failure(monkeypatch):
|
|
19
|
+
calls = []
|
|
20
|
+
|
|
21
|
+
def fake_run(cmd, **kw):
|
|
22
|
+
calls.append(cmd)
|
|
23
|
+
return subprocess.CompletedProcess(cmd, 0, stdout="Successfully installed nvdc-9.9.9", stderr="")
|
|
24
|
+
|
|
25
|
+
monkeypatch.setattr(update.subprocess, "run", fake_run)
|
|
26
|
+
ok, detail = update.self_update()
|
|
27
|
+
assert ok and "9.9.9" in detail
|
|
28
|
+
assert calls[0] == update.update_command()
|
|
29
|
+
|
|
30
|
+
monkeypatch.setattr(update.subprocess, "run",
|
|
31
|
+
lambda cmd, **kw: subprocess.CompletedProcess(cmd, 1, "", "boom"))
|
|
32
|
+
ok, detail = update.self_update()
|
|
33
|
+
assert not ok and "boom" in detail
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@pytest.fixture()
|
|
37
|
+
def client_app(monkeypatch, tmp_path):
|
|
38
|
+
monkeypatch.setenv("HOME", str(tmp_path)) # keep the wallet out of ~/.nvdc
|
|
39
|
+
from nvdc.app import create_app
|
|
40
|
+
from nvdc.runtime import NodeRuntime
|
|
41
|
+
runtime = NodeRuntime(name="t", mock=True)
|
|
42
|
+
triggered = []
|
|
43
|
+
|
|
44
|
+
async def fake_apply():
|
|
45
|
+
triggered.append(True)
|
|
46
|
+
return True, "restarting"
|
|
47
|
+
|
|
48
|
+
monkeypatch.setattr(runtime, "apply_update", fake_apply)
|
|
49
|
+
return TestClient(create_app(runtime)), triggered
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_update_endpoint_rejects_cross_origin(client_app):
|
|
53
|
+
client, triggered = client_app
|
|
54
|
+
r = client.post("/api/update", headers={"origin": "https://evil.example",
|
|
55
|
+
"host": "localhost:8765"})
|
|
56
|
+
assert r.status_code == 403
|
|
57
|
+
assert triggered == []
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_update_endpoint_allows_own_ui_and_local_tools(client_app):
|
|
61
|
+
client, triggered = client_app
|
|
62
|
+
# Same-origin browser POST (Origin matches Host).
|
|
63
|
+
r = client.post("/api/update", headers={"origin": "http://localhost:8765",
|
|
64
|
+
"host": "localhost:8765"})
|
|
65
|
+
assert r.status_code == 200 and r.json()["ok"] is True
|
|
66
|
+
# Local curl (no Origin header at all).
|
|
67
|
+
r = client.post("/api/update")
|
|
68
|
+
assert r.status_code == 200
|
|
69
|
+
assert len(triggered) == 2
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_snapshot_carries_update_state(monkeypatch, tmp_path):
|
|
73
|
+
monkeypatch.setenv("HOME", str(tmp_path))
|
|
74
|
+
from nvdc.runtime import NodeRuntime, UpdateState
|
|
75
|
+
rt = NodeRuntime(name="t", mock=True, auto_update=True)
|
|
76
|
+
rt.update = UpdateState(status="updating", detail="installing")
|
|
77
|
+
snap = rt.snapshot()
|
|
78
|
+
assert snap["auto_update"] is True
|
|
79
|
+
assert snap["update"] == {"status": "updating", "detail": "installing"}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|