nvdc 0.1.0__py3-none-any.whl

nvdc/__init__.py

@@ -0,0 +1,3 @@
+"""nvdc: bring your GPU onto the network as a verifiable, OpenAI-compatible inference node."""
+
+__version__ = "0.1.0"

nvdc/agent.py

@@ -0,0 +1,329 @@
+"""Node agent: the thing `nvdc serve` runs.
+
+Opens ONE outbound WebSocket to the coordinator (so the node never needs an
+inbound public port and its IP stays private), registers its GPU + attestation
+profile, then services inference requests over that tunnel.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json as _json
+import logging
+from typing import Any, Dict, Optional
+
+import websockets
+
+from . import __version__, keys, protocol
+from .attestation import attest
+from .gpu import detect_gpu, detect_gpus, detect_interconnect
+from .hardware import detect_hardware, machine_id as hw_machine_id
+from .inference import Backend, make_backend
+from .keys import Identity
+
+log = logging.getLogger("nvdc.agent")
+
+
+def _extract_content(sse_line: str) -> str:
+    """Pull the delta content out of one OpenAI SSE 'data: {...}' line."""
+    line = sse_line.strip()
+    if not line.startswith("data:"):
+        return ""
+    try:
+        obj = _json.loads(line[len("data:"):].strip())
+        return obj["choices"][0]["delta"].get("content") or ""
+    except Exception:
+        return ""
+
+
+class NodeAgent:
+    def __init__(
+        self,
+        coordinator_url: str,
+        name: str,
+        backend: Backend,
+        model: str,
+        token: str = "",
+        require_attestation: bool = False,
+        status_cb=None,
+        drain_timeout: float = 120.0,
+        price_per_mtok: float = 0.0,
+        account_id: str = "",
+        identity: Optional[Identity] = None,
+        owner_account: str = "",
+        machine_id: str = "",
+        cluster: str = "",
+    ):
+        # A node commits to exactly ONE hot-loaded model at a time — the
+        # "mining algorithm" it has chosen. It advertises and serves only this
+        # model; requests for anything else are rejected at the node boundary.
+        self.coordinator_url = coordinator_url
+        self.name = name
+        self.backend = backend
+        self.model = model
+        self.price_per_mtok = price_per_mtok
+        self.identity = identity or Identity()
+        self.account_id = account_id or self.identity.account_id
+        # Earnings credit the owner account; a single machine owns itself.
+        self.owner_account = owner_account or self.account_id
+        self.machine_id = machine_id or hw_machine_id()
+        self.cluster = cluster
+        self.token = token
+        self.require_attestation = require_attestation
+        self.status_cb = status_cb
+        self.drain_timeout = drain_timeout
+        self._ws = None
+        self._send_lock = asyncio.Lock()
+        # graceful drain bookkeeping
+        self._stopped = False
+        self._draining = False
+        self._inflight_ids = set()  # request ids currently being served
+        self._inflight_zero = asyncio.Event()
+        self._inflight_zero.set()  # starts idle
+
+    def _emit(self, status: str, **info):
+        if self.status_cb:
+            try:
+                self.status_cb(status, info)
+            except Exception:
+                log.debug("status_cb error", exc_info=True)
+
+    async def run_forever(self):
+        backoff = 1
+        while not self._stopped:
+            try:
+                await self._connect_and_serve()
+                backoff = 1
+            except (OSError, websockets.WebSocketException) as e:
+                if self._stopped:
+                    break
+                log.warning("connection lost (%s); reconnecting in %ss", e, backoff)
+                self._emit("connecting", detail=str(e))
+                await asyncio.sleep(backoff)
+                backoff = min(backoff * 2, 30)
+
+    async def drain(self):
+        """Gracefully leave: tell the coordinator to stop routing new work,
+        let in-flight requests finish, then disconnect. In-flight responses are
+        never interrupted, so the node's delivery/completion score is preserved.
+        """
+        if self._draining:
+            return
+        self._draining = True
+        log.info("draining: %d request(s) in flight", len(self._inflight_ids))
+        self._emit("draining", inflight=len(self._inflight_ids))
+        try:
+            await self._send(protocol.MSG_DRAIN)  # coordinator stops routing now
+        except Exception:
+            pass
+        try:
+            # block until all in-flight complete, but not forever
+            await asyncio.wait_for(self._inflight_zero.wait(), timeout=self.drain_timeout)
+            log.info("drain complete; disconnecting")
+        except asyncio.TimeoutError:
+            stuck = list(self._inflight_ids)
+            log.warning(
+                "drain timeout after %ss; force-failing %d stuck request(s): %s",
+                self.drain_timeout, len(stuck), stuck,
+            )
+            # Fail only the stuck requests as node_failed; completed ones are
+            # already done and unaffected.
+            for rid in stuck:
+                try:
+                    await self._send(
+                        protocol.MSG_ERROR, id=rid,
+                        error=f"node_failed: drain timeout after {self.drain_timeout}s",
+                    )
+                except Exception:
+                    pass
+        self._stopped = True
+        if self._ws is not None:
+            try:
+                await self._ws.close()
+            except Exception:
+                pass
+
+    async def _connect_and_serve(self):
+        headers = {"Authorization": f"Bearer {self.token}"} if self.token else {}
+        log.info("connecting to coordinator %s", self.coordinator_url)
+        async with websockets.connect(
+            self.coordinator_url,
+            additional_headers=headers,
+            max_size=32 * 1024 * 1024,
+            ping_interval=20,
+        ) as ws:
+            self._ws = ws
+            await self._register()
+            try:
+                async for raw in ws:
+                    msg = protocol.decode(raw)
+                    await self._dispatch(msg)
+            finally:
+                self._ws = None
+                self._emit("offline")
+
+    async def _register(self):
+        gpus = detect_gpus()
+        gpu = gpus[0] if gpus else detect_gpu()
+        interconnect = detect_interconnect() if len(gpus) > 1 else ""
+        hw = detect_hardware()
+        att = attest(require=self.require_attestation)
+        if self.require_attestation and not att.verified:
+            raise RuntimeError(
+                f"attestation required but not verified: {att.reason or att.mode}"
+            )
+        profile = protocol.NodeProfile(
+            name=self.name,
+            models=[self.model],
+            gpu=gpu,
+            attestation=att,
+            gpus=gpus,
+            gpu_count=len(gpus),
+            interconnect=interconnect,
+            ram_mb=hw.ram_mb,
+            memory_budget_mb=hw.memory_budget_mb,
+            accelerator=hw.accelerator.type,
+            price_per_mtok=self.price_per_mtok,
+            account_id=self.account_id,
+            owner_account=self.owner_account,
+            machine_id=self.machine_id,
+            cluster=self.cluster,
+            agent_version=__version__,
+        )
+        await self._send(protocol.MSG_REGISTER, profile=protocol.node_profile_to_dict(profile))
+        log.info(
+            "registered '%s' | gpu=%s | serving=%s | attestation=%s(verified=%s)",
+            self.name, gpu.name, self.model, att.mode, att.verified,
+        )
+
+    async def _dispatch(self, msg: Dict[str, Any]):
+        t = msg.get("t")
+        if t == protocol.MSG_INFER:
+            asyncio.create_task(self._handle_infer(msg))
+        elif t == protocol.MSG_PING:
+            await self._send(protocol.MSG_PONG)
+        elif t == protocol.MSG_REGISTERED:
+            log.info("coordinator assigned node_id=%s", msg.get("node_id"))
+            self._emit("live", node_id=msg.get("node_id"))
+        else:
+            log.debug("ignoring message type %s", t)
+
+    async def _handle_infer(self, msg: Dict[str, Any]):
+        req_id = msg.get("id")
+        body = dict(msg.get("body", {}))
+        requested = body.get("model", "")
+
+        # Once draining, refuse new work so it can be routed elsewhere. This
+        # closes the race between the operator leaving and the coordinator
+        # marking us un-routable; in-flight requests (already past this point)
+        # are unaffected and run to completion.
+        if self._draining:
+            await self._send(
+                protocol.MSG_ERROR, id=req_id,
+                error="node is draining; request not accepted",
+            )
+            return
+
+        # Enforce the single committed model at the node boundary. A node only
+        # serves the model it has hot-loaded; anything else is refused so it
+        # can never be coerced into running a cold/different model.
+        if requested and requested != self.model:
+            await self._send(
+                protocol.MSG_ERROR, id=req_id,
+                error=f"this node only serves '{self.model}', not '{requested}'",
+            )
+            return
+        body["model"] = self.model  # pin, in case the request omitted it
+        prompt_commit = hashlib.sha256(
+            _json.dumps(body.get("messages", []), sort_keys=True).encode()).hexdigest()
+
+        self._inflight_ids.add(req_id)
+        self._inflight_zero.clear()
+        stream = bool(body.get("stream", False))
+        try:
+            if stream:
+                acc, tokens = [], 0
+                async for line in self.backend.chat_stream(body):
+                    await self._send(protocol.MSG_CHUNK, id=req_id, data=line)
+                    if '"content"' in line:
+                        tokens += 1
+                        c = _extract_content(line)
+                        if c:
+                            acc.append(c)
+                response_commit = hashlib.sha256("".join(acc).encode()).hexdigest()
+                sig = self._sign_work(req_id, prompt_commit, tokens, response_commit, "complete")
+                await self._send(protocol.MSG_END, id=req_id, tokens=tokens,
+                                 response_commit=response_commit, sig=sig)
+            else:
+                result = await self.backend.chat_once(body)
+                content = ""
+                try:
+                    content = result["choices"][0]["message"].get("content") or ""
+                except Exception:
+                    pass
+                tokens = (result.get("usage", {}) or {}).get("completion_tokens", 0)
+                response_commit = hashlib.sha256(content.encode()).hexdigest()
+                sig = self._sign_work(req_id, prompt_commit, tokens, response_commit, "complete")
+                await self._send(protocol.MSG_RESULT, id=req_id, body=result,
+                                 tokens=tokens, response_commit=response_commit, sig=sig)
+        except Exception as e:
+            log.exception("inference failed for %s", req_id)
+            await self._send(protocol.MSG_ERROR, id=req_id, error=str(e))
+        finally:
+            self._inflight_ids.discard(req_id)
+            if not self._inflight_ids:
+                self._inflight_zero.set()
+
+    def _sign_work(self, req_id, prompt_commit, tokens, response_commit, delivery) -> str:
+        payload = keys.work_payload(req_id, self.model, prompt_commit, tokens,
+                                    response_commit, delivery)
+        return self.identity.sign(payload)
+
+    async def _send(self, msg_type: str, **fields: Any):
+        if self._ws is None:
+            return
+        frame = protocol.encode(msg_type, **fields)
+        async with self._send_lock:
+            await self._ws.send(frame)
+
+
+async def serve(
+    coordinator_url: str,
+    name: str,
+    backend_kind: str,
+    model: str,
+    ollama_url: str,
+    token: str = "",
+    require_attestation: bool = False,
+    warm: bool = True,
+    drain_timeout: float = 120.0,
+    owner_account: str = "",
+    cluster: str = "",
+):
+    if not model:
+        raise ValueError("a single --model must be specified; a node serves exactly one model")
+    backend = make_backend(backend_kind, model=model, ollama_url=ollama_url)
+
+    # Hot-load the committed model before advertising it to the network, so the
+    # node is never live with a cold model (low TTFT guarantee).
+    if warm:
+        log.info("hot-loading '%s' into memory ...", model)
+        try:
+            await backend.warm(model)
+            log.info("'%s' is hot", model)
+        except Exception as e:
+            log.warning("warm-up failed for '%s' (%s); serving anyway", model, e)
+
+    agent = NodeAgent(
+        coordinator_url=coordinator_url,
+        name=name,
+        backend=backend,
+        model=model,
+        token=token,
+        require_attestation=require_attestation,
+        drain_timeout=drain_timeout,
+        owner_account=owner_account,
+        cluster=cluster,
+    )
+    await agent.run_forever()

nvdc/app.py

@@ -0,0 +1,306 @@
+"""Local visual client: a small web app the node operator opens in a browser.
+
+`nvdc app` starts this server and opens the page. It shows hardware + inference
+mechanism, a catalog of popular models with fit indicators, lets the operator
+load one into memory (hot), and only then enables "Go Live" on the network.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json as _json
+import logging
+import secrets
+import time
+from pathlib import Path
+
+import httpx
+from fastapi import FastAPI, Request, Response
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
+
+from . import keys
+from .runtime import NodeRuntime
+
+log = logging.getLogger("nvdc.app")
+
+WEB_DIR = Path(__file__).parent / "web"
+
+
+def create_app(runtime: NodeRuntime) -> FastAPI:
+    app = FastAPI(title="nvdc node client")
+    app.state.runtime = runtime
+
+    # Let the hosted Vercel site detect and read THIS locally-running client.
+    app.add_middleware(
+        CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
+    )
+
+    @app.middleware("http")
+    async def private_network_access(request: Request, call_next):
+        # A public (https) page fetching localhost triggers Chrome's Private
+        # Network Access preflight, which needs this header to be allowed.
+        if request.method == "OPTIONS" and request.headers.get(
+                "access-control-request-private-network") == "true":
+            resp = Response(status_code=204)
+            resp.headers["Access-Control-Allow-Origin"] = request.headers.get("origin", "*")
+            resp.headers["Access-Control-Allow-Methods"] = "*"
+            resp.headers["Access-Control-Allow-Headers"] = "*"
+            resp.headers["Access-Control-Allow-Private-Network"] = "true"
+            return resp
+        resp = await call_next(request)
+        resp.headers["Access-Control-Allow-Private-Network"] = "true"
+        return resp
+
+    @app.get("/", response_class=HTMLResponse)
+    async def index():
+        return (WEB_DIR / "index.html").read_text(encoding="utf-8")
+
+    @app.get("/api/state")
+    async def state():
+        await runtime.refresh_installed()
+        return runtime.snapshot()
+
+    @app.post("/api/load")
+    async def load(request: Request):
+        body = await request.json()
+        model_id = body.get("model")
+        if not model_id:
+            return JSONResponse({"ok": False, "error": "model required"}, status_code=400)
+        ok, msg = runtime.start_load(model_id)
+        return {"ok": ok, "message": msg}
+
+    @app.post("/api/network/live")
+    async def live(request: Request):
+        body = await request.json()
+        ok, msg = runtime.go_live(
+            coordinator=body.get("coordinator", ""),
+            token=body.get("token", ""),
+        )
+        return {"ok": ok, "message": msg}
+
+    @app.post("/api/network/offline")
+    async def offline():
+        ok, msg = runtime.go_offline()
+        return {"ok": ok, "message": msg}
+
+    # ---- consumer proxies to the coordinator --------------------------------
+    # Keep the UI same-origin (no CORS) by proxying the coordinator's public API
+    # through this local app. These power the Home / Chat / Network tabs.
+    @app.get("/api/coordinator")
+    async def coordinator_info():
+        base = runtime.coordinator_http
+        return {"http": base, "openai_base": (base + "/v1") if base else ""}
+
+    async def _proxy_get(path: str):
+        base = runtime.coordinator_http
+        if not base:
+            return JSONResponse({"error": "no coordinator configured"}, status_code=503)
+        try:
+            async with httpx.AsyncClient(timeout=10) as c:
+                r = await c.get(base + path)
+                return JSONResponse(r.json(), status_code=r.status_code)
+        except Exception as e:
+            return JSONResponse({"error": f"coordinator unreachable: {e}"}, status_code=502)
+
+    @app.get("/api/net/status")
+    async def net_status():
+        return await _proxy_get("/api/network")
+
+    @app.get("/api/net/models")
+    async def net_models():
+        return await _proxy_get("/v1/models")
+
+    @app.get("/api/net/miners")
+    async def net_miners():
+        return await _proxy_get("/nodes")
+
+    @app.get("/api/net/market")
+    async def net_market():
+        return await _proxy_get("/api/market")
+
+    @app.get("/api/net/marketplace")
+    async def net_marketplace():
+        return await _proxy_get("/v1/marketplace")
+
+    @app.get("/api/net/ledger")
+    async def net_ledger():
+        return await _proxy_get("/api/ledger?limit=40")
+
+    @app.post("/api/net/verify")
+    async def net_verify(request: Request):
+        base = runtime.coordinator_http
+        if not base:
+            return JSONResponse({"error": "no coordinator"}, status_code=503)
+        receipt = await request.json()
+        try:
+            async with httpx.AsyncClient(timeout=8) as c:
+                r = await c.post(base + "/api/verify", json=receipt)
+                return JSONResponse(r.json(), status_code=r.status_code)
+        except Exception as e:
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    # ---- wallet (backed by the coordinator account) -------------------------
+    def _acct_headers():
+        return {"X-NVDC-Account": runtime.wallet.account_id}
+
+    @app.get("/api/wallet")
+    async def wallet():
+        base = runtime.coordinator_http
+        if not base:
+            return JSONResponse({"account_id": runtime.wallet.account_id,
+                                 "balance_usd": 0, "total_earned": 0, "total_spent": 0,
+                                 "receipts": [], "error": "no coordinator"}, status_code=200)
+        try:
+            async with httpx.AsyncClient(timeout=8) as c:
+                r = await c.get(base + "/api/account", headers=_acct_headers())
+                return JSONResponse(r.json(), status_code=r.status_code)
+        except Exception as e:
+            return JSONResponse({"account_id": runtime.wallet.account_id,
+                                 "balance_usd": 0, "error": str(e)}, status_code=200)
+
+    @app.post("/api/wallet/buy")
+    async def wallet_buy(request: Request):
+        base = runtime.coordinator_http
+        if not base:
+            return JSONResponse({"ok": False, "error": "no coordinator configured"}, status_code=503)
+        body = await request.json()
+        try:
+            amount = float(body.get("amount_usd", 0))
+        except (TypeError, ValueError):
+            return JSONResponse({"ok": False, "error": "invalid amount"}, status_code=400)
+        try:
+            async with httpx.AsyncClient(timeout=8) as c:
+                r = await c.post(base + "/api/account/deposit",
+                                 json={"amount_usd": amount}, headers=_acct_headers())
+                j = r.json()
+                return {"ok": r.status_code == 200, "bought_usd": amount, **j}
+        except Exception as e:
+            return JSONResponse({"ok": False, "error": str(e)}, status_code=502)
+
+    async def _coordinator_post(path: str, json_body=None):
+        base = runtime.coordinator_http
+        if not base:
+            return JSONResponse({"error": "no coordinator configured"}, status_code=503)
+        try:
+            async with httpx.AsyncClient(timeout=20) as c:
+                r = await c.post(base + path, json=json_body or {}, headers=_acct_headers())
+                return JSONResponse(r.json(), status_code=r.status_code)
+        except Exception as e:
+            return JSONResponse({"error": f"coordinator unreachable: {e}"}, status_code=502)
+
+    @app.get("/api/payments/config")
+    async def payments_config():
+        return await _proxy_get("/api/payments/config")
+
+    @app.post("/api/wallet/checkout")
+    async def wallet_checkout(request: Request):
+        body = await request.json()
+        return await _coordinator_post("/api/account/checkout",
+                                       {"amount_usd": body.get("amount_usd", 0)})
+
+    @app.post("/api/payout/onboard")
+    async def payout_onboard():
+        return await _coordinator_post("/api/payout/onboard")
+
+    @app.post("/api/payout/withdraw")
+    async def payout_withdraw():
+        return await _coordinator_post("/api/payout/withdraw")
+
+    @app.post("/api/price")
+    async def set_price(request: Request):
+        body = await request.json()
+        try:
+            runtime.set_price(float(body.get("price_per_mtok", 0)))
+        except (TypeError, ValueError):
+            return JSONResponse({"ok": False, "error": "invalid price"}, status_code=400)
+        return {"ok": True, "price_per_mtok": runtime.price_per_mtok}
+
+    @app.post("/api/net/chat")
+    async def net_chat(request: Request):
+        base = runtime.coordinator_http
+        if not base:
+            return JSONResponse({"error": "no coordinator configured"}, status_code=503)
+        body = await request.json()
+        # The app's own consumer account funds these requests.
+        account = runtime.wallet.account_id
+        headers = {"Content-Type": "application/json", "X-NVDC-Account": account}
+        for h in ("x-nvdc-pin", "x-nvdc-min-rating", "x-nvdc-max-price", "x-nvdc-models"):
+            if request.headers.get(h):
+                headers[h] = request.headers[h]
+
+        # Sign the request authorization so the receipt is dual-signed.
+        prompt_commit = hashlib.sha256(
+            _json.dumps(body.get("messages", []), sort_keys=True).encode()).hexdigest()
+        nonce = secrets.token_hex(8)
+        ts = int(time.time())
+        try:
+            max_price = float(headers.get("x-nvdc-max-price", -1))
+        except ValueError:
+            max_price = -1
+        auth = keys.auth_payload(account, body.get("model", ""), prompt_commit,
+                                 max_price, nonce, ts)
+        headers["X-NVDC-Consumer-Sig"] = runtime.wallet.sign(auth)
+        headers["X-NVDC-Nonce"] = nonce
+        headers["X-NVDC-Ts"] = str(ts)
+
+        async def _balance():
+            try:
+                async with httpx.AsyncClient(timeout=5) as c:
+                    r = await c.get(base + "/api/account", headers={"X-NVDC-Account": runtime.wallet.account_id})
+                    return float(r.json().get("balance_usd", 0))
+            except Exception:
+                return None
+
+        if body.get("stream"):
+            async def gen():
+                import json as _j
+                tokens, price, node, status = 0, 0.0, "", 200
+                try:
+                    async with httpx.AsyncClient(timeout=None) as c:
+                        async with c.stream("POST", base + "/v1/chat/completions",
+                                            json=body, headers=headers) as r:
+                            status = r.status_code
+                            price = float(r.headers.get("x-nvdc-price-per-mtok", 0) or 0)
+                            node = r.headers.get("x-nvdc-node", "")
+                            if status != 200:
+                                txt = (await r.aread()).decode("utf-8", "replace")
+                                yield "data: " + _j.dumps({"error": {"message": txt}}) + "\n\n"
+                                return
+                            async for line in r.aiter_lines():
+                                if not line:
+                                    continue
+                                if line.strip() == "data: [DONE]":
+                                    break
+                                if '"content"' in line:
+                                    tokens += 1
+                                yield line + "\n\n"
+                    cost = tokens * price / 1_000_000.0
+                    bal = await _balance()
+                    yield "data: " + _j.dumps({"nvdc_billing": {
+                        "node": node, "price_per_mtok": price, "tokens": tokens,
+                        "cost_usd": round(cost, 8),
+                        "balance_usd": round(bal, 6) if bal is not None else None}}) + "\n\n"
+                    yield "data: [DONE]\n\n"
+                except Exception as e:
+                    yield "data: " + _j.dumps({"error": {"message": str(e)}}) + "\n\n"
+            return StreamingResponse(gen(), media_type="text/event-stream")
+
+        try:
+            async with httpx.AsyncClient(timeout=None) as c:
+                r = await c.post(base + "/v1/chat/completions", json=body, headers=headers)
+                j = r.json()
+                if r.status_code == 200:
+                    bal = await _balance()
+                    j["nvdc_billing"] = {
+                        "node": r.headers.get("x-nvdc-node", ""),
+                        "price_per_mtok": float(r.headers.get("x-nvdc-price-per-mtok", 0) or 0),
+                        "tokens": int(r.headers.get("x-nvdc-tokens", 0) or 0),
+                        "cost_usd": float(r.headers.get("x-nvdc-cost", 0) or 0),
+                        "receipt": r.headers.get("x-nvdc-receipt", ""),
+                        "balance_usd": round(bal, 6) if bal is not None else None}
+                return JSONResponse(j, status_code=r.status_code)
+        except Exception as e:
+            return JSONResponse({"error": f"coordinator unreachable: {e}"}, status_code=502)
+
+    return app