fuin 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fuin/__init__.py +5 -0
- fuin/asker.py +161 -0
- fuin/attestation.py +235 -0
- fuin/cli.py +220 -0
- fuin/core.py +195 -0
- fuin/diagnose.py +46 -0
- fuin/e2ee.py +92 -0
- fuin/inference.py +114 -0
- fuin/near.py +176 -0
- fuin/owner.py +311 -0
- fuin/policy/attestation-v1.json +24 -0
- fuin/policy/attestation-v2.json +42 -0
- fuin/policy/attestation-v3.json +56 -0
- fuin/policy/attestation-v4.json +70 -0
- fuin/policy/attestation-v5.json +102 -0
- fuin/policy/attestation-v6.json +128 -0
- fuin/policy/attestation-v7.json +232 -0
- fuin/policy/attestation-v8.json +254 -0
- fuin/policy/attestation-v9.json +298 -0
- fuin/policy/sigstore-trusted-root.json +126 -0
- fuin/policy/tee-v2.json +85 -0
- fuin/policy.py +97 -0
- fuin/policyupdate.py +73 -0
- fuin/provenance.py +245 -0
- fuin/py.typed +0 -0
- fuin/scorer.py +46 -0
- fuin/scorer_v1.py +45 -0
- fuin/tee.py +548 -0
- fuin/tooling.py +180 -0
- fuin/verify.py +568 -0
- fuin-1.0.0.dist-info/METADATA +223 -0
- fuin-1.0.0.dist-info/RECORD +35 -0
- fuin-1.0.0.dist-info/WHEEL +4 -0
- fuin-1.0.0.dist-info/entry_points.txt +2 -0
- fuin-1.0.0.dist-info/licenses/LICENSE +21 -0
fuin/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
# The verifier's policies: shipped inside the package when installed (uv tool install, pip), else policy/ in the checkout.
|
|
4
|
+
POLICY = next((p for p in (Path(__file__).parent / "policy", Path(__file__).resolve().parents[2] / "policy") if p.is_dir()),
|
|
5
|
+
Path(__file__).resolve().parents[2] / "policy")
|
fuin/asker.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Enclave-asked rounds (audit mode tee-asked-v1): the owner side.
|
|
2
|
+
|
|
3
|
+
The asker (ask/, a wasm32-wasip2 build pinned in policy/tee-v2.json `askers`) runs as an OutLayer project whose owner
|
|
4
|
+
account holds no keys, so its code and its storage can't be changed or wiped. For each sealed item it records the ask,
|
|
5
|
+
makes the one model call itself with the owner's NEAR AI key (an OutLayer secret), checks and scores the signed answer
|
|
6
|
+
and records it final; a second ask returns the recorded answer. No question, answer or response body leaves the enclave:
|
|
7
|
+
the owner gets each answer's hashes, score and signature, and the aggregate task attests the run's roots and total.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import requests
|
|
15
|
+
|
|
16
|
+
from fuin import core, owner, tee
|
|
17
|
+
from fuin import policy as P
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def askers() -> list[dict]:
|
|
21
|
+
return tee.tee_policy().get("askers", [])
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def asker_for(policy: dict) -> dict:
|
|
25
|
+
a = policy["audit"]
|
|
26
|
+
found = [x for x in askers() if x["wasm_sha256"] == a["asker_wasm_sha256"] and x["project"] == a["asker_project"]]
|
|
27
|
+
if not found:
|
|
28
|
+
raise SystemExit(f"the round sealed asker {a['asker_wasm_sha256'][:12]}… in {a['asker_project']}, which policy/tee-v2.json doesn't list")
|
|
29
|
+
return found[0]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def secrets_ref(st: dict) -> dict:
|
|
33
|
+
return {"profile": "nearai", "account_id": st["owner"]}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def keygen(owner_id: str, ak: dict) -> dict:
|
|
37
|
+
"""The asker's in-enclave key for this owner (sealed under the project's key: only the locked project's code opens it)."""
|
|
38
|
+
# one per (owner, build, project): the secret is sealed under the project's key, so another project can't open it
|
|
39
|
+
path = Path("private") / f"ask-key-{owner_id}-{ak['wasm_sha256'][:12]}-{ak['project'].replace('/', '_')}.json"
|
|
40
|
+
if path.exists():
|
|
41
|
+
return json.loads(path.read_text())
|
|
42
|
+
tx, input_data, out = tee.execute(owner_id, {"mode": "keygen"}, ak, project=ak["project"])
|
|
43
|
+
if "pub" not in out:
|
|
44
|
+
raise RuntimeError(f"asker keygen failed: {out}")
|
|
45
|
+
key = {"owner": owner_id, "wasm_sha256": ak["wasm_sha256"], "project": ak["project"], "pub": out["pub"], "sealed": out["sealed"],
|
|
46
|
+
"transaction_hash": tx}
|
|
47
|
+
path.write_text(json.dumps(key, indent=1))
|
|
48
|
+
return key
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def plaintext(st: dict, model: str, rn: dict) -> dict:
|
|
52
|
+
"""The sealed round as the asker reads it: tee.plaintext's fields without rows (the enclave makes them)."""
|
|
53
|
+
return {"v": "fuin-tee-round-v1", "model": model, "policy_jcs": core.jcs(st["policy"]).decode(),
|
|
54
|
+
**({"seal_block_hash": st["seal"]["block_hash"]} if P.is_seal_bound(st["policy"]) else {}),
|
|
55
|
+
**({"model_pub": tee._signer_pub(rn).hex()} if P.is_e2ee(st["policy"]) else {}),
|
|
56
|
+
"items": [{"salt": r["salt"], "item_jcs": core.jcs(r["item"]).decode()} for r in st["items"]]}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _check_item(out: dict, st: dict, model: str, i: int, signer: str) -> dict:
|
|
60
|
+
"""An ask's output, checked before it is kept: this round, this item, this signer, a valid provider_tee signature."""
|
|
61
|
+
from eth_account import Account
|
|
62
|
+
from eth_account.messages import encode_defunct
|
|
63
|
+
if out.get("v") != "fuin-ask-item-v1" or (out.get("model"), out.get("i"), out.get("benchmark_root")) != (model, i, st["seal"]["root"]):
|
|
64
|
+
raise RuntimeError(f"item {i}: unexpected asker output {str(out)[:200]}")
|
|
65
|
+
text = f"{model}:{out['req_sha256']}:{out['resp_sha256']}"
|
|
66
|
+
who = Account.recover_message(encode_defunct(text=text), signature=out["signature"]).lower()
|
|
67
|
+
if who != signer or str(out["signing_address"]).lower() != signer or out["score"] not in (0, 1):
|
|
68
|
+
raise RuntimeError(f"item {i}: signature doesn't recover to the preflighted signer")
|
|
69
|
+
return {"i": i, "req_sha256": out["req_sha256"], "resp_sha256": out["resp_sha256"], "score": out["score"],
|
|
70
|
+
"unreadable": bool(out.get("unreadable")), "attempts": out.get("attempts"),
|
|
71
|
+
"signature": {"text": text, "signature": out["signature"], "signing_address": out["signing_address"],
|
|
72
|
+
"signature_kind": "provider_tee", "signing_algo": "ecdsa"}}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def run(bid: str, model: str, host, tries: int = 3) -> dict:
|
|
76
|
+
"""Ask every item of this model's run inside the enclave, then have the enclave attest the run (aggregate)."""
|
|
77
|
+
st = owner.load(bid)
|
|
78
|
+
if not P.is_asked_round(st["policy"]):
|
|
79
|
+
raise SystemExit("not an enclave-asked round (seal it with --ask)")
|
|
80
|
+
if "seal" not in st:
|
|
81
|
+
raise SystemExit("round not sealed yet")
|
|
82
|
+
ak = asker_for(st["policy"])
|
|
83
|
+
rn = owner.preflight(st, model)
|
|
84
|
+
if "commit" in rn:
|
|
85
|
+
raise SystemExit(f"{model} run already committed")
|
|
86
|
+
key = keygen(st["owner"], ak)
|
|
87
|
+
blob, eph_pub, nonce = tee.encrypt(tee.jdump(plaintext(st, model, rn)).encode(), key["pub"])
|
|
88
|
+
url, encoding, cleanup = host(blob) # one hosted ciphertext for every ask and the aggregate; deleted at the end
|
|
89
|
+
try:
|
|
90
|
+
served = requests.get(url, timeout=120).content
|
|
91
|
+
base = {"url": url, "blob_sha256": hashlib.sha256(served).hexdigest(), "encoding": encoding, "sealed": key["sealed"],
|
|
92
|
+
"eph_pub": eph_pub, "nonce": nonce}
|
|
93
|
+
done = {r["i"] for r in rn["rows"]}
|
|
94
|
+
for i in range(len(st["items"])):
|
|
95
|
+
if i in done:
|
|
96
|
+
continue
|
|
97
|
+
for attempt in range(tries): # a pending ask (nothing left the enclave) may be retried; the enclave counts it
|
|
98
|
+
try:
|
|
99
|
+
tx, _, out = tee.execute(st["owner"], {"mode": "ask", **base, "i": i, "signer": rn["signer"]}, ak,
|
|
100
|
+
project=ak["project"], secrets_ref=secrets_ref(st))
|
|
101
|
+
if "error" in out:
|
|
102
|
+
raise RuntimeError(out["error"])
|
|
103
|
+
row = _check_item(out, st, model, i, rn["signer"])
|
|
104
|
+
row["ask_tx"] = tx
|
|
105
|
+
break
|
|
106
|
+
except RuntimeError as e:
|
|
107
|
+
rn["attempt_log"].append({"i": i, "attempt": attempt, "error": str(e)[:200]})
|
|
108
|
+
owner.save(st)
|
|
109
|
+
print(f" {i}: attempt {attempt + 1} failed: {str(e)[:120]}", flush=True)
|
|
110
|
+
else:
|
|
111
|
+
raise SystemExit(f"item {i}: {tries} asks failed; rerun `fuin ask` to resume (a recorded answer is never re-asked)")
|
|
112
|
+
rn["rows"].append(row)
|
|
113
|
+
owner.save(st)
|
|
114
|
+
print(f" {i}: score {row['score']} (attempt {row['attempts']})", flush=True)
|
|
115
|
+
tx, input_data, out = tee.execute(st["owner"], {"mode": "aggregate", **base}, ak, project=ak["project"])
|
|
116
|
+
finally:
|
|
117
|
+
if cleanup:
|
|
118
|
+
cleanup()
|
|
119
|
+
want = expected(st, model, rn)
|
|
120
|
+
diff = {k: (out.get(k), v) for k, v in want.items() if out.get(k) != v}
|
|
121
|
+
if out.get("v") != "fuin-tee-asked-v1" or diff:
|
|
122
|
+
raise RuntimeError(f"asker aggregate differs from the asked rows: {out.get('error') or diff}")
|
|
123
|
+
rn["tee_asked"] = {"v": "fuin-tee-asked-evidence-v1", "transaction_hash": tx, "input_data": input_data, "output": out,
|
|
124
|
+
"record": tee.fetch_record(tx), "scorer_url": ak["url"]}
|
|
125
|
+
owner.save(st)
|
|
126
|
+
rn["tee_asked_check"] = check(st, model, host)
|
|
127
|
+
owner.save(st)
|
|
128
|
+
return st
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def check(st: dict, model: str, host) -> dict:
|
|
132
|
+
"""Have the scorer rebuild every request of the asked run with the true seal block hash (mode asked-check): each must
|
|
133
|
+
equal the request the model signed, so every recorded ask came after the seal, and the roots must equal the run's."""
|
|
134
|
+
rn = st["runs"][model]
|
|
135
|
+
tp = tee.tee_policy()
|
|
136
|
+
sc = next(x for x in reversed(tp["scorers"]) if x["wasm_sha256"] in tp.get("asked_checkers", []))
|
|
137
|
+
rows = sorted(rn["rows"], key=lambda r: r["i"])
|
|
138
|
+
plain = {**plaintext(st, model, rn), "signer": rn["signer"],
|
|
139
|
+
"asked": [{"req_sha256": r["req_sha256"], "resp_sha256": r["resp_sha256"], "score": r["score"]} for r in rows]}
|
|
140
|
+
tx, input_data, out = tee._sealed_task(st["owner"], host, sc, "asked-check", plain)
|
|
141
|
+
want = {k: v for k, v in expected(st, model, rn).items() if k not in ("retried", "unreadable")}
|
|
142
|
+
diff = {k: (out.get(k), v) for k, v in want.items() if out.get(k) != v}
|
|
143
|
+
if out.get("v") != "fuin-tee-asked-check-v1" or diff:
|
|
144
|
+
raise RuntimeError(f"asked-check differs from the asked run: {out.get('error') or diff}")
|
|
145
|
+
return {"v": "fuin-tee-asked-check-evidence-v1", "transaction_hash": tx, "input_data": input_data, "output": out,
|
|
146
|
+
"record": tee.fetch_record(tx), "scorer_url": sc["url"]}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def expected(st: dict, model: str, rn: dict) -> dict:
|
|
150
|
+
"""What a faithful aggregate must output, from the asked rows (the commit uses the same rows)."""
|
|
151
|
+
rows = sorted(rn["rows"], key=lambda r: r["i"])
|
|
152
|
+
bl = owner.benchmark_leaves(st)
|
|
153
|
+
rl = [core.run_leaf(r["i"], bl[r["i"]], bytes.fromhex(r["req_sha256"]), bytes.fromhex(r["resp_sha256"]), r["score"]) for r in rows]
|
|
154
|
+
out = {"model": model, "n": len(st["items"]), "total": sum(r["score"] for r in rows), "benchmark_root": st["seal"]["root"],
|
|
155
|
+
"policy_hash": P.policy_hash(st["policy"]), "run_root": core.merkle_root(rl).hex(), "signer": rn["signer"],
|
|
156
|
+
"retried": sum(max(0, (r.get("attempts") or 1) - 1) for r in rows)}
|
|
157
|
+
if P.is_seal_bound(st["policy"]):
|
|
158
|
+
out["seal_block_hash"] = st["seal"]["block_hash"]
|
|
159
|
+
if P.is_e2ee(st["policy"]):
|
|
160
|
+
out["unreadable"] = sum(1 for r in rows if r.get("unreadable"))
|
|
161
|
+
return out
|
fuin/attestation.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Model attestation report checks (spec: Attestation acceptance policy).
|
|
2
|
+
|
|
3
|
+
Credential-free: needs network access to Intel PCS (quote collateral) and NVIDIA NRAS (GPU verdict).
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
import base64
|
|
8
|
+
import hashlib
|
|
9
|
+
import inspect
|
|
10
|
+
import json
|
|
11
|
+
import time
|
|
12
|
+
import re
|
|
13
|
+
import struct
|
|
14
|
+
|
|
15
|
+
import dcap_qvl
|
|
16
|
+
import requests
|
|
17
|
+
|
|
18
|
+
NRAS_URL = "https://nras.attestation.nvidia.com/v3/attest/gpu"
|
|
19
|
+
CATALOG_URL = "https://cloud-api.near.ai/v1/model/list"
|
|
20
|
+
IMAGE_RE = re.compile(r"image:\s*\S*?([a-z0-9][\w./-]*@sha256:[0-9a-f]{64})")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _json(v):
|
|
24
|
+
return json.loads(v) if isinstance(v, str) else v
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def app_compose(report: dict) -> str:
|
|
28
|
+
return _json(report["info"])["tcb_info"]["app_compose"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def compose_hash(report: dict) -> str:
|
|
32
|
+
return hashlib.sha256(app_compose(report).encode()).hexdigest()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def image_digests(report: dict) -> list[str]:
|
|
36
|
+
compose = _json(app_compose(report))
|
|
37
|
+
return sorted(set(IMAGE_RE.findall(compose.get("docker_compose_file", ""))))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def replay_rtmr3(event_log) -> bytes:
|
|
41
|
+
rtmr = bytes(48)
|
|
42
|
+
for ev in _json(event_log):
|
|
43
|
+
if ev.get("imr") != 3:
|
|
44
|
+
continue
|
|
45
|
+
digest = hashlib.sha384(
|
|
46
|
+
struct.pack("<I", ev["event_type"]) + b":" + ev["event"].encode() + b":" + bytes.fromhex(ev["event_payload"])
|
|
47
|
+
).digest()
|
|
48
|
+
if ev.get("digest") and bytes.fromhex(ev["digest"]) != digest:
|
|
49
|
+
raise ValueError("event log digest mismatch")
|
|
50
|
+
rtmr = hashlib.sha384(rtmr + digest).digest()
|
|
51
|
+
return rtmr
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def parse_report_data(report_data: bytes) -> tuple[bytes, bytes]:
|
|
55
|
+
"""64 bytes: signer (20-byte address, zero-padded to 32) || 32-byte client nonce."""
|
|
56
|
+
if len(report_data) != 64:
|
|
57
|
+
raise ValueError("report data must be 64 bytes")
|
|
58
|
+
return report_data[:32], report_data[32:]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
async def _collateral(quote: bytes):
|
|
62
|
+
r = dcap_qvl.get_collateral(dcap_qvl.INTEL_PCS_URL, quote)
|
|
63
|
+
return await r if inspect.isawaitable(r) else r
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _pcs_reachable() -> bool:
|
|
67
|
+
try:
|
|
68
|
+
requests.get(f"{dcap_qvl.INTEL_PCS_URL.rstrip('/')}/tdx/certification/v4/qe/identity", timeout=15)
|
|
69
|
+
return True
|
|
70
|
+
except requests.RequestException:
|
|
71
|
+
return False
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def verify_quote(quote_hex: str):
|
|
75
|
+
"""Returns (status, advisory_ids, td_report dict).
|
|
76
|
+
|
|
77
|
+
Raises ValueError/TypeError for a malformed or invalid quote, and RuntimeError when Intel's collateral can't be
|
|
78
|
+
fetched for a well-formed quote: an outage, not evidence of tampering. (The combined dcap call raised ValueError
|
|
79
|
+
for both, so an Intel outage was refused as an invalid quote.)"""
|
|
80
|
+
raw = bytes.fromhex(quote_hex)
|
|
81
|
+
dcap_qvl.parse_quote(raw)
|
|
82
|
+
for attempt in (1, 2): # one retry: a single dropped request to Intel would otherwise read as an outage (the pilot hit one)
|
|
83
|
+
try:
|
|
84
|
+
collateral = asyncio.run(_collateral(raw))
|
|
85
|
+
break
|
|
86
|
+
except ValueError as e: # an outage only if the request failed or Intel doesn't answer; otherwise the quote is at fault
|
|
87
|
+
dropped = any(m in str(e) for m in ("error sending request", "timed out", "dns error"))
|
|
88
|
+
if dropped and attempt == 1:
|
|
89
|
+
time.sleep(3)
|
|
90
|
+
continue
|
|
91
|
+
if dropped or not _pcs_reachable():
|
|
92
|
+
raise RuntimeError(f"Intel collateral unavailable: {str(e)[:160]}") from e
|
|
93
|
+
raise ValueError(f"quote rejected at the collateral step: {str(e)[:160]}") from e
|
|
94
|
+
res = dcap_qvl.verify(raw, collateral, int(time.time()))
|
|
95
|
+
rep = json.loads(res.to_json())["report"]
|
|
96
|
+
td = rep.get("TD10") or rep.get("TD15")
|
|
97
|
+
if td is None:
|
|
98
|
+
raise ValueError("not a TDX quote")
|
|
99
|
+
return res.status, list(res.advisory_ids), td
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
NRAS_ISSUER = "https://nras.attestation.nvidia.com"
|
|
103
|
+
NRAS_JWKS = "https://nras.attestation.nvidia.com/.well-known/jwks.json"
|
|
104
|
+
_NRAS_KEYS: dict = {} # NVIDIA's signing keys by kid, fetched once per process
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _b64u(s: str) -> bytes:
|
|
108
|
+
return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _nras_key(kid: str):
|
|
112
|
+
"""NVIDIA's ES384 (P-384) public key for this kid, from its published key set."""
|
|
113
|
+
from cryptography.hazmat.primitives.asymmetric import ec
|
|
114
|
+
if kid not in _NRAS_KEYS:
|
|
115
|
+
r = requests.get(NRAS_JWKS, timeout=30)
|
|
116
|
+
r.raise_for_status()
|
|
117
|
+
for k in r.json().get("keys", []):
|
|
118
|
+
if k.get("kty") == "EC" and k.get("crv") == "P-384":
|
|
119
|
+
_NRAS_KEYS[k["kid"]] = ec.EllipticCurvePublicNumbers(
|
|
120
|
+
int.from_bytes(_b64u(k["x"]), "big"), int.from_bytes(_b64u(k["y"]), "big"), ec.SECP384R1()).public_key()
|
|
121
|
+
return _NRAS_KEYS.get(kid)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def nras_token_ok(jwt: str, nonce: str, now: float | None = None) -> bool:
|
|
125
|
+
"""NVIDIA's verdict token: ES384-signed by a key in NVIDIA's key set, issued by NRAS, inside its validity window,
|
|
126
|
+
bound to this report's nonce (eat_nonce), with an explicit overall pass (review 5)."""
|
|
127
|
+
from cryptography.exceptions import InvalidSignature
|
|
128
|
+
from cryptography.hazmat.primitives import hashes
|
|
129
|
+
from cryptography.hazmat.primitives.asymmetric import ec, utils
|
|
130
|
+
try:
|
|
131
|
+
h64, c64, s64 = jwt.split(".")
|
|
132
|
+
header, claims, sig = json.loads(_b64u(h64)), json.loads(_b64u(c64)), _b64u(s64)
|
|
133
|
+
except ValueError:
|
|
134
|
+
return False
|
|
135
|
+
key = _nras_key(header.get("kid", "")) if header.get("alg") == "ES384" else None
|
|
136
|
+
if key is None or len(sig) != 96:
|
|
137
|
+
return False
|
|
138
|
+
try:
|
|
139
|
+
key.verify(utils.encode_dss_signature(int.from_bytes(sig[:48], "big"), int.from_bytes(sig[48:], "big")),
|
|
140
|
+
f"{h64}.{c64}".encode(), ec.ECDSA(hashes.SHA384()))
|
|
141
|
+
except InvalidSignature:
|
|
142
|
+
return False
|
|
143
|
+
now = time.time() if now is None else now
|
|
144
|
+
return (claims.get("iss") == NRAS_ISSUER and claims.get("nbf", 0) <= now + 300 and claims.get("exp", 0) >= now - 300
|
|
145
|
+
and str(claims.get("eat_nonce", "")).lower() == nonce.lower() and claims.get("x-nvidia-overall-att-result") is True)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def nras_verdict(nvidia_payload, nonce: str | None = None) -> bool:
|
|
149
|
+
"""POST the GPU evidence unchanged to NVIDIA NRAS; True only on a verified overall pass for this nonce (the
|
|
150
|
+
evidence's own nonce unless given)."""
|
|
151
|
+
body = nvidia_payload if isinstance(nvidia_payload, str) else json.dumps(nvidia_payload)
|
|
152
|
+
r = requests.post(NRAS_URL, data=body, headers={"Content-Type": "application/json"}, timeout=60)
|
|
153
|
+
r.raise_for_status()
|
|
154
|
+
return nras_token_ok(r.json()[0][1], nonce if nonce is not None else str(_json(nvidia_payload).get("nonce", "")))
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def tcb_outcome(status: str, advisories: list[str]) -> str:
|
|
158
|
+
"""User's TCB policy (2026-10-27): clean / warning / revoked."""
|
|
159
|
+
if status == "Revoked":
|
|
160
|
+
return "revoked"
|
|
161
|
+
return "clean" if status == "UpToDate" and not advisories else "warning"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def check_report(report: dict, nonce_hex: str, policy: dict, *, use_nras: bool = True) -> dict:
|
|
165
|
+
"""Check one model report under an attestation policy. Returns per-check results and an outcome.
|
|
166
|
+
|
|
167
|
+
outcome: "accepted", "accepted_with_warning", ("refused", code) or ("cant_certify", code).
|
|
168
|
+
"""
|
|
169
|
+
if not all(isinstance(report.get(k), str) for k in ("model_name", "signing_address", "intel_quote")):
|
|
170
|
+
# NEAR also serves models from partners' own TEEs (Chutes, Tinfoil) with their own evidence format; Fūin checks only NEAR's
|
|
171
|
+
return {"model": report.get("model_name") or report.get("model"), "signing_address": str(report.get("signing_address", "")).lower(),
|
|
172
|
+
"error": f"attested by {report.get('provider') or 'another scheme'}, in a format Fūin doesn't verify",
|
|
173
|
+
"outcome": ("cant_certify", "policy_unsupported")}
|
|
174
|
+
checks: dict = {"model": report["model_name"], "signing_address": report["signing_address"].lower()}
|
|
175
|
+
nonce = nonce_hex.lower()
|
|
176
|
+
try:
|
|
177
|
+
status, advisories, td = verify_quote(report["intel_quote"])
|
|
178
|
+
except (ValueError, KeyError, TypeError) as e:
|
|
179
|
+
return {**checks, "error": str(e), "outcome": ("refused", "attestation_invalid")}
|
|
180
|
+
except RuntimeError as e: # collateral service unreachable
|
|
181
|
+
return {**checks, "error": str(e), "outcome": ("cant_certify", "evidence_unavailable")}
|
|
182
|
+
signer_rd, nonce_rd = parse_report_data(bytes.fromhex(td["report_data"]))
|
|
183
|
+
signer = bytes.fromhex(report["signing_address"].removeprefix("0x"))
|
|
184
|
+
checks.update(
|
|
185
|
+
quote_status=status,
|
|
186
|
+
advisories=advisories,
|
|
187
|
+
signing_algo_ok=report.get("signing_algo") == "ecdsa",
|
|
188
|
+
report_data_signer_ok=signer_rd == signer.ljust(32, b"\0"),
|
|
189
|
+
report_data_nonce_ok=nonce_rd.hex() == nonce,
|
|
190
|
+
request_nonce_ok=str(report.get("request_nonce", "")).lower() == nonce,
|
|
191
|
+
debug_off=int(td["td_attributes"][:2], 16) & 1 == 0,
|
|
192
|
+
mrconfigid_ok=bytes.fromhex(td["mr_config_id"]) == b"\x01" + bytes.fromhex(compose_hash(report)) + bytes(15),
|
|
193
|
+
gpu_nonce_ok=str(_json(report["nvidia_payload"]).get("nonce", "")).lower() == nonce,
|
|
194
|
+
)
|
|
195
|
+
try:
|
|
196
|
+
checks["rtmr3_ok"] = replay_rtmr3(report["event_log"]).hex() == td["rt_mr3"]
|
|
197
|
+
except ValueError:
|
|
198
|
+
checks["rtmr3_ok"] = False
|
|
199
|
+
bad = [k for k, v in checks.items() if k.endswith(("_ok", "_off")) and v is not True]
|
|
200
|
+
if bad:
|
|
201
|
+
return {**checks, "failed": bad, "outcome": ("refused", "attestation_invalid")}
|
|
202
|
+
tcb = tcb_outcome(status, advisories)
|
|
203
|
+
if tcb == "revoked":
|
|
204
|
+
return {**checks, "outcome": ("refused", "tcb_revoked")}
|
|
205
|
+
accepted = policy.get("models", {}).get(report["model_name"], {}).get("compose_hashes", {})
|
|
206
|
+
checks["compose_hash"] = compose_hash(report)
|
|
207
|
+
if checks["compose_hash"] not in accepted:
|
|
208
|
+
return {**checks, "outcome": ("cant_certify", "measurement_unapproved")}
|
|
209
|
+
# the OS the application runs on (firmware, kernel, initrd, cmdline) must be an approved measurement too; MRCONFIGID and
|
|
210
|
+
# RTMR3 only identify the application, and an unapproved OS could fake those extends
|
|
211
|
+
entry = accepted[checks["compose_hash"]]
|
|
212
|
+
os_seen = {k: td[k] for k in ("mr_td", "rt_mr0", "rt_mr1", "rt_mr2")}
|
|
213
|
+
checks["os_approved"] = (os_seen in entry.get("os_measurements", [])
|
|
214
|
+
and (report.get("info") or {}).get("os_image_hash") == entry.get("os_image_hash"))
|
|
215
|
+
if not checks["os_approved"]:
|
|
216
|
+
return {**checks, "os_seen": os_seen, "outcome": ("cant_certify", "measurement_unapproved")}
|
|
217
|
+
if use_nras:
|
|
218
|
+
try:
|
|
219
|
+
checks["nras_ok"] = nras_verdict(report["nvidia_payload"], nonce)
|
|
220
|
+
except (requests.RequestException, ValueError, KeyError, IndexError) as e:
|
|
221
|
+
return {**checks, "error": f"NRAS: {e}", "outcome": ("cant_certify", "evidence_unavailable")}
|
|
222
|
+
if not checks["nras_ok"]:
|
|
223
|
+
return {**checks, "outcome": ("refused", "attestation_invalid")}
|
|
224
|
+
checks["tcb"] = tcb
|
|
225
|
+
return {**checks, "td": td, "outcome": ("accepted",) if tcb == "clean" else ("accepted_with_warning",)}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def is_tee_model(model_id: str, catalog: dict | None = None) -> bool:
|
|
229
|
+
"""Catalog says verifiable + attestationSupported; external models (OpenAI, Anthropic, ...) don't."""
|
|
230
|
+
catalog = catalog or requests.get(CATALOG_URL, timeout=30).json()
|
|
231
|
+
for m in catalog["models"]:
|
|
232
|
+
if m.get("modelId") == model_id:
|
|
233
|
+
md = m.get("metadata", {})
|
|
234
|
+
return bool(md.get("verifiable") and md.get("attestationSupported"))
|
|
235
|
+
return False
|
fuin/cli.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""fuin: check | seal | run | commit | close | reveal | status (owner) and verify | show (anyone, no credentials)."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import base64
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from fuin import diagnose, owner, tooling, verify
|
|
10
|
+
|
|
11
|
+
QWEN = "Qwen/Qwen3.8-27B"
|
|
12
|
+
SETTINGS = {"max_tokens": 4096, "temperature": 0} # user, Nov 3: both models reason, 4096-token cap
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _print_verdict(path: str, res: dict) -> None:
|
|
16
|
+
head = {"VERIFIED": "VERIFIED", "REFUSED": f"REFUSED {res.get('code')}", "CANT_CERTIFY": f"CAN'T CERTIFY {res.get('code')}"}
|
|
17
|
+
print(f"{path}: {head[res['outcome']]}")
|
|
18
|
+
for name, status, detail in res["checks"]:
|
|
19
|
+
print(f" [{status}] {name}: {detail}")
|
|
20
|
+
if res.get("detail"):
|
|
21
|
+
print(f" detail: {res['detail']}")
|
|
22
|
+
if res["outcome"] == "VERIFIED":
|
|
23
|
+
print(f" scope: {res['scope']}")
|
|
24
|
+
for w in res["warnings"]:
|
|
25
|
+
print(f" warning: {w}")
|
|
26
|
+
print(" limits:")
|
|
27
|
+
for lim in res["limits"]:
|
|
28
|
+
print(f" - {lim}")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def main(argv=None) -> int:
|
|
32
|
+
ap = argparse.ArgumentParser(prog="fuin")
|
|
33
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
34
|
+
s = sub.add_parser("seal"); s.add_argument("suite"); s.add_argument("--id", required=True)
|
|
35
|
+
s.add_argument("--owner", required=True); s.add_argument("--model", action="append", default=None)
|
|
36
|
+
s.add_argument("--max-tokens", type=int, default=SETTINGS["max_tokens"], help="sealed per-model token cap (reasoning included)")
|
|
37
|
+
s.add_argument("--tee", action="store_true", help="zero-reveal round: every item scored inside OutLayer's TEE by the current "
|
|
38
|
+
"pinned scorer build (sealed in the policy), requests bound to the seal block, nothing revealed")
|
|
39
|
+
s.add_argument("--e2ee", action="store_true", help="request template v3: each question and answer end-to-end encrypted "
|
|
40
|
+
"between you and the model's attested TEE key, so NEAR's gateway never sees them; requests bound to the seal block")
|
|
41
|
+
s.add_argument("--ask", action="store_true", help="zero-reveal round whose model calls are made inside OutLayer's TEE by the "
|
|
42
|
+
"current pinned asker build in its locked project, each item asked once (sealed in the policy); run `fuin ask`")
|
|
43
|
+
for c in ("run", "commit"):
|
|
44
|
+
p = sub.add_parser(c); p.add_argument("--id", required=True); p.add_argument("--model", default=QWEN)
|
|
45
|
+
if c == "run":
|
|
46
|
+
p.add_argument("--parallel", type=int, default=1, help="questions in flight at once (rows are still saved in order)")
|
|
47
|
+
p = sub.add_parser("close"); p.add_argument("--id", required=True)
|
|
48
|
+
p = sub.add_parser("reveal"); p.add_argument("--id", required=True); p.add_argument("--out", default="examples")
|
|
49
|
+
p = sub.add_parser("verify"); p.add_argument("certs", nargs="+"); p.add_argument("--json", action="store_true")
|
|
50
|
+
p = sub.add_parser("check", help="validate a suite and the prerequisites before sealing"); p.add_argument("suite")
|
|
51
|
+
p.add_argument("--owner"); p.add_argument("--model", action="append", default=None)
|
|
52
|
+
p.add_argument("--tee", action="store_true", help="state the disclosure for a zero-reveal (TEE-scored) round")
|
|
53
|
+
p = sub.add_parser("status", help="show where a round stands and the next command"); p.add_argument("--id", required=True)
|
|
54
|
+
p.add_argument("--out", default="examples", help="where the round's certificates were written (the --out of reveal or tee-score)")
|
|
55
|
+
p = sub.add_parser("show"); p.add_argument("cert"); p.add_argument("--audit", action="store_true")
|
|
56
|
+
p = sub.add_parser("policy-update", help="draft the next attestation policy from fresh reports when NEAR redeploys a model "
|
|
57
|
+
"(needs a key; writes policy/attestation-v<N+1>.json for review, approves nothing)")
|
|
58
|
+
p.add_argument("--model", action="append", default=None, help="model to check (default: every model in the current policy)")
|
|
59
|
+
p = sub.add_parser("tee-key", help="make the TEE scorer's in-enclave key for this owner (once per scorer build)")
|
|
60
|
+
p.add_argument("--owner", required=True)
|
|
61
|
+
p = sub.add_parser("tee-score", help="re-score a committed run inside OutLayer's TEE and add the evidence to its certificate")
|
|
62
|
+
p.add_argument("--id", required=True); p.add_argument("--model", required=True); p.add_argument("--out", default="examples")
|
|
63
|
+
p.add_argument("--host", required=True, choices=["fastfs", "gist"],
|
|
64
|
+
help="where the encrypted run is hosted for the scorer: fastfs (permanent, public) or gist (secret, deleted after)")
|
|
65
|
+
p = sub.add_parser("tee-ground", help="check every answer key against its cited near/docs line inside OutLayer's TEE (once per round)")
|
|
66
|
+
p.add_argument("--id", required=True); p.add_argument("--out", default="examples")
|
|
67
|
+
p.add_argument("--host", required=True, choices=["fastfs", "gist"],
|
|
68
|
+
help="where the encrypted items are hosted for the enclave: fastfs (permanent, public) or gist (secret, deleted after)")
|
|
69
|
+
p = sub.add_parser("ask", help="enclave-asked round: ask every item inside the TEE, then have it attest the run")
|
|
70
|
+
p.add_argument("--id", required=True); p.add_argument("--model", required=True)
|
|
71
|
+
p.add_argument("--host", required=True, choices=["fastfs", "gist"], help="where the encrypted round is hosted for the asker")
|
|
72
|
+
p = sub.add_parser("ask-check", help="enclave-asked round: have the TEE rebuild every request with the true seal block hash")
|
|
73
|
+
p.add_argument("--id", required=True); p.add_argument("--model", required=True); p.add_argument("--out", default="examples")
|
|
74
|
+
p.add_argument("--host", required=True, choices=["fastfs", "gist"])
|
|
75
|
+
a = ap.parse_args(argv)
|
|
76
|
+
|
|
77
|
+
if a.cmd == "check":
|
|
78
|
+
return tooling.print_check(a.suite, a.owner, a.model or [QWEN], tee=a.tee)
|
|
79
|
+
if a.cmd == "status":
|
|
80
|
+
print("\n".join(tooling.status(a.id, a.out)))
|
|
81
|
+
return 0
|
|
82
|
+
held = owner.lock(a.id) if a.cmd in ("run", "commit", "close", "tee-score", "tee-ground", "reveal", "ask", "ask-check") else None # noqa: F841 (held until exit)
|
|
83
|
+
if a.cmd == "seal":
|
|
84
|
+
res = tooling.check_suite(a.suite)
|
|
85
|
+
if res["errors"]: # a seal can't be undone: refuse a broken suite
|
|
86
|
+
print("Not sealing: the suite has errors. Run `fuin check` for details.\n" + "\n".join(" ✗ " + e for e in res["errors"][:10]))
|
|
87
|
+
return 1
|
|
88
|
+
print(tooling.disclosure(res["n"], tee=a.tee or a.ask))
|
|
89
|
+
tee_wasm, ask_with = None, None
|
|
90
|
+
if a.tee and a.ask:
|
|
91
|
+
raise SystemExit("--tee and --ask are two kinds of zero-reveal round: pick one")
|
|
92
|
+
if a.tee:
|
|
93
|
+
from fuin import tee
|
|
94
|
+
tee_wasm = tee.tee_policy()["current"]
|
|
95
|
+
if a.ask:
|
|
96
|
+
from fuin import asker
|
|
97
|
+
if not asker.askers():
|
|
98
|
+
raise SystemExit("policy/tee-v2.json pins no asker build")
|
|
99
|
+
ask_with = (asker.askers()[-1]["wasm_sha256"], asker.askers()[-1]["project"])
|
|
100
|
+
owner.seal(a.suite, a.id, a.owner, {m: {**SETTINGS, "max_tokens": a.max_tokens} for m in (a.model or [QWEN])}, tee_wasm=tee_wasm,
|
|
101
|
+
encrypt=a.e2ee, asker=ask_with)
|
|
102
|
+
elif a.cmd == "ask":
|
|
103
|
+
from fuin import asker, tee
|
|
104
|
+
st = owner.load(a.id)
|
|
105
|
+
host = tee.fastfs_host(st["owner"]) if a.host == "fastfs" else tee.gist_host
|
|
106
|
+
st = asker.run(a.id, a.model, host)
|
|
107
|
+
o = st["runs"][a.model]["tee_asked"]["output"]
|
|
108
|
+
print(f"every item asked once inside the TEE: {o['total']}/{o['n']}, {o['retried']} retried asks "
|
|
109
|
+
f"(aggregate tx {st['runs'][a.model]['tee_asked']['transaction_hash']}); next: fuin commit")
|
|
110
|
+
elif a.cmd == "ask-check":
|
|
111
|
+
from fuin import asker, tee
|
|
112
|
+
st = owner.load(a.id)
|
|
113
|
+
host = tee.fastfs_host(st["owner"]) if a.host == "fastfs" else tee.gist_host
|
|
114
|
+
ev = st["runs"][a.model]["tee_asked_check"] = asker.check(st, a.model, host)
|
|
115
|
+
owner.save(st)
|
|
116
|
+
print(f"every request rebuilt with the seal block's hash matches the signed one (task {ev['record']['task_id']})")
|
|
117
|
+
if st.get("close"):
|
|
118
|
+
for path in owner.reveal(a.id, a.out):
|
|
119
|
+
print(path)
|
|
120
|
+
elif a.cmd == "run":
|
|
121
|
+
owner.run(a.id, a.model, a.parallel)
|
|
122
|
+
elif a.cmd == "commit":
|
|
123
|
+
owner.commit(a.id, a.model)
|
|
124
|
+
elif a.cmd == "close":
|
|
125
|
+
owner.close(a.id)
|
|
126
|
+
elif a.cmd == "reveal":
|
|
127
|
+
for p in owner.reveal(a.id, a.out):
|
|
128
|
+
print(p)
|
|
129
|
+
elif a.cmd == "policy-update":
|
|
130
|
+
from fuin import policyupdate
|
|
131
|
+
draft, notes = policyupdate.propose(a.model or sorted(owner.attestation_policy()["models"]))
|
|
132
|
+
print("\n".join(notes))
|
|
133
|
+
if draft is None:
|
|
134
|
+
print("nothing new: every candidate is already approved or was not proposed")
|
|
135
|
+
else:
|
|
136
|
+
path = policyupdate.write(draft)
|
|
137
|
+
print(f"wrote {path} for review; the verifier keeps using {owner.ATTESTATION_POLICY.name} until "
|
|
138
|
+
"owner.ATTESTATION_POLICY points at the reviewed file")
|
|
139
|
+
elif a.cmd == "tee-key":
|
|
140
|
+
from fuin import tee
|
|
141
|
+
k = tee.keygen(a.owner)
|
|
142
|
+
print(f"in-TEE key for {a.owner}: pub {k['pub']} (keygen tx {k['transaction_hash']})")
|
|
143
|
+
elif a.cmd == "tee-score":
|
|
144
|
+
from fuin import tee
|
|
145
|
+
st = owner.load(a.id)
|
|
146
|
+
if "commit" not in st["runs"].get(a.model, {}):
|
|
147
|
+
raise SystemExit(f"{a.model} run is not committed yet")
|
|
148
|
+
host = tee.fastfs_host(st["owner"]) if a.host == "fastfs" else tee.gist_host
|
|
149
|
+
ev = tee.score(st, a.model, st["owner"], host)
|
|
150
|
+
st["runs"][a.model]["tee_score"] = ev
|
|
151
|
+
owner.save(st)
|
|
152
|
+
print(f"TEE score {ev['output']['total']}/{ev['output']['n']} matches the commitment (tx {ev['transaction_hash']})")
|
|
153
|
+
from fuin import policy as P
|
|
154
|
+
waiting = [m for m, rn in st["runs"].items() if "commit" in rn and not rn.get("tee_score")]
|
|
155
|
+
if P.is_tee_round(st["policy"]) and waiting: # a zero-reveal round's certificates need every run's TEE score
|
|
156
|
+
print(f"certificates are written once every run has its TEE score; still to score: {', '.join(waiting)}")
|
|
157
|
+
elif st.get("close"): # a closed round's certificates are regenerated with the evidence; the sample is re-derived, unchanged
|
|
158
|
+
for p in owner.reveal(a.id, a.out):
|
|
159
|
+
print(p)
|
|
160
|
+
elif a.cmd == "tee-ground":
|
|
161
|
+
from fuin import policy as P, tee
|
|
162
|
+
st = owner.load(a.id)
|
|
163
|
+
if "seal" not in st:
|
|
164
|
+
raise SystemExit("round not sealed yet")
|
|
165
|
+
host = tee.fastfs_host(st["owner"]) if a.host == "fastfs" else tee.gist_host
|
|
166
|
+
st["tee_ground"] = ev = tee.ground(st, st["owner"], host)
|
|
167
|
+
owner.save(st)
|
|
168
|
+
o = ev["output"]
|
|
169
|
+
print(f"answer keys grounded in the TEE: {o['n'] - o['flagged']}/{o['n']} appear in their cited line; "
|
|
170
|
+
f"{o['flagged']} flagged (tx {ev['transaction_hash']})")
|
|
171
|
+
if st.get("close") and all("commit" not in rn or rn.get("tee_score") or rn.get("tee_asked") or not P.is_tee_round(st["policy"]) for rn in st["runs"].values()):
|
|
172
|
+
for path in owner.reveal(a.id, a.out): # certificates regenerated with the grounding; nothing else changes
|
|
173
|
+
print(path)
|
|
174
|
+
elif a.cmd == "verify":
|
|
175
|
+
outcomes = set()
|
|
176
|
+
for path in a.certs:
|
|
177
|
+
raw = b""
|
|
178
|
+
try:
|
|
179
|
+
raw = open(path, "rb").read()
|
|
180
|
+
cert = json.loads(raw.decode("utf-8"))
|
|
181
|
+
except (OSError, ValueError) as e:
|
|
182
|
+
res = {"outcome": "REFUSED", "code": "certificate_invalid", "detail": str(e), "checks": []}
|
|
183
|
+
else:
|
|
184
|
+
res = verify.verify(cert)
|
|
185
|
+
# the file's hash ties a recorded result to these exact bytes (scripts/build_page.py refuses a mismatch)
|
|
186
|
+
print(json.dumps({"cert": path, "cert_sha256": hashlib.sha256(raw).hexdigest(), **res}, default=list)) if a.json else _print_verdict(path, res)
|
|
187
|
+
outcomes.add(res["outcome"])
|
|
188
|
+
return 1 if "REFUSED" in outcomes else 2 if "CANT_CERTIFY" in outcomes else 0 # REFUSED outranks CAN'T CERTIFY
|
|
189
|
+
elif a.cmd == "show":
|
|
190
|
+
cert = json.loads(open(a.cert, encoding="utf-8").read())
|
|
191
|
+
b, r, au = cert["benchmark"], cert["run"], cert["audit"]
|
|
192
|
+
print(f"{b['id']} · {r['model_id']} run {r['run_index']} · committed total {r['total_score']}/{b['n']}")
|
|
193
|
+
from fuin import policy as P
|
|
194
|
+
if P.is_tee_round(b["policy"]):
|
|
195
|
+
print(f"audit: none · a TEE-scored round reveals nothing: all {b['n']} items stay private and were re-scored inside "
|
|
196
|
+
f"OutLayer's TEE by scorer {b['policy']['audit']['scorer_wasm_sha256'][:12]}… (fuin verify checks that evidence)")
|
|
197
|
+
return 0
|
|
198
|
+
print(f"audit: D={au['D']} k={au['k']} (target 95%) · revealed {au['k']} of {b['n']}, unrevealed {b['n'] - au['k']}")
|
|
199
|
+
print(f"settings: {b['policy']['models'][r['model_id']]} · scorer {b['policy']['scorer']['name']}")
|
|
200
|
+
items = {it["i"]: it for it in r["items"]}
|
|
201
|
+
rows = []
|
|
202
|
+
for x in au["reveals"]:
|
|
203
|
+
resp = json.loads(base64.b64decode(x["response_b64"]))
|
|
204
|
+
ch = resp["choices"][0]
|
|
205
|
+
rows.append((x, ch.get("finish_reason"), (ch["message"].get("content") or "").strip()))
|
|
206
|
+
cut = sum(f == "length" for _, f, _ in rows)
|
|
207
|
+
right = sum(items[x["i"]]["score"] for x, _, _ in rows)
|
|
208
|
+
print(f"revealed sample: {right}/{len(rows)} correct; {cut}/{len(rows)} cut off at max_tokens (scored 0) — checkable from the revealed bytes")
|
|
209
|
+
bd = diagnose.breakdown(cert)
|
|
210
|
+
print(f"revealed misses: {bd['wrong']} wrong, {bd['format']} format (right value, rejected form), {bd['cut off']} cut off — a reading aid, not a score")
|
|
211
|
+
if a.audit:
|
|
212
|
+
for x, fin, content in rows:
|
|
213
|
+
said = "(cut off before answering)" if fin == "length" and not content else repr(content[:50])
|
|
214
|
+
print(f" #{x['i']:>2} score {items[x['i']]['score']} · Q: {x['item']['question']}\n expected {x['item']['answer']!r} · model said {said}"
|
|
215
|
+
f" · source {x['item'].get('source', {}).get('url', '-')}")
|
|
216
|
+
return 0
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
if __name__ == "__main__":
|
|
220
|
+
sys.exit(main())
|