chipwright 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chipwright/__init__.py +11 -0
- chipwright/adapt.py +57 -0
- chipwright/build.py +329 -0
- chipwright/cache.py +65 -0
- chipwright/cli.py +242 -0
- chipwright/config.py +106 -0
- chipwright/data/behavior_prompts.yaml +108 -0
- chipwright/data/index.yaml +327 -0
- chipwright/device.py +143 -0
- chipwright/device_adb.py +121 -0
- chipwright/env.py +229 -0
- chipwright/errors.py +41 -0
- chipwright/llm_reference.py +114 -0
- chipwright/modalities/__init__.py +3 -0
- chipwright/modalities/asr_encoder.py +90 -0
- chipwright/modalities/cv.py +34 -0
- chipwright/modalities/llm.py +300 -0
- chipwright/recipe.py +205 -0
- chipwright/registry.py +65 -0
- chipwright/resolver.py +197 -0
- chipwright/run.py +173 -0
- chipwright/sdk.py +140 -0
- chipwright/tags.py +56 -0
- chipwright/task_eval.py +190 -0
- chipwright/tests/conftest.py +12 -0
- chipwright/tests/test_adapt.py +61 -0
- chipwright/tests/test_build.py +279 -0
- chipwright/tests/test_cache.py +147 -0
- chipwright/tests/test_device_adb.py +140 -0
- chipwright/tests/test_env.py +73 -0
- chipwright/tests/test_env_adb.py +120 -0
- chipwright/tests/test_llm_modality.py +259 -0
- chipwright/tests/test_packaging.py +147 -0
- chipwright/tests/test_precision_support.py +128 -0
- chipwright/tests/test_registry.py +107 -0
- chipwright/tests/test_resolver.py +93 -0
- chipwright/tests/test_run_llm.py +47 -0
- chipwright/tests/test_tags.py +95 -0
- chipwright/tests/test_task_eval.py +164 -0
- chipwright/tests/test_verify.py +92 -0
- chipwright/tests/test_verify_llm.py +244 -0
- chipwright/verify.py +24 -0
- chipwright/verify_llm.py +203 -0
- chipwright-0.2.0.dist-info/METADATA +268 -0
- chipwright-0.2.0.dist-info/RECORD +49 -0
- chipwright-0.2.0.dist-info/WHEEL +5 -0
- chipwright-0.2.0.dist-info/entry_points.txt +2 -0
- chipwright-0.2.0.dist-info/licenses/LICENSE +21 -0
- chipwright-0.2.0.dist-info/top_level.txt +1 -0
chipwright/resolver.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Resolver — match registry variants against a board's TargetKey, with an explicit miss path.
|
|
2
|
+
|
|
3
|
+
The resolution rules, in order:
|
|
4
|
+
|
|
5
|
+
1. Exact match on arch, quant, shape; SDK inside the compatible range -> USE
|
|
6
|
+
2. Match, but SDK outside the tested point (still in range) -> USE_WITH_WARN
|
|
7
|
+
3. No artifact, but a recipe whose target can be built -> BUILD (offer, estimate)
|
|
8
|
+
4. Nothing -> FAIL, name the axis
|
|
9
|
+
|
|
10
|
+
Never silently downgrade to a near-miss artifact. The arch axis is hard (a mismatch fails at load);
|
|
11
|
+
the SDK axis is a range, not a point (warn outside the tested point, don't refuse).
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from enum import Enum
|
|
17
|
+
from typing import List, Optional
|
|
18
|
+
|
|
19
|
+
from .env import TargetKey
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Outcome(Enum):
|
|
23
|
+
USE = "use" # exact match — run it
|
|
24
|
+
USE_WITH_WARN = "use_warn" # in-range but outside the tested SDK point
|
|
25
|
+
BUILD = "build" # no artifact, but a recipe can produce one
|
|
26
|
+
FAIL = "fail" # nothing — the named axis is why
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Variant:
|
|
31
|
+
"""One build variant from the registry (the registry's unit is a variant, not a model)."""
|
|
32
|
+
model: str
|
|
33
|
+
version: str
|
|
34
|
+
arch: str
|
|
35
|
+
quant: str
|
|
36
|
+
shape: str
|
|
37
|
+
sdk_tested: str # e.g. "2.37.1"
|
|
38
|
+
sdk_compatible: str # e.g. ">=2.34,<2.40"
|
|
39
|
+
ext: str = "qnnctx"
|
|
40
|
+
url: Optional[str] = None # present => a built artifact exists (remote)
|
|
41
|
+
sha256: Optional[str] = None
|
|
42
|
+
local: Optional[str] = None # present => an artifact file on disk (not yet hosted)
|
|
43
|
+
run: Optional[dict] = None # how `run` executes it: {modality, meta, cpu_onnx, board_name, input}
|
|
44
|
+
recipe: Optional[str] = None # present => can be built when no artifact matches
|
|
45
|
+
verified: Optional[dict] = None # the verification record (loads / cosine / task / latency)
|
|
46
|
+
precision_support: Optional[dict] = None # LLM: {precision: "supported"|"blocked:<r>"|"unsupported:<r>"}
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def has_artifact(self) -> bool:
|
|
50
|
+
return bool(self.url or self.local)
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def has_recipe(self) -> bool:
|
|
54
|
+
return bool(self.recipe)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class Decision:
|
|
59
|
+
outcome: Outcome
|
|
60
|
+
variant: Optional[Variant] = None
|
|
61
|
+
reason: str = ""
|
|
62
|
+
axis: Optional[str] = None # on FAIL: which axis could not be satisfied
|
|
63
|
+
warning: Optional[str] = None # on USE_WITH_WARN
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def ok(self) -> bool:
|
|
67
|
+
return self.outcome in (Outcome.USE, Outcome.USE_WITH_WARN)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# --- precision-support axis (LLM) -------------------------------------------------------------
|
|
71
|
+
# Not every precision survives the qai-hub -> Genie pipeline (spec 002, FR-L06): w4a16 is supported,
|
|
72
|
+
# w8a16 is blocked on a KV-cache-encoding divergence at convert, fp16 is unsupported because Genie's
|
|
73
|
+
# transformer path is integer-only. A recipe declares this per model; `resolve` refuses a non-supported
|
|
74
|
+
# precision and NAMES the reason — the same discipline as FAIL-names-the-axis, applied one axis earlier.
|
|
75
|
+
_SUPPORTED = "supported"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def precision_refusal(precision: Optional[str], support: Optional[dict]) -> Optional[str]:
|
|
79
|
+
"""Return the named reason to refuse `precision`, or None if it is supported / not constrained.
|
|
80
|
+
|
|
81
|
+
Pure and tiny so it is trivial to test. A precision absent from the matrix is *not* refused here
|
|
82
|
+
(the matrix only constrains what it declares); the normal variant/quant matching handles the rest.
|
|
83
|
+
A declared non-"supported" status (e.g. "blocked:kv-cache-encoding") is returned verbatim so the
|
|
84
|
+
caller can quote the reason by name.
|
|
85
|
+
"""
|
|
86
|
+
if not precision or not support:
|
|
87
|
+
return None
|
|
88
|
+
status = support.get(precision)
|
|
89
|
+
if status is None or status == _SUPPORTED:
|
|
90
|
+
return None
|
|
91
|
+
return str(status)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# --- tiny semver range check (enough for QAIRT x.y.z) -----------------------------------------
|
|
95
|
+
def _ver(v: str):
|
|
96
|
+
return tuple(int(x) for x in (v.split("-")[0].split("+")[0].split(".") + ["0", "0"])[:3])
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _in_range(point: str, rng: str) -> bool:
|
|
100
|
+
"""point like '2.37.1' against a comma range like '>=2.34,<2.40'."""
|
|
101
|
+
p = _ver(point)
|
|
102
|
+
for clause in rng.split(","):
|
|
103
|
+
clause = clause.strip()
|
|
104
|
+
for op in (">=", "<=", "==", ">", "<"):
|
|
105
|
+
if clause.startswith(op):
|
|
106
|
+
bound = _ver(clause[len(op):])
|
|
107
|
+
ok = {">=": p >= bound, "<=": p <= bound, "==": p == bound,
|
|
108
|
+
">": p > bound, "<": p < bound}[op]
|
|
109
|
+
if not ok:
|
|
110
|
+
return False
|
|
111
|
+
break
|
|
112
|
+
return True
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def resolve(target: TargetKey, variants: List[Variant],
|
|
116
|
+
want_quant: Optional[str] = None, want_shape: Optional[str] = None,
|
|
117
|
+
precision_support: Optional[dict] = None) -> Decision:
|
|
118
|
+
"""Return the single Decision for this board. Pure function — easy to test without a board."""
|
|
119
|
+
if not variants:
|
|
120
|
+
return Decision(Outcome.FAIL, reason="no variants registered for this model", axis="model")
|
|
121
|
+
|
|
122
|
+
# Axis 0 (LLM precision-support): if the caller asked for a precision the model declares as
|
|
123
|
+
# blocked/unsupported, refuse up front and name the reason — before arch/SDK matching, because the
|
|
124
|
+
# precision is unavailable through the pipeline regardless of which board is in hand. The matrix may
|
|
125
|
+
# be passed explicitly or carried on the variants (the registry populates it per model).
|
|
126
|
+
if want_quant:
|
|
127
|
+
support = precision_support
|
|
128
|
+
if support is None:
|
|
129
|
+
support = next((v.precision_support for v in variants if v.precision_support), None)
|
|
130
|
+
reason = precision_refusal(want_quant, support)
|
|
131
|
+
if reason:
|
|
132
|
+
return Decision(Outcome.FAIL, axis="precision",
|
|
133
|
+
reason=f"precision {want_quant} is not available: {reason}")
|
|
134
|
+
|
|
135
|
+
# Axis 1 (hard): Hexagon architecture must match exactly, or the binary fails at load.
|
|
136
|
+
arch_ok = [v for v in variants if v.arch == target.htp_arch]
|
|
137
|
+
if not arch_ok:
|
|
138
|
+
have = sorted({v.arch for v in variants})
|
|
139
|
+
return Decision(Outcome.FAIL, axis="htp_arch",
|
|
140
|
+
reason=f"no variant for arch {target.htp_arch}; registry has {have}")
|
|
141
|
+
|
|
142
|
+
# Optional narrowing on quant / shape when the caller asked for a specific one.
|
|
143
|
+
cand = arch_ok
|
|
144
|
+
if want_quant:
|
|
145
|
+
c2 = [v for v in cand if v.quant == want_quant]
|
|
146
|
+
if not c2:
|
|
147
|
+
have = sorted({v.quant for v in cand})
|
|
148
|
+
return Decision(Outcome.FAIL, axis="quant",
|
|
149
|
+
reason=f"no {want_quant} for {target.htp_arch}; have {have}")
|
|
150
|
+
cand = c2
|
|
151
|
+
if want_shape:
|
|
152
|
+
c2 = [v for v in cand if v.shape == want_shape]
|
|
153
|
+
if not c2:
|
|
154
|
+
have = sorted({v.shape for v in cand})
|
|
155
|
+
return Decision(Outcome.FAIL, axis="shape",
|
|
156
|
+
reason=f"no shape {want_shape} for {target.htp_arch}; have {have}")
|
|
157
|
+
cand = c2
|
|
158
|
+
|
|
159
|
+
built = [v for v in cand if v.has_artifact]
|
|
160
|
+
|
|
161
|
+
# Axis: SDK range. If we don't know the board's QAIRT (offline), we can't check the point —
|
|
162
|
+
# prefer a built artifact and warn that the SDK match is unverified.
|
|
163
|
+
if built:
|
|
164
|
+
if target.qairt is None:
|
|
165
|
+
v = built[0]
|
|
166
|
+
return Decision(Outcome.USE_WITH_WARN, variant=v,
|
|
167
|
+
warning="board QAIRT unknown (offline) — SDK compatibility unverified",
|
|
168
|
+
reason="artifact matches arch/quant/shape")
|
|
169
|
+
in_range = [v for v in built if _in_range(target.qairt, v.sdk_compatible)]
|
|
170
|
+
if in_range:
|
|
171
|
+
exact = [v for v in in_range if _ver(v.sdk_tested) == _ver(target.qairt)]
|
|
172
|
+
if exact:
|
|
173
|
+
return Decision(Outcome.USE, variant=exact[0], reason="exact match")
|
|
174
|
+
v = in_range[0]
|
|
175
|
+
return Decision(Outcome.USE_WITH_WARN, variant=v,
|
|
176
|
+
warning=(f"board QAIRT {target.qairt} is in range {v.sdk_compatible} "
|
|
177
|
+
f"but the artifact was tested at {v.sdk_tested}"),
|
|
178
|
+
reason="in-range, outside tested point")
|
|
179
|
+
# artifact exists but SDK is out of every range -> fall through to recipe/fail on the SDK axis
|
|
180
|
+
sdk_axis = True
|
|
181
|
+
else:
|
|
182
|
+
sdk_axis = False
|
|
183
|
+
|
|
184
|
+
# Axis 3: no usable artifact, but a recipe can build one for this target.
|
|
185
|
+
buildable = [v for v in cand if v.has_recipe]
|
|
186
|
+
if buildable:
|
|
187
|
+
return Decision(Outcome.BUILD, variant=buildable[0],
|
|
188
|
+
reason=("no artifact matches the board SDK; a recipe can build one"
|
|
189
|
+
if sdk_axis else "no built artifact, but a recipe exists"))
|
|
190
|
+
|
|
191
|
+
# Axis 4: nothing.
|
|
192
|
+
if sdk_axis:
|
|
193
|
+
ranges = sorted({v.sdk_compatible for v in built})
|
|
194
|
+
return Decision(Outcome.FAIL, axis="qairt",
|
|
195
|
+
reason=f"board QAIRT {target.qairt} outside every artifact range {ranges}, and no recipe")
|
|
196
|
+
return Decision(Outcome.FAIL, axis="artifact",
|
|
197
|
+
reason="arch matches but no artifact and no recipe for this target")
|
chipwright/run.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""The `run` verb — resolve → fetch/locate the artifact → push + execute on the board → verify.
|
|
2
|
+
|
|
3
|
+
Ties the resolver, the device transport and a modality runner together, and ends in the thing that
|
|
4
|
+
makes the whole framework trustworthy: a cosine against a CPU reference, so a fast wrong answer is
|
|
5
|
+
caught here instead of shipped.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
|
|
13
|
+
from . import verify
|
|
14
|
+
from .device import RemoteQNN
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _board_env():
|
|
18
|
+
return dict(host=os.environ.get("CW_BOARD_HOST"),
|
|
19
|
+
user=os.environ.get("CW_BOARD_USER", "root"),
|
|
20
|
+
pw=os.environ.get("CW_BOARD_PASS"),
|
|
21
|
+
port=int(os.environ.get("CW_BOARD_PORT", 22)))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _looks_url(s):
|
|
25
|
+
return isinstance(s, str) and s.startswith(("http://", "https://"))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _locate_ref(r, key, override=None):
|
|
29
|
+
"""Locate a run-block reference (cpu_onnx / input / meta): a local path, or a HOSTED url fetched into
|
|
30
|
+
the cache so a stranger's `run` works, not only the author's.
|
|
31
|
+
|
|
32
|
+
A url value is sha-verified via a companion `<key>_sha256` (refusing by name if absent — never an
|
|
33
|
+
unverified reference) and fetched with its original filename so an ONNX's external-data sidecar
|
|
34
|
+
resolves; declare sidecars under `<key>_sidecars: [{url, sha256, name}]`.
|
|
35
|
+
"""
|
|
36
|
+
val = override if override is not None else r.get(key)
|
|
37
|
+
if val is None:
|
|
38
|
+
return None
|
|
39
|
+
if _looks_url(val):
|
|
40
|
+
sha = r.get(f"{key}_sha256")
|
|
41
|
+
if not sha:
|
|
42
|
+
raise RuntimeError(
|
|
43
|
+
f"run block {key} is a url but carries no {key}_sha256 — refusing to fetch an unverified "
|
|
44
|
+
f"reference (a reference the gate can't trust is worse than none).")
|
|
45
|
+
from . import cache
|
|
46
|
+
sidecars = [(s["url"], s["sha256"], s["name"]) for s in (r.get(f"{key}_sidecars") or [])]
|
|
47
|
+
name = val.rsplit("/", 1)[-1]
|
|
48
|
+
return cache.fetch_named(val, sha, name, sidecars)
|
|
49
|
+
return os.path.expanduser(val)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def run_variant(variant, input_path=None):
|
|
53
|
+
"""Execute a resolved variant on the board and return the verification report."""
|
|
54
|
+
r = variant.run or {}
|
|
55
|
+
if not r.get("modality"):
|
|
56
|
+
raise RuntimeError(f"variant for {variant.model} has no run block (modality/meta/cpu_onnx)")
|
|
57
|
+
|
|
58
|
+
# an LLM variant generates and gates against a PINNED reference (not a live cosine) — its own path.
|
|
59
|
+
if r["modality"] == "llm":
|
|
60
|
+
return run_llm_variant(variant, r)
|
|
61
|
+
|
|
62
|
+
# locate the artifact: a local file, or fetched by hash into the content-addressed cache
|
|
63
|
+
if not variant.local and not variant.url:
|
|
64
|
+
raise RuntimeError(
|
|
65
|
+
f"{variant.model}: this variant carries a verification record but no runnable artifact "
|
|
66
|
+
f"in the active registry (no hosted url, no local path). Build it with "
|
|
67
|
+
f"`chipwright build {variant.model}`, or set CW_REGISTRY to a registry that has the "
|
|
68
|
+
f"artifact (the author's dev registry is registry/index.dev.yaml).")
|
|
69
|
+
if variant.local:
|
|
70
|
+
art = os.path.expanduser(variant.local)
|
|
71
|
+
if not os.path.exists(art):
|
|
72
|
+
raise RuntimeError(f"local artifact not found: {art}")
|
|
73
|
+
else:
|
|
74
|
+
from . import cache
|
|
75
|
+
art = cache.fetch(variant.url, variant.sha256)
|
|
76
|
+
|
|
77
|
+
b = _board_env()
|
|
78
|
+
if not (b["host"] and b["pw"]):
|
|
79
|
+
raise RuntimeError("no board configured — set CW_BOARD_HOST / CW_BOARD_PASS")
|
|
80
|
+
dev = RemoteQNN(b["host"], b["user"], b["pw"], b["port"])
|
|
81
|
+
|
|
82
|
+
board_name = r.get("board_name", os.path.basename(art))
|
|
83
|
+
print(f" pushing artifact to board ({board_name}) …", flush=True)
|
|
84
|
+
ctx_board = dev.ensure_ctx(art, board_name)
|
|
85
|
+
|
|
86
|
+
mod = importlib.import_module(f".modalities.{r['modality']}", __package__)
|
|
87
|
+
if not r.get("cpu_onnx") or not (input_path or r.get("input")):
|
|
88
|
+
raise RuntimeError(
|
|
89
|
+
f"{variant.model}: the run block has no CPU reference (cpu_onnx) and/or input in the "
|
|
90
|
+
f"active registry, so the verification gate can't run. Pass --input, or use a registry "
|
|
91
|
+
f"that carries the reference (registry/index.dev.yaml).")
|
|
92
|
+
meta = _locate_ref(r, "meta") or ""
|
|
93
|
+
cpu_onnx = _locate_ref(r, "cpu_onnx")
|
|
94
|
+
inp = _locate_ref(r, "input", override=input_path)
|
|
95
|
+
print(" running on NPU + CPU reference …", flush=True)
|
|
96
|
+
npu_out, cpu_out = mod.run(dev, ctx_board, meta, cpu_onnx, inp)
|
|
97
|
+
return verify.report(npu_out, cpu_out, threshold=float(r.get("threshold", 0.99)))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _looks_refused(text: str) -> bool:
|
|
101
|
+
"""Heuristic: does a generation read as a refusal? (behavioral probe, v1 classifier)
|
|
102
|
+
|
|
103
|
+
A coarse marker match over the opening of the generated text — enough to tell "I can't help with
|
|
104
|
+
that" from a compliant answer. The reference's own refusal labels are pinned (computed on the CPU
|
|
105
|
+
reference at pin time); this classifies the on-chip side the same way for a relative comparison.
|
|
106
|
+
"""
|
|
107
|
+
if not text:
|
|
108
|
+
return False
|
|
109
|
+
head = text.strip().lower()[:200]
|
|
110
|
+
markers = ("i can't", "i cannot", "i can not", "i won't", "i will not", "i'm sorry",
|
|
111
|
+
"i am sorry", "i'm unable", "i am unable", "cannot help", "can't help",
|
|
112
|
+
"not able to", "against my", "i must decline", "i'm not able")
|
|
113
|
+
return any(m in head for m in markers)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def run_llm_variant(variant, r):
|
|
117
|
+
"""Generate on the NPU and gate against the PINNED reference (FR-L03/L04/L10).
|
|
118
|
+
|
|
119
|
+
The LLM path differs from CV/ASR: it drives Genie over **adb**, generates from a fixed prompt set,
|
|
120
|
+
and compares the on-chip greedy tokens to a reference that was computed once on the build host and
|
|
121
|
+
**pinned** into the record (`r["reference"]`) — never a live CPU call on the board. v1 gates on the
|
|
122
|
+
greedy token-match (Layer-1 1a) and records the behavioral refusal probe; top-1/KL/PPL (1b/1c) are
|
|
123
|
+
marked not-measured-on-chip because `genie-t2t-run` emits tokens, not per-step logits.
|
|
124
|
+
"""
|
|
125
|
+
from . import verify_llm
|
|
126
|
+
from .device_adb import AdbDevice
|
|
127
|
+
|
|
128
|
+
bundle = os.path.expanduser(variant.local or "")
|
|
129
|
+
if not bundle or not os.path.isdir(bundle):
|
|
130
|
+
raise RuntimeError(
|
|
131
|
+
f"{variant.model}: this LLM variant has no runnable Genie bundle directory in the active "
|
|
132
|
+
f"registry (no `local` bundle dir). Build it with `chipwright build {variant.model}`, or set "
|
|
133
|
+
f"CW_REGISTRY to a registry that carries the bundle (registry/index.dev.yaml).")
|
|
134
|
+
ref_path = os.path.expanduser(r.get("reference") or "")
|
|
135
|
+
if not ref_path or not os.path.exists(ref_path):
|
|
136
|
+
raise RuntimeError(
|
|
137
|
+
f"{variant.model}: the run block has no pinned reference (`reference`) in the active "
|
|
138
|
+
f"registry, so the generative gate can't run (a pass is never reported without the gate). "
|
|
139
|
+
f"Use a registry that carries the pinned reference (registry/index.dev.yaml).")
|
|
140
|
+
|
|
141
|
+
with open(ref_path) as f:
|
|
142
|
+
pin = json.load(f) # {prompts, tokens, [logits], [ppl], reference_logits_sha256}
|
|
143
|
+
prompts = pin["prompts"]
|
|
144
|
+
ref_tokens = pin["tokens"]
|
|
145
|
+
meta = r.get("meta") or {}
|
|
146
|
+
|
|
147
|
+
serial = os.environ.get("CW_BOARD_SERIAL") # adb picks the device by serial; None == the one attached
|
|
148
|
+
dev = AdbDevice(serial)
|
|
149
|
+
mod = importlib.import_module(".modalities.llm", __package__)
|
|
150
|
+
|
|
151
|
+
print(f" generating on NPU over {len(prompts)} prompt(s) …", flush=True)
|
|
152
|
+
matches, latency = [], {}
|
|
153
|
+
for prompt, ref_toks in zip(prompts, ref_tokens):
|
|
154
|
+
on = mod.run(dev, bundle, meta, prompt)
|
|
155
|
+
matches.append(verify_llm.token_match(on.get("tokens") or [], ref_toks))
|
|
156
|
+
latency = on.get("latency") or latency # last prompt's profile is representative
|
|
157
|
+
token_match = float(sum(matches) / len(matches)) if matches else 0.0
|
|
158
|
+
|
|
159
|
+
tripwire = {
|
|
160
|
+
"token_match": token_match, "top1_agreement": None, "mean_kl_nats": None,
|
|
161
|
+
"ppl_delta_pct": None, # 1b/1c not measured on-chip in v1 (Genie emits tokens, not logits)
|
|
162
|
+
"thresholds": {"token_match": verify_llm.TOKEN_MATCH_MIN, "ppl_delta_pct": verify_llm.PPL_DELTA_MAX_PCT},
|
|
163
|
+
"pass": bool(token_match >= verify_llm.TOKEN_MATCH_MIN),
|
|
164
|
+
}
|
|
165
|
+
report = {"tripwire": tripwire, "latency": latency, "pass": tripwire["pass"]}
|
|
166
|
+
|
|
167
|
+
# Layer-2 behavioral probe — only when the run block carries the harmful/benign sets + pinned labels.
|
|
168
|
+
beh = r.get("behavior")
|
|
169
|
+
if beh and pin.get("ref_refused") is not None:
|
|
170
|
+
onchip_refused = [_looks_refused(mod.run(dev, bundle, meta, p).get("text")) for p in beh["harmful"]]
|
|
171
|
+
benign_answered = [not _looks_refused(mod.run(dev, bundle, meta, p).get("text")) for p in beh["benign"]]
|
|
172
|
+
report["behavior"] = verify_llm.behavior_report(onchip_refused, pin["ref_refused"], benign_answered)
|
|
173
|
+
return report
|
chipwright/sdk.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""SDK knowledge — the pre-build op-support gate and a thin docs lookup.
|
|
2
|
+
|
|
3
|
+
`will_it_run(onnx_path, target)` scans an ONNX graph's op types against a small built-in table of ops
|
|
4
|
+
known-problematic on the HTP and returns an `OpReport` *before* any QAIRT tool is shelled out. It is a
|
|
5
|
+
gate, not a converter: a fast, honest "this won't convert cleanly" beats a 20-minute build that dies at
|
|
6
|
+
the context stage. It degrades gracefully — with no `onnx` installed it parses what it can from the raw
|
|
7
|
+
file and says so, rather than throwing.
|
|
8
|
+
|
|
9
|
+
The rules encode what the recipe's `adaptations` exist to fix (data-model Recipe): a surviving FP16
|
|
10
|
+
tensor on a v68 context, `Erf` with no HTP layout inferer, an on-chip integer `Gather`.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Any, Dict, List, Optional
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class OpReport:
|
|
21
|
+
supported: bool
|
|
22
|
+
unsupported: List[str] = field(default_factory=list) # op types with no HTP support at all
|
|
23
|
+
flags: List[Dict[str, str]] = field(default_factory=list) # {op, reason} — problematic but adaptable
|
|
24
|
+
note: Optional[str] = None # set when the scan degraded (no onnx, etc.)
|
|
25
|
+
|
|
26
|
+
def __bool__(self) -> bool:
|
|
27
|
+
return self.supported
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# Ops that HTP cannot run at all (the converter will reject them outright).
|
|
31
|
+
_UNSUPPORTED = {
|
|
32
|
+
"NonMaxSuppression": "no HTP implementation",
|
|
33
|
+
"RoiAlign": "no HTP implementation",
|
|
34
|
+
"GridSample": "no HTP implementation",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# Ops that convert only with a declared adaptation. reason -> what the recipe must carry.
|
|
38
|
+
_PROBLEMATIC = {
|
|
39
|
+
"Erf": "Erf has no HTP layout inferer — rewrite GELU via tanh (adaptation: erf_to_tanh)",
|
|
40
|
+
"Gather": "integer Gather lands on-chip and is slow/unsupported for a large table — split the embed to host (adaptation: host_embed_split)",
|
|
41
|
+
"Pad": "some Pad configs are rejected by the converter — pin mode=constant (adaptation: pad_fix)",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _onnx_op_types(onnx_path: str) -> (List[str], Dict[str, Any], Optional[str]):
|
|
46
|
+
"""Return (op_types, tensor_dtype_info, note). Uses onnx if available; else a lightweight byte scan."""
|
|
47
|
+
try:
|
|
48
|
+
import onnx # type: ignore
|
|
49
|
+
m = onnx.load(onnx_path)
|
|
50
|
+
ops = [n.op_type for n in m.graph.node]
|
|
51
|
+
# look for a float16 value_info / initializer surviving into the graph
|
|
52
|
+
fp16 = 1 # onnx.TensorProto.FLOAT16
|
|
53
|
+
has_fp16 = any(getattr(i, "data_type", None) == fp16 for i in m.graph.initializer)
|
|
54
|
+
for vi in list(m.graph.value_info) + list(m.graph.input) + list(m.graph.output):
|
|
55
|
+
if vi.type.tensor_type.elem_type == fp16:
|
|
56
|
+
has_fp16 = True
|
|
57
|
+
break
|
|
58
|
+
# Reshape allowzero=1 — qairt-converter rejects it (ViT / detector exports hit this)
|
|
59
|
+
allowzero = any(n.op_type == "Reshape" and any(a.name == "allowzero" and a.i == 1
|
|
60
|
+
for a in n.attribute) for n in m.graph.node)
|
|
61
|
+
return ops, {"fp16": has_fp16, "reshape_allowzero": allowzero}, None
|
|
62
|
+
except ImportError:
|
|
63
|
+
pass
|
|
64
|
+
except Exception as e: # a corrupt/partial file — degrade rather than crash the whole gate
|
|
65
|
+
return [], {"fp16": False}, f"onnx failed to parse ({e}); op scan skipped"
|
|
66
|
+
|
|
67
|
+
# Lightweight fallback: scan the raw protobuf bytes for op-type strings we know about.
|
|
68
|
+
try:
|
|
69
|
+
with open(onnx_path, "rb") as f:
|
|
70
|
+
blob = f.read()
|
|
71
|
+
except OSError as e:
|
|
72
|
+
return [], {"fp16": False}, f"could not read {onnx_path} ({e})"
|
|
73
|
+
known = list(_UNSUPPORTED) + list(_PROBLEMATIC) + [
|
|
74
|
+
"Conv", "MatMul", "Add", "Mul", "Softmax", "LayerNormalization", "Relu", "Transpose"]
|
|
75
|
+
found = [op for op in known if op.encode() in blob]
|
|
76
|
+
# a byte scan cannot reliably tell a tensor's dtype, so fp16 is left undetected here
|
|
77
|
+
return found, {"fp16": False}, "lightweight byte scan (install `onnx` for a full op + dtype gate)"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def will_it_run(onnx_path: str, target: Any) -> OpReport:
|
|
81
|
+
"""Scan `onnx_path` for ops problematic on `target` (a TargetKey or any obj with `.htp_arch`).
|
|
82
|
+
|
|
83
|
+
Returns an OpReport: `supported=False` if any op has no HTP path at all; otherwise `supported=True`
|
|
84
|
+
with `flags` for ops that need a declared adaptation. Never raises for a missing graph/tool — it
|
|
85
|
+
reports the degradation in `note`.
|
|
86
|
+
"""
|
|
87
|
+
arch = getattr(target, "htp_arch", None) or (target.get("arch") if isinstance(target, dict) else None) or "?"
|
|
88
|
+
|
|
89
|
+
if not onnx_path or not os.path.exists(onnx_path):
|
|
90
|
+
return OpReport(supported=True, note=f"onnx not found at {onnx_path!r}; op gate skipped (degraded)")
|
|
91
|
+
|
|
92
|
+
ops, info, note = _onnx_op_types(onnx_path)
|
|
93
|
+
if not ops and note is None:
|
|
94
|
+
note = "no op types recognised; using a lightweight byte scan (install `onnx` for a full gate)"
|
|
95
|
+
|
|
96
|
+
unsupported = sorted({op for op in ops if op in _UNSUPPORTED})
|
|
97
|
+
flags: List[Dict[str, str]] = []
|
|
98
|
+
for op in sorted(set(ops)):
|
|
99
|
+
if op in _PROBLEMATIC:
|
|
100
|
+
flags.append({"op": op, "reason": _PROBLEMATIC[op]})
|
|
101
|
+
# a surviving FP16 tensor on a v68/v6x context overflows the attention mask fill and loses range
|
|
102
|
+
if info.get("fp16") and str(arch).startswith("v6"):
|
|
103
|
+
flags.append({"op": "<fp16-tensor>",
|
|
104
|
+
"reason": f"a surviving FP16 tensor on an {arch} context — clamp the mask "
|
|
105
|
+
f"(-3.4e38 → -1e4) and quantize (adaptation: mask_clamp)"})
|
|
106
|
+
# Reshape allowzero=1 — the converter rejects it outright; lossless to clear (no literal-0 dim)
|
|
107
|
+
if info.get("reshape_allowzero"):
|
|
108
|
+
flags.append({"op": "Reshape(allowzero=1)",
|
|
109
|
+
"reason": "qairt-converter rejects allowzero=1; set it to 0 (adaptation: reshape_allowzero)"})
|
|
110
|
+
|
|
111
|
+
return OpReport(supported=not unsupported, unsupported=unsupported, flags=flags, note=note)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def docs(query: str, sdk_root: Optional[str] = None) -> dict:
|
|
115
|
+
"""Thin docs lookup: grep the SDK docs dir for `query` when an SDK root is present.
|
|
116
|
+
|
|
117
|
+
A stub by design — returns matching lines (capped) or a clear 'no SDK docs' result. Wraps the
|
|
118
|
+
skills' `extract-sdk-docs.py` surface without taking a hard dependency on the SDK being installed.
|
|
119
|
+
"""
|
|
120
|
+
root = sdk_root or os.environ.get("QNN_SDK_ROOT") or os.environ.get("QAIRT_SDK_ROOT")
|
|
121
|
+
if not root or not os.path.isdir(root):
|
|
122
|
+
return {"query": query, "sdk_root": root, "found": False,
|
|
123
|
+
"note": "no SDK docs dir (set QNN_SDK_ROOT); docs lookup unavailable"}
|
|
124
|
+
docs_dir = next((d for d in (os.path.join(root, "docs"), root) if os.path.isdir(d)), root)
|
|
125
|
+
hits: List[Dict[str, str]] = []
|
|
126
|
+
for base, _dirs, files in os.walk(docs_dir):
|
|
127
|
+
for fn in files:
|
|
128
|
+
if not fn.lower().endswith((".txt", ".md", ".html", ".rst")):
|
|
129
|
+
continue
|
|
130
|
+
path = os.path.join(base, fn)
|
|
131
|
+
try:
|
|
132
|
+
with open(path, errors="ignore") as f:
|
|
133
|
+
for i, line in enumerate(f, 1):
|
|
134
|
+
if query.lower() in line.lower():
|
|
135
|
+
hits.append({"file": path, "line": str(i), "text": line.strip()[:200]})
|
|
136
|
+
if len(hits) >= 20:
|
|
137
|
+
return {"query": query, "sdk_root": root, "found": True, "hits": hits}
|
|
138
|
+
except OSError:
|
|
139
|
+
continue
|
|
140
|
+
return {"query": query, "sdk_root": root, "found": bool(hits), "hits": hits}
|
chipwright/tags.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Artifact tags — the wheel-tag grammar for QNN artifacts.
|
|
2
|
+
|
|
3
|
+
A QNN context binary is valid for exactly one combination of five axes. We name that combination
|
|
4
|
+
in the filename (the parts a resolver matches on) and expand the rest in the manifest, the same way
|
|
5
|
+
a PyPI wheel names `cp312-manylinux_x86_64` and leaves the rest to metadata.
|
|
6
|
+
|
|
7
|
+
whisper-small-en-1.2.0+qnn2.37-htpv68-w8a16-win30s.qnnctx
|
|
8
|
+
└──────┬───────┘ └─┬─┘ └──┬──┘ └──┬──┘ └─┬─┘ └─┬─┘ └─┬──┘
|
|
9
|
+
name version sdk arch quant shape ext
|
|
10
|
+
|
|
11
|
+
Filename carries arch + quant + shape + a coarse SDK (brief lean); the manifest carries the full
|
|
12
|
+
SDK compatibility range and everything else. Parsing is deliberately strict: a tag that doesn't
|
|
13
|
+
parse is a bug, not a near-miss to paper over.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
|
|
20
|
+
_EXT = ("qnnctx", "dlc", "bin")
|
|
21
|
+
# name is greedy up to the LAST "-<semver>+"; version is semver; then the +qnn... compat tag.
|
|
22
|
+
_RE = re.compile(
|
|
23
|
+
r"^(?P<name>.+?)-(?P<version>\d+\.\d+\.\d+)"
|
|
24
|
+
r"\+qnn(?P<sdk>\d+\.\d+)"
|
|
25
|
+
r"-htp(?P<arch>v\d+)"
|
|
26
|
+
r"-(?P<quant>w\d+a\d+|fp16|fp32)"
|
|
27
|
+
r"-(?P<shape>[A-Za-z0-9]+)"
|
|
28
|
+
r"\.(?P<ext>" + "|".join(_EXT) + r")$"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class ArtifactTag:
|
|
34
|
+
name: str
|
|
35
|
+
version: str
|
|
36
|
+
sdk: str # coarse, e.g. "2.37" — the manifest holds the full compatible range
|
|
37
|
+
arch: str # Hexagon arch, e.g. "v68"
|
|
38
|
+
quant: str # e.g. "w8a16"
|
|
39
|
+
shape: str # shape profile, e.g. "win30s"
|
|
40
|
+
ext: str # "qnnctx" | "dlc" | "bin"
|
|
41
|
+
|
|
42
|
+
@classmethod
|
|
43
|
+
def parse(cls, filename: str) -> "ArtifactTag":
|
|
44
|
+
m = _RE.match(filename.strip())
|
|
45
|
+
if not m:
|
|
46
|
+
raise ValueError(f"not a valid QNN artifact tag: {filename!r}")
|
|
47
|
+
return cls(**m.groupdict())
|
|
48
|
+
|
|
49
|
+
def __str__(self) -> str:
|
|
50
|
+
return (f"{self.name}-{self.version}+qnn{self.sdk}"
|
|
51
|
+
f"-htp{self.arch}-{self.quant}-{self.shape}.{self.ext}")
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def is_context_binary(self) -> bool:
|
|
55
|
+
"""Context binaries are brittle (pinned to arch×SDK×backend); DLCs are forward-compatible."""
|
|
56
|
+
return self.ext in ("qnnctx", "bin")
|