chipwright 0.2.4__tar.gz → 0.2.6__tar.gz

{chipwright-0.2.4 → chipwright-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chipwright
-Version: 0.2.4
+Version: 0.2.6
 Summary: A verified package manager for Qualcomm NPUs — resolve the artifact that loads on your exact chip, run it, and prove it against a CPU reference.
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/nameissakthi25/chipwright

{chipwright-0.2.4 → chipwright-0.2.6}/chipwright/__init__.py

@@ -8,4 +8,4 @@ from .resolver import Decision, Outcome, Variant, resolve
 from .tags import ArtifactTag
 
 __all__ = ["TargetKey", "probe", "Registry", "Decision", "Outcome", "Variant", "resolve", "ArtifactTag"]
-__version__ = "0.2.4"
+__version__ = "0.2.6"

{chipwright-0.2.4 → chipwright-0.2.6}/chipwright/cli.py

@@ -303,6 +303,26 @@ def cmd_verify(args):
     return _replay(reg, args.model)
 
 
+def _render_run_report(rep, d):
+    """Print a run's verification report (human door) and return its exit code. Shape-safe: the LLM
+    generative gate reports a `tripwire` (token-match), CV/ASR a cosine — the human door must not diverge
+    from the already-shape-safe `--json` projection (Principle IV, and never a KeyError traceback)."""
+    mark = f"{GREEN}✔ VERIFIED{RST}" if rep["pass"] else f"{RED}✗ MISMATCH{RST}"
+    if "tripwire" in rep:                        # LLM generative gate (token-match), not a cosine
+        tw = rep["tripwire"]
+        thr = (tw.get("thresholds") or {}).get("token_match")
+        print(f"\n{mark}  generative gate  token_match = {float(tw.get('token_match', 0.0)):.5f}"
+              + (f"  (threshold {thr})" if thr is not None else ""))
+        if rep.get("behavior"):
+            print(f"    behavioral probe: {rep['behavior']}")
+    else:
+        print(f"\n{mark}  NPU vs CPU-reference  cosine_min = {rep['cosine_min']:.5f}  "
+              f"(threshold {rep['threshold']})")
+        for name, c in rep["per_output"].items():
+            print(f"    {name}: {c:.5f}")
+    return jsonout.exit_code_for(d) if rep["pass"] else jsonout.EXIT_VERIFY_MISMATCH
+
+
 def cmd_run(args):
     reg = Registry(args.registry)
     if args.model not in reg.models():
@@ -333,12 +353,7 @@ def cmd_run(args):
     except Exception as e:
         print(f"{RED}run failed:{RST} {e}")
         return jsonout.EXIT_UNEXPECTED
-    mark = f"{GREEN}✔ VERIFIED{RST}" if rep["pass"] else f"{RED}✗ MISMATCH{RST}"
-    print(f"\n{mark}  NPU vs CPU-reference  cosine_min = {rep['cosine_min']:.5f}  "
-          f"(threshold {rep['threshold']})")
-    for name, c in rep["per_output"].items():
-        print(f"    {name}: {c:.5f}")
-    return jsonout.exit_code_for(d) if rep["pass"] else jsonout.EXIT_VERIFY_MISMATCH
+    return _render_run_report(rep, d)
 
 
 def _run_json(args, reg, target) -> int:

{chipwright-0.2.4 → chipwright-0.2.6}/chipwright/data/index.yaml

@@ -242,6 +242,12 @@ models:
         metric: cosine_min
         value: 0.9999
         note: 4x super-resolution; w8a8 upscaled image is near-lossless vs float
+        npu_output:
+          - name: upscaled_image
+            url: https://huggingface.co/nameissakthi/chipwright-cv/resolve/main/quicksrnet-small/npu_upscaled_image.npy
+            sha256: 8a4f01ec0b69e815cca4ef75bf50db87b3922dca2eb921f0328286516339c293
+            dtype: float32
+            shape: [1, 3, 512, 512]
       latency_ms:
         note: 128x128 -> 512x512; DLC compiled to context on-board
 - model: fcn-resnet50
@@ -280,6 +286,12 @@ models:
         metric: cosine_min
         value: 0.998
         note: per-pixel semantic mask; w8a8 near-lossless vs float
+        npu_output:
+          - name: mask
+            url: https://huggingface.co/nameissakthi/chipwright-cv/resolve/main/fcn-resnet50/npu_mask.npy
+            sha256: 39e8e3b0dc4ea9f81037242bc403755c16b9c08914652092e5028b1807b9b1dc
+            dtype: float32
+            shape: [1, 520, 520]
       latency_ms:
         note: 520x520; DLC compiled to context on-board (spills to DDR)
 - model: mobilenet-cv-built
@@ -346,6 +358,12 @@ models:
         backend: onnxruntime-cpu
         metric: depth_cosine
         value: 0.99996
+        npu_output:
+          - name: depth_estimates
+            url: https://huggingface.co/nameissakthi/chipwright-cv/resolve/main/depth-anything-v2/npu_depth_estimates.npy
+            sha256: 2ad552242a1152056d358b6b80a6967862c132a240d142fe59916051110dd8f8
+            dtype: float32
+            shape: [1, 1, 518, 518]
 - model: movenet
   source: qualcomm/MoveNet
   license: apache-2.0
@@ -381,3 +399,9 @@ models:
         backend: onnxruntime-cpu
         metric: keypoint_cosine
         value: 0.91607
+        npu_output:
+          - name: kpt_with_conf
+            url: https://huggingface.co/nameissakthi/chipwright-cv/resolve/main/movenet/npu_kpt_with_conf.npy
+            sha256: 37148303f83b4acbb1fee4de088921f33259d5fea7ab58b06703b28fccf39bca
+            dtype: float32
+            shape: [1, 1, 17, 3]

{chipwright-0.2.4 → chipwright-0.2.6}/chipwright/recipe.py

@@ -23,6 +23,13 @@ RUNS_ON = ("npu", "host")
 # through to the resolver's refusal verbatim.
 _PRECISION_KINDS = ("supported", "blocked", "unsupported")
 
+# Runtime-support statuses a recipe may declare (spec 004, FR-RT04). The runtime axis is the precision
+# axis a second time — the same {key: "supported" | "<kind>:<reason>"} matrix the resolver already refuses
+# on — only one level deeper, because a runtime's support is *per arch*: `{runtime: {arch: status}}`. A
+# status is the bare word "supported", or "deprecated:<successor>" (runnable, but deprecating upstream —
+# resolve warns), or "unsupported:<reason>" (the board's arch cannot host it — resolve refuses by name).
+_RUNTIME_KINDS = ("supported", "deprecated", "unsupported")
+
 
 @dataclass
 class Source:
@@ -86,6 +93,7 @@ class Recipe:
     host: Host = field(default_factory=Host)
     graphs: List[Graph] = field(default_factory=list)
     precision_support: Dict[str, str] = field(default_factory=dict)  # {precision: status} — LLM, FR-L06
+    runtime_support: Dict[str, Dict[str, str]] = field(default_factory=dict)  # {runtime: {arch: status}} — FR-RT04
 
     @property
     def quant_tag(self) -> str:
@@ -101,6 +109,15 @@ class Recipe:
         """The precisions this recipe declares runnable through the pipeline."""
         return [p for p, s in self.precision_support.items() if s == "supported"]
 
+    def runtime_status(self, runtime: str, arch: str) -> Optional[str]:
+        """The declared status for `runtime` on `arch` ('supported'/'deprecated:<s>'/'unsupported:<r>'), or None."""
+        return self.runtime_support.get(runtime, {}).get(arch)
+
+    def runnable_runtimes(self, arch: str) -> List[str]:
+        """The runtimes this recipe declares runnable on `arch` — 'supported' OR 'deprecated' (not 'unsupported')."""
+        return sorted(rt for rt, am in self.runtime_support.items()
+                      if str(am.get(arch, "")).split(":", 1)[0] in ("supported", "deprecated"))
+
     def adaptation(self, kind: str) -> Optional[Adaptation]:
         """The declared adaptation of a given kind, or None (e.g. recipe.adaptation('erf_to_tanh'))."""
         return next((a for a in self.adaptations if a.kind == kind), None)
@@ -153,6 +170,41 @@ def _parse_precision_support(items: Any) -> Dict[str, str]:
     return out
 
 
+def _parse_runtime_support(items: Any) -> Dict[str, Dict[str, str]]:
+    """Parse + validate `runtime_support:` — a nested map {runtime: {arch: status}}. Raises ValueError if malformed.
+
+    A near-clone of `_parse_precision_support`, one level deeper (runtime -> arch -> status) because a
+    runtime's support is per-arch. A status must be "supported" or "<deprecated|unsupported>:<tail>" with a
+    non-empty tail (the successor for `deprecated`, the reason for `unsupported`) — so a recipe cannot smuggle
+    in a vague "no": a limitation is named or it is not a valid recipe (the same strictness as the tag grammar).
+    """
+    if not items:
+        return {}
+    if not isinstance(items, dict):
+        raise ValueError(f"runtime_support must be a mapping {{runtime: {{arch: status}}}}, got {type(items).__name__}")
+    out: Dict[str, Dict[str, str]] = {}
+    for runtime, archmap in items.items():
+        if not isinstance(archmap, dict):
+            raise ValueError(
+                f"runtime_support[{runtime!r}] must be a mapping {{arch: status}}, got {type(archmap).__name__}")
+        per_arch: Dict[str, str] = {}
+        for arch, status in archmap.items():
+            status = str(status)
+            head = status.split(":", 1)[0]
+            if head not in _RUNTIME_KINDS:
+                raise ValueError(
+                    f"runtime_support[{runtime!r}][{arch!r}] = {status!r}: status must start with one of "
+                    f"{_RUNTIME_KINDS} (e.g. 'supported' or 'unsupported:htp-backend-excludes-v68')")
+            if head != "supported" and (":" not in status or not status.split(":", 1)[1].strip()):
+                raise ValueError(
+                    f"runtime_support[{runtime!r}][{arch!r}] = {status!r}: a {head!r} status must name a "
+                    f"reason/successor after the colon (e.g. 'deprecated:geniex-qairt' or "
+                    f"'unsupported:htp-backend-excludes-v68'); only 'supported' stands alone")
+            per_arch[str(arch)] = status
+        out[str(runtime)] = per_arch
+    return out
+
+
 def from_dict(doc: Dict[str, Any]) -> Recipe:
     """Build a Recipe from an already-parsed YAML mapping. Pure — no I/O. Raises ValueError if malformed."""
     if not isinstance(doc, dict):
@@ -195,7 +247,8 @@ def from_dict(doc: Dict[str, Any]) -> Recipe:
     return Recipe(name=str(doc["name"]), version=str(doc["version"]), route=route,
                   source=source, target=target, adaptations=_parse_adaptations(doc.get("adaptations")),
                   quant=quant, host=host, graphs=graphs,
-                  precision_support=_parse_precision_support(doc.get("precision_support")))
+                  precision_support=_parse_precision_support(doc.get("precision_support")),
+                  runtime_support=_parse_runtime_support(doc.get("runtime_support")))
 
 
 def load(path: str) -> Recipe:

{chipwright-0.2.4 → chipwright-0.2.6}/chipwright/registry.py

@@ -57,6 +57,8 @@ class Registry:
                     url=v.get("url"), sha256=v.get("sha256"),
                     local=v.get("local"), run=v.get("run"),
                     recipe=v.get("recipe"), verified=v.get("verified"),
+                    runtime=v.get("runtime"), runtime_support=v.get("runtime_support"),
+                    precision_support=v.get("precision_support"),
                 ))
             self._models[model] = vs
 

{chipwright-0.2.4 → chipwright-0.2.6}/chipwright/resolver.py

@@ -44,11 +44,19 @@ class Variant:
     recipe: Optional[str] = None         # present => can be built when no artifact matches
     verified: Optional[dict] = None      # the verification record (loads / cosine / task / latency)
     precision_support: Optional[dict] = None  # LLM: {precision: "supported"|"blocked:<r>"|"unsupported:<r>"}
+    runtime: Optional[str] = None        # the inference runtime this variant binds to (None => classic genie)
+    runtime_support: Optional[dict] = None    # {runtime: {arch: "supported"|"deprecated:<s>"|"unsupported:<r>"}} — FR-RT04
 
     @property
     def has_artifact(self) -> bool:
         return bool(self.url or self.local)
 
+    @property
+    def effective_runtime(self) -> str:
+        """The runtime this variant runs on. Historically every variant silently assumed classic Genie
+        (spec 004), so an unset runtime IS `genie` — the deprecating-but-default floor."""
+        return self.runtime or "genie"
+
     @property
     def has_recipe(self) -> bool:
         return bool(self.recipe)
@@ -91,6 +99,80 @@ def precision_refusal(precision: Optional[str], support: Optional[dict]) -> Opti
     return str(status)
 
 
+# --- runtime axis (spec 004) ------------------------------------------------------------------
+# The same shape as precision, one axis over. A model+board can be served by different inference runtimes
+# ({genie, geniex-qairt, llamacpp-htp}) with genuinely different arch + precision support (classic Genie is
+# w4a16-only and deprecating; the llama.cpp HTP backend does Q4_0/Q8_0 on v73+ but NOT v68). So `resolve`
+# must (a) refuse a runtime the board's arch cannot host, BY NAME (`axis=runtime`), and (b) among the
+# runtimes the board CAN host, pick the one matching the caller's priority. The support table is per-arch:
+# `{runtime: {arch: status}}`, validated in recipe.py (`_parse_runtime_support`).
+#
+# Priority -> the order runtimes are preferred in, among those runnable on the board's arch. Grounded in the
+# v73 measurement: llamacpp-htp Q8_0 is the quality/prefill win (WikiText PPL 11.66, near-lossless, the
+# >4-bit integer route Genie structurally lacks) but regresses decode (14.4 vs 22.5 tok/s); genie w4a16 is
+# the decode win and the default floor. So quality-first prefers llamacpp-htp, decode-latency-first prefers
+# genie, and the no-priority default is the conservative floor (genie) with a warn to choose explicitly.
+_RUNTIME_PRIORITY = {
+    "quality":        ("llamacpp-htp", "geniex-qairt", "genie"),
+    "decode-latency": ("genie", "geniex-qairt", "llamacpp-htp"),
+}
+_DEFAULT_RUNTIME_ORDER = ("genie", "geniex-qairt", "llamacpp-htp")
+_RUNNABLE = ("supported", "deprecated")   # 'unsupported' is the only hard refusal; 'deprecated' still runs
+
+
+def runtime_refusal(runtime: Optional[str], arch: str, support: Optional[dict]) -> Optional[str]:
+    """Return the named reason to refuse `runtime` on `arch`, or None if it is runnable / not constrained.
+
+    Pure and tiny, mirroring `precision_refusal`. A runtime absent from the matrix (or an arch the matrix
+    doesn't declare for it) is *not* refused here — the matrix only constrains what it declares. Only an
+    explicit `unsupported:<reason>` is a refusal; a `deprecated:<successor>` status still runs (resolve warns).
+    """
+    if not runtime or not support:
+        return None
+    archmap = support.get(runtime)
+    if not isinstance(archmap, dict):
+        return None
+    status = archmap.get(arch)
+    if status is None or str(status).split(":", 1)[0] != "unsupported":
+        return None
+    return str(status)
+
+
+def runnable_runtimes(arch: str, support: Optional[dict]) -> List[str]:
+    """The runtimes the matrix declares runnable on `arch` (status 'supported' or 'deprecated')."""
+    if not support:
+        return []
+    return sorted(rt for rt, am in support.items()
+                  if isinstance(am, dict) and str(am.get(arch, "")).split(":", 1)[0] in _RUNNABLE)
+
+
+def pick_runtime(runtimes: List[str], priority: Optional[str]):
+    """Choose one runtime from `runtimes` (all runnable on the board's arch) by `priority`.
+
+    Returns (chosen, note). `note` records the trade-off (which runtime was chosen over which) so the
+    Decision can carry it — tested-is-a-record, now across runtimes.
+    """
+    order = _RUNTIME_PRIORITY.get(priority)
+    if order:
+        for rt in order:
+            if rt in runtimes:
+                others = [r for r in runtimes if r != rt]
+                return rt, f"priority={priority}: chose runtime {rt} over {others}"
+    # no (or unknown) priority -> the conservative default floor, and say how to choose
+    for rt in _DEFAULT_RUNTIME_ORDER:
+        if rt in runtimes:
+            others = [r for r in runtimes if r != rt]
+            return rt, (f"multiple runtimes available {sorted(runtimes)}; defaulted to {rt} "
+                        f"(pass priority=quality|decode-latency to choose)" + (f"; alternatives {others}" if others else ""))
+    rt = sorted(runtimes)[0]                      # only unrecognised runtimes present
+    return rt, f"multiple runtimes {sorted(runtimes)}; defaulted to {rt}"
+
+
+def _merge_warn(*parts: Optional[str]) -> Optional[str]:
+    """Join the non-empty warning fragments, or None."""
+    return " | ".join(p for p in parts if p) or None
+
+
 # --- tiny semver range check (enough for QAIRT x.y.z) -----------------------------------------
 def _ver(v: str):
     return tuple(int(x) for x in (v.split("-")[0].split("+")[0].split(".") + ["0", "0"])[:3])
@@ -114,11 +196,31 @@ def _in_range(point: str, rng: str) -> bool:
 
 def resolve(target: TargetKey, variants: List[Variant],
             want_quant: Optional[str] = None, want_shape: Optional[str] = None,
-            precision_support: Optional[dict] = None) -> Decision:
+            precision_support: Optional[dict] = None,
+            want_runtime: Optional[str] = None, priority: Optional[str] = None,
+            runtime_support: Optional[dict] = None) -> Decision:
     """Return the single Decision for this board. Pure function — easy to test without a board."""
     if not variants:
         return Decision(Outcome.FAIL, reason="no variants registered for this model", axis="model")
 
+    # The runtime-support matrix may be passed explicitly (e.g. straight from the recipe) or carried on the
+    # variants (the registry populates it). Resolve it once so both the refusal and the selection use it.
+    rsupport = runtime_support
+    if rsupport is None:
+        rsupport = next((v.runtime_support for v in variants if v.runtime_support), None)
+
+    # Axis 0b (runtime refusal): mirror the precision refusal one axis over. If the caller names a runtime
+    # the board's arch cannot host, refuse up front and NAME it — listing the runtimes that ARE runnable on
+    # this arch — before arch/SDK matching, because the runtime is unavailable on this silicon regardless of
+    # which artifact is in hand (FAIL-names-the-axis, FR-RT03 / SC-RT02).
+    if want_runtime:
+        reason = runtime_refusal(want_runtime, target.htp_arch, rsupport)
+        if reason:
+            ok = runnable_runtimes(target.htp_arch, rsupport)
+            return Decision(Outcome.FAIL, axis="runtime",
+                            reason=f"runtime {want_runtime} is not available on {target.htp_arch}: {reason} "
+                                   f"(supported on {target.htp_arch}: {ok})")
+
     # Axis 0 (LLM precision-support): if the caller asked for a precision the model declares as
     # blocked/unsupported, refuse up front and name the reason — before arch/SDK matching, because the
     # precision is unavailable through the pipeline regardless of which board is in hand. The matrix may
@@ -156,6 +258,31 @@ def resolve(target: TargetKey, variants: List[Variant],
                             reason=f"no shape {want_shape} for {target.htp_arch}; have {have}")
         cand = c2
 
+    # Runtime axis (spec 004): pick WHICH runtime among the candidates runnable on this board's arch.
+    # If the caller named a runtime, narrow to it (its refusal was already handled above). Otherwise, only
+    # when the candidates span more than one runtime do we choose by priority — a single-runtime model
+    # resolves exactly as before, so nothing that pre-dates the runtime axis changes behaviour.
+    runtime_warning = None
+    if want_runtime:
+        c2 = [v for v in cand if v.effective_runtime == want_runtime]
+        if not c2:
+            have = sorted({v.effective_runtime for v in cand})
+            return Decision(Outcome.FAIL, axis="runtime",
+                            reason=f"runtime {want_runtime} has no variant for {target.htp_arch}; have {have}")
+        cand = c2
+    else:
+        runtimes_here = sorted({v.effective_runtime for v in cand})
+        if len(runtimes_here) > 1:
+            chosen, runtime_warning = pick_runtime(runtimes_here, priority)
+            cand = [v for v in cand if v.effective_runtime == chosen]
+    # Warn if the chosen runtime is marked deprecated on this arch (its successor is named in the matrix).
+    if rsupport and cand:
+        dep = rsupport.get(cand[0].effective_runtime, {})
+        dep = dep.get(target.htp_arch, "") if isinstance(dep, dict) else ""
+        if str(dep).split(":", 1)[0] == "deprecated":
+            runtime_warning = _merge_warn(runtime_warning,
+                                          f"runtime {cand[0].effective_runtime} on {target.htp_arch} is {dep}")
+
     built = [v for v in cand if v.has_artifact]
 
     # Axis: SDK range. If we don't know the board's QAIRT (offline), we can't check the point —
@@ -164,17 +291,18 @@ def resolve(target: TargetKey, variants: List[Variant],
         if target.qairt is None:
             v = built[0]
             return Decision(Outcome.USE_WITH_WARN, variant=v,
-                            warning="board QAIRT unknown (offline) — SDK compatibility unverified",
+                            warning=_merge_warn("board QAIRT unknown (offline) — SDK compatibility unverified",
+                                                runtime_warning),
                             reason="artifact matches arch/quant/shape")
         in_range = [v for v in built if _in_range(target.qairt, v.sdk_compatible)]
         if in_range:
             exact = [v for v in in_range if _ver(v.sdk_tested) == _ver(target.qairt)]
             if exact:
-                return Decision(Outcome.USE, variant=exact[0], reason="exact match")
+                return Decision(Outcome.USE, variant=exact[0], reason="exact match", warning=runtime_warning)
             v = in_range[0]
             return Decision(Outcome.USE_WITH_WARN, variant=v,
-                            warning=(f"board QAIRT {target.qairt} is in range {v.sdk_compatible} "
-                                     f"but the artifact was tested at {v.sdk_tested}"),
+                            warning=_merge_warn(f"board QAIRT {target.qairt} is in range {v.sdk_compatible} "
+                                                f"but the artifact was tested at {v.sdk_tested}", runtime_warning),
                             reason="in-range, outside tested point")
         # artifact exists but SDK is out of every range -> fall through to recipe/fail on the SDK axis
         sdk_axis = True
@@ -184,7 +312,7 @@ def resolve(target: TargetKey, variants: List[Variant],
     # Axis 3: no usable artifact, but a recipe can build one for this target.
     buildable = [v for v in cand if v.has_recipe]
     if buildable:
-        return Decision(Outcome.BUILD, variant=buildable[0],
+        return Decision(Outcome.BUILD, variant=buildable[0], warning=runtime_warning,
                         reason=("no artifact matches the board SDK; a recipe can build one"
                                 if sdk_axis else "no built artifact, but a recipe exists"))
 

chipwright-0.2.6/chipwright/tests/test_precision_registry.py

@@ -0,0 +1,68 @@
+"""Registry→resolve wiring for the LLM precision-support axis (FR-L06, Principle II).
+
+Analyzer finding (0.2.5): the precision-support matrix was parsed by the recipe and enforced by the
+resolver, but `Registry._load` never carried `precision_support` onto the Variant and no catalogue entry
+declared one — so the Axis-0 "refuse by name" could never fire through the real registry→resolve path,
+yet the suite was green (it only exercised hand-built Variants / an explicit kwarg). This test drives the
+REGISTRY path, so it would have failed before the fix (registry.py now threads precision_support).
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright.registry import Registry
+from chipwright.resolver import resolve, Outcome
+from chipwright.env import TargetKey
+
+_FIXTURE = """registry_schema: 1
+models:
+  - model: demo-llm
+    modality: llm
+    version: 1.0.0
+    variants:
+      - arch: v73
+        quant: w4a16
+        shape: ctx4096
+        sdk_tested: "2.37.1"
+        sdk_compatible: ">=2.34,<2.40"
+        ext: qnnctx
+        precision_support:
+          w4a16: supported
+          w8a16: "blocked:genie-kv-cache-quant-mismatch"
+          fp16: "unsupported:genie-engine-is-integer-only"
+        recipe: recipes/demo-llm/1.0.0.yaml
+"""
+
+
+def _registry(tmp_path):
+    p = tmp_path / "prec_index.yaml"
+    p.write_text(_FIXTURE)
+    return Registry(index=str(p))
+
+
+def test_registry_carries_precision_support_onto_variants(tmp_path):
+    vs = _registry(tmp_path).variants("demo-llm")
+    assert vs and vs[0].precision_support is not None, "registry must thread precision_support onto the Variant"
+    assert vs[0].precision_support["w8a16"].startswith("blocked:")
+
+
+def test_resolve_refuses_blocked_precision_through_the_registry_path(tmp_path):
+    """The Axis-0 refusal fires from a registry-loaded variant — not only a hand-built one."""
+    vs = _registry(tmp_path).variants("demo-llm")
+    d = resolve(TargetKey(htp_arch="v73", qairt="2.37.1"), vs, want_quant="w8a16")
+    assert d.outcome == Outcome.FAIL and d.axis == "precision", d
+    assert "blocked:genie-kv-cache-quant-mismatch" in d.reason
+
+
+def test_resolve_refuses_unsupported_precision_through_the_registry_path(tmp_path):
+    vs = _registry(tmp_path).variants("demo-llm")
+    d = resolve(TargetKey(htp_arch="v73", qairt="2.37.1"), vs, want_quant="fp16")
+    assert d.outcome == Outcome.FAIL and d.axis == "precision"
+    assert "unsupported:genie-engine-is-integer-only" in d.reason
+
+
+def test_supported_precision_is_not_refused_through_the_registry_path(tmp_path):
+    vs = _registry(tmp_path).variants("demo-llm")
+    d = resolve(TargetKey(htp_arch="v73", qairt="2.37.1"), vs, want_quant="w4a16")
+    assert d.axis != "precision", d   # w4a16 is supported → no precision refusal

chipwright-0.2.6/chipwright/tests/test_run_render.py

@@ -0,0 +1,47 @@
+"""The human run-report renderer is shape-safe (analyzer finding, 0.2.5).
+
+`run_variant` returns a cosine-shaped report for CV/ASR but a tripwire-shaped one for the LLM generative
+gate (`{tripwire, latency, pass}` — no `cosine_min`/`per_output`). The human door used to index the cosine
+keys unconditionally → a bare KeyError traceback on an LLM success, while the `--json` door was already
+shape-safe. These lock both shapes through the single `_render_run_report` helper (Principle IV: the two
+front doors don't diverge).
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright import cli, jsonout
+from chipwright.resolver import Decision, Outcome
+
+_USE = Decision(Outcome.USE)
+
+
+def test_llm_tripwire_report_renders_without_crash(capsys):
+    rep = {"pass": True, "latency": {},
+           "tripwire": {"token_match": 0.991, "thresholds": {"token_match": 0.98}}}
+    code = cli._render_run_report(rep, _USE)
+    out = capsys.readouterr().out
+    assert "generative gate" in out and "token_match = 0.99100" in out
+    assert "Traceback" not in out
+    assert code == 0                               # USE pass → 0
+
+
+def test_llm_tripwire_fail_exits_verify_mismatch(capsys):
+    rep = {"pass": False, "tripwire": {"token_match": 0.50, "thresholds": {"token_match": 0.98}}}
+    assert cli._render_run_report(rep, _USE) == jsonout.EXIT_VERIFY_MISMATCH
+
+
+def test_llm_behavior_block_is_shown_when_present(capsys):
+    rep = {"pass": True, "tripwire": {"token_match": 0.99, "thresholds": {"token_match": 0.98}},
+           "behavior": {"refusal_retention": 1.0, "benign_answer_rate": 0.96}}
+    cli._render_run_report(rep, _USE)
+    assert "behavioral probe" in capsys.readouterr().out
+
+
+def test_cosine_report_still_renders(capsys):
+    rep = {"pass": True, "cosine_min": 0.95585, "threshold": 0.95, "per_output": {"class_logits": 0.95585}}
+    code = cli._render_run_report(rep, _USE)
+    out = capsys.readouterr().out
+    assert "NPU vs CPU-reference" in out and "class_logits: 0.95585" in out
+    assert code == 0

chipwright-0.2.6/chipwright/tests/test_runtime_support.py

@@ -0,0 +1,261 @@
+"""Runtime axis (spec 004, FR-RT01/03/04/05, SC-RT01/02/04) — the OFFLINE subset.
+
+The runtime axis is the precision axis a second time: a per-key "supported"|"<kind>:<reason>" matrix the
+resolver refuses on, one level deeper (`{runtime: {arch: status}}`) because a runtime's support is per-arch.
+Everything here is pure and board-free — the recipe parse/validate, the resolver priority + the axis=runtime
+refusal, and the runtime-variant record threaded through the registry. Also runnable as
+`python3 test_runtime_support.py`. (The on-device adapter run + dispatch check are Track 2 — they need silicon.)
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright.env import TargetKey
+from chipwright.recipe import from_dict
+from chipwright.registry import Registry
+from chipwright.resolver import (Outcome, Variant, resolve, runnable_runtimes,
+                                 runtime_refusal)
+
+
+# --- recipe schema: runtime_support parses + validates (T-RT01) -----------------------------------
+_BASE = {"name": "qwen2.5-1.5b", "version": "1.0.0",
+         "source": {"format": "onnx", "sha256": "abc"},
+         "target": {"arch": "v73", "quant": "w4a16", "shape": "ctx4096"}}
+
+_MATRIX = {
+    "genie": {"v68": "deprecated:geniex-qairt", "v73": "deprecated:geniex-qairt"},
+    "geniex-qairt": {"v73": "supported"},
+    "llamacpp-htp": {"v68": "unsupported:htp-backend-excludes-v68", "v73": "supported"},
+}
+
+
+def _recipe(**extra):
+    doc = dict(_BASE)
+    doc.update(extra)
+    return from_dict(doc)
+
+
+def test_recipe_parses_runtime_matrix():
+    r = _recipe(runtime_support=_MATRIX)
+    assert r.runtime_status("llamacpp-htp", "v73") == "supported"
+    assert r.runtime_status("llamacpp-htp", "v68") == "unsupported:htp-backend-excludes-v68"
+    assert r.runtime_status("genie", "v73") == "deprecated:geniex-qairt"
+    # 'deprecated' still RUNS (so it's runnable), 'unsupported' does not
+    assert r.runnable_runtimes("v73") == ["genie", "geniex-qairt", "llamacpp-htp"]
+    assert r.runnable_runtimes("v68") == ["genie"]
+
+
+def test_recipe_without_matrix_is_empty_not_error():
+    r = _recipe()
+    assert r.runtime_support == {} and r.runnable_runtimes("v73") == []
+
+
+def test_recipe_rejects_unknown_status_kind():
+    for bad in ({"llamacpp-htp": {"v73": "maybe"}}, {"genie": {"v73": "gone:x"}}):
+        try:
+            _recipe(runtime_support=bad)
+        except ValueError:
+            continue
+        raise AssertionError(f"expected ValueError for {bad!r}")
+
+
+def test_recipe_rejects_deprecated_or_unsupported_without_reason():
+    for bad in ({"genie": {"v73": "deprecated"}},
+                {"llamacpp-htp": {"v68": "unsupported"}},
+                {"llamacpp-htp": {"v68": "unsupported: "}}):
+        try:
+            _recipe(runtime_support=bad)
+        except ValueError:
+            continue
+        raise AssertionError(f"a deprecated/unsupported status must NAME a reason/successor: {bad!r}")
+
+
+def test_recipe_rejects_non_mapping_arch_entry():
+    try:
+        _recipe(runtime_support={"genie": "supported"})   # must be {arch: status}, not a bare status
+    except ValueError:
+        return
+    raise AssertionError("expected ValueError for a non-mapping runtime entry")
+
+
+# --- the pure refusal helpers --------------------------------------------------------------------
+def test_runtime_refusal_helper():
+    assert runtime_refusal("llamacpp-htp", "v68", _MATRIX) == "unsupported:htp-backend-excludes-v68"
+    assert runtime_refusal("llamacpp-htp", "v73", _MATRIX) is None   # supported -> no refusal
+    assert runtime_refusal("genie", "v73", _MATRIX) is None          # deprecated still RUNS -> no refusal
+    assert runtime_refusal("llamacpp-htp", "v73", None) is None      # no matrix -> nothing to refuse
+    assert runtime_refusal("nonesuch", "v73", _MATRIX) is None       # undeclared -> not refused here
+
+
+def test_runnable_runtimes_helper():
+    assert runnable_runtimes("v68", _MATRIX) == ["genie"]            # llamacpp unsupported; genie deprecated-but-runs
+    assert runnable_runtimes("v73", _MATRIX) == ["genie", "geniex-qairt", "llamacpp-htp"]
+
+
+# --- resolver: priority picks the runtime (SC-RT01) ----------------------------------------------
+def _genie(arch="v73", **kw):
+    base = dict(model="qwen2.5-1.5b", version="1.0.0", arch=arch, quant="w4a16", shape="ctx4096",
+                sdk_tested="2.37.1", sdk_compatible=">=2.34,<2.40", runtime="genie",
+                runtime_support=_MATRIX, url="u", sha256="s")
+    base.update(kw)
+    return Variant(**base)
+
+
+def _htp(arch="v73", **kw):
+    base = dict(model="qwen2.5-1.5b", version="1.0.0", arch=arch, quant="Q8_0", shape="ctx4096",
+                sdk_tested="2.37.1", sdk_compatible=">=2.34,<2.40", runtime="llamacpp-htp",
+                ext="gguf", runtime_support=_MATRIX, local="/tmp/q.gguf")
+    base.update(kw)
+    return Variant(**base)
+
+
+def test_priority_quality_picks_llamacpp_htp():
+    t = TargetKey(htp_arch="v73", qairt="2.37.1")
+    d = resolve(t, [_genie(), _htp()], priority="quality")
+    assert d.ok and d.variant.effective_runtime == "llamacpp-htp" and d.variant.quant == "Q8_0"
+    assert "priority=quality" in (d.warning or "")        # the trade-off is recorded on the Decision
+
+
+def test_priority_decode_latency_picks_genie():
+    t = TargetKey(htp_arch="v73", qairt="2.37.1")
+    d = resolve(t, [_genie(), _htp()], priority="decode-latency")
+    assert d.ok and d.variant.effective_runtime == "genie" and d.variant.quant == "w4a16"
+
+
+def test_default_picks_genie_floor_and_warns_to_choose():
+    t = TargetKey(htp_arch="v73", qairt="2.37.1")
+    d = resolve(t, [_genie(), _htp()])                    # no priority
+    assert d.ok and d.variant.effective_runtime == "genie"
+    assert "priority=quality|decode-latency" in (d.warning or "")   # tells the caller how to choose
+
+
+def test_deprecated_runtime_is_flagged_with_successor():
+    t = TargetKey(htp_arch="v73", qairt="2.37.1")
+    d = resolve(t, [_genie(), _htp()], want_runtime="genie")
+    assert d.ok and "deprecated:geniex-qairt" in (d.warning or "")  # resolve warns on a deprecated runtime
+
+
+def test_single_runtime_model_is_unchanged_by_the_axis():
+    # a model with one runtime resolves exactly as before — no priority machinery engages
+    t = TargetKey(htp_arch="v73", qairt="2.37.1")
+    d = resolve(t, [_htp()])
+    assert d.ok and d.variant.effective_runtime == "llamacpp-htp"
+    assert "priority" not in (d.warning or "")
+
+
+# --- resolver: refuse a runtime the board can't host, BY NAME (SC-RT02) --------------------------
+def test_resolve_refuses_llamacpp_htp_on_v68_by_name():
+    t = TargetKey(htp_arch="v68", qairt="2.37.1")
+    d = resolve(t, [_genie(arch="v68")], want_runtime="llamacpp-htp")
+    assert d.outcome is Outcome.FAIL and d.axis == "runtime"
+    # names the runtime, the arch, the reason, and lists what IS runnable there
+    for token in ("llamacpp-htp", "v68", "htp-backend-excludes-v68", "genie"):
+        assert token in d.reason, token
+
+
+def test_resolve_refusal_reads_matrix_off_variants_or_explicit():
+    t = TargetKey(htp_arch="v68", qairt="2.37.1")
+    # matrix passed explicitly (e.g. straight from the recipe) also drives the refusal
+    v = _genie(arch="v68", runtime_support=None)
+    d = resolve(t, [v], want_runtime="llamacpp-htp", runtime_support=_MATRIX)
+    assert d.outcome is Outcome.FAIL and d.axis == "runtime"
+
+
+def test_supported_runtime_on_arch_is_not_refused():
+    t = TargetKey(htp_arch="v73", qairt="2.37.1")
+    d = resolve(t, [_genie(), _htp()], want_runtime="llamacpp-htp")
+    assert d.ok and d.variant.effective_runtime == "llamacpp-htp"
+
+
+# --- the runtime-variant RECORD carries runtime+precision+PPL+decode/prefill (SC-RT04) ----------
+# A self-contained fixture registry (mirrors registry/index.dev.yaml, which is the author-local overlay)
+# so the test is green in any checkout AND exercises registry._load threading runtime + runtime_support.
+_FIXTURE = """registry_schema: 1
+models:
+  - model: qwen2.5-1.5b-instruct
+    modality: llm
+    version: 1.0.0
+    variants:
+      - arch: v73
+        quant: w4a16
+        shape: ctx4096
+        sdk_tested: "2.37.1"
+        sdk_compatible: ">=2.34,<2.40"
+        ext: qnnctx
+        runtime: genie
+        runtime_support:
+          genie: {v73: "deprecated:geniex-qairt"}
+          llamacpp-htp: {v68: "unsupported:htp-backend-excludes-v68", v73: "supported"}
+        recipe: recipes/qwen2.5-1.5b/1.0.0.yaml
+      - arch: v73
+        quant: Q8_0
+        shape: ctx4096
+        sdk_tested: "2.37.1"
+        sdk_compatible: ">=2.34,<2.40"
+        ext: gguf
+        runtime: llamacpp-htp
+        runtime_support:
+          genie: {v73: "deprecated:geniex-qairt"}
+          llamacpp-htp: {v68: "unsupported:htp-backend-excludes-v68", v73: "supported"}
+        local: /tmp/qwen2.5-1.5b-instruct-q8_0.gguf
+        verified:
+          "on": "QCS8550 (Hexagon v73, soc_id 603)"
+          runtime: llamacpp-htp
+          precision: Q8_0
+          loads: true
+          quality: {metric: wikitext_ppl, value: 11.66, reference_f16: 11.64}
+          latency: {decode_tok_s: 14.4, prefill_speedup_vs_cpu: 1.74, prefill_tok_s: 266.7}
+          deferred: {genie_v73_comparison: "no w4a16-Genie-on-v73 bundle built"}
+"""
+
+
+def _fixture_registry(tmp_path):
+    p = tmp_path / "runtime_index.yaml"
+    p.write_text(_FIXTURE)
+    return Registry(index=str(p))
+
+
+def test_registry_threads_runtime_and_matrix_onto_variants(tmp_path):
+    vs = _fixture_registry(tmp_path).variants("qwen2.5-1.5b-instruct")
+    htp = next(v for v in vs if v.effective_runtime == "llamacpp-htp")
+    assert htp.runtime == "llamacpp-htp" and htp.quant == "Q8_0" and htp.ext == "gguf"
+    assert isinstance(htp.runtime_support, dict)
+    assert htp.runtime_support["llamacpp-htp"]["v68"] == "unsupported:htp-backend-excludes-v68"
+
+
+def test_record_carries_runtime_precision_ppl_and_latency(tmp_path):
+    vs = _fixture_registry(tmp_path).variants("qwen2.5-1.5b-instruct")
+    htp = next(v for v in vs if v.effective_runtime == "llamacpp-htp")
+    rec = htp.verified
+    assert rec["runtime"] == "llamacpp-htp"               # which runtime
+    assert rec["precision"] == "Q8_0"                     # its precision
+    assert rec["quality"]["value"] == 11.66              # on-device quality (PPL)
+    assert rec["quality"]["reference_f16"] == 11.64
+    assert rec["latency"]["decode_tok_s"] == 14.4         # decode tok/s
+    assert rec["latency"]["prefill_speedup_vs_cpu"] == 1.74   # prefill (dispatch proof)
+    assert "deferred" in rec                              # the Genie-v73 head-to-head is recorded as deferred
+
+
+def test_resolve_over_fixture_registry_quality_vs_v68_refusal(tmp_path):
+    vs = _fixture_registry(tmp_path).variants("qwen2.5-1.5b-instruct")
+    # SC-RT01 on the registry: v73 + quality -> the Q8_0 HTP variant
+    d = resolve(TargetKey(htp_arch="v73", qairt="2.37.1"), vs, priority="quality")
+    assert d.ok and d.variant.effective_runtime == "llamacpp-htp"
+    # SC-RT02 on the registry: v68 + llamacpp-htp -> FAIL axis=runtime
+    d = resolve(TargetKey(htp_arch="v68", qairt="2.37.1"), vs, want_runtime="llamacpp-htp")
+    assert d.outcome is Outcome.FAIL and d.axis == "runtime"
+
+
+if __name__ == "__main__":
+    import tempfile
+    from pathlib import Path
+    fns = [f for n, f in sorted(globals().items()) if n.startswith("test_")]
+    for f in fns:
+        if "tmp_path" in f.__code__.co_varnames[:f.__code__.co_argcount]:
+            with tempfile.TemporaryDirectory() as d:
+                f(Path(d))
+        else:
+            f()
+        print("ok", f.__name__)
+    print(f"\n{len(fns)} passed")

{chipwright-0.2.4 → chipwright-0.2.6}/chipwright.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chipwright
-Version: 0.2.4
+Version: 0.2.6
 Summary: A verified package manager for Qualcomm NPUs — resolve the artifact that loads on your exact chip, run it, and prove it against a CPU reference.
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/nameissakthi25/chipwright

{chipwright-0.2.4 → chipwright-0.2.6}/chipwright.egg-info/SOURCES.txt

@@ -48,11 +48,14 @@ chipwright/tests/test_env_adb.py
 chipwright/tests/test_json_cli.py
 chipwright/tests/test_llm_modality.py
 chipwright/tests/test_packaging.py
+chipwright/tests/test_precision_registry.py
 chipwright/tests/test_precision_support.py
 chipwright/tests/test_registry.py
 chipwright/tests/test_replay.py
 chipwright/tests/test_resolver.py
 chipwright/tests/test_run_llm.py
+chipwright/tests/test_run_render.py
+chipwright/tests/test_runtime_support.py
 chipwright/tests/test_security.py
 chipwright/tests/test_tags.py
 chipwright/tests/test_task_eval.py

{chipwright-0.2.4 → chipwright-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "chipwright"
-version = "0.2.4"
+version = "0.2.6"
 description = "A verified package manager for Qualcomm NPUs — resolve the artifact that loads on your exact chip, run it, and prove it against a CPU reference."
 readme = "README.md"
 license = "MIT"