chipwright 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chipwright-0.2.3 → chipwright-0.2.4}/PKG-INFO +43 -1
- {chipwright-0.2.3 → chipwright-0.2.4}/README.md +42 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/__init__.py +1 -1
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/cache.py +31 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/cli.py +230 -19
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/data/index.yaml +6 -0
- chipwright-0.2.4/chipwright/jsonout.py +133 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/modalities/cv.py +16 -5
- chipwright-0.2.4/chipwright/replay.py +148 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/run.py +33 -30
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_cli_ux.py +5 -2
- chipwright-0.2.4/chipwright/tests/test_emit_record.py +46 -0
- chipwright-0.2.4/chipwright/tests/test_json_cli.py +245 -0
- chipwright-0.2.4/chipwright/tests/test_replay.py +183 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright.egg-info/PKG-INFO +43 -1
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright.egg-info/SOURCES.txt +5 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/pyproject.toml +1 -1
- {chipwright-0.2.3 → chipwright-0.2.4}/LICENSE +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/adapt.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/build.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/config.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/data/behavior_prompts.yaml +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/device.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/device_adb.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/env.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/errors.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/llm_reference.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/modalities/__init__.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/modalities/asr_encoder.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/modalities/llm.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/recipe.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/registry.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/resolver.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/sdk.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tags.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/task_eval.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/conftest.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_adapt.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_build.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_cache.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_device_adb.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_device_ctx.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_env.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_env_adb.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_llm_modality.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_packaging.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_precision_support.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_registry.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_resolver.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_run_llm.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_security.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_tags.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_task_eval.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_verify.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/tests/test_verify_llm.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/verify.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright/verify_llm.py +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright.egg-info/dependency_links.txt +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright.egg-info/entry_points.txt +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright.egg-info/requires.txt +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/chipwright.egg-info/top_level.txt +0 -0
- {chipwright-0.2.3 → chipwright-0.2.4}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chipwright
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: A verified package manager for Qualcomm NPUs — resolve the artifact that loads on your exact chip, run it, and prove it against a CPU reference.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/nameissakthi25/chipwright
|
|
@@ -161,6 +161,48 @@ That `0.95600` is not a claim in a README. It is the number Chipwright measured
|
|
|
161
161
|
- `--target <arch[,sdk]>` — resolve **offline** against a hand-named target instead of a probed board, e.g. `--target htpv68` or `--target htpv68,qnn2.37`. The arch is required; the SDK is optional — omit it and the SDK axis is reported unverified rather than guessed. It is a global flag, so it goes before the subcommand (`chipwright --target htpv68 resolve …`).
|
|
162
162
|
- `CW_REGISTRY` — point the client at an alternate index (a local path or a URL) instead of the public index bundled in the wheel. Use it to resolve/run against a dev or private registry; the author's development overlay is `registry/index.dev.yaml`.
|
|
163
163
|
|
|
164
|
+
### Machine interface — `--json` and exit codes
|
|
165
|
+
|
|
166
|
+
Chipwright is meant to be a CI/CD gate, so every outcome is machine-readable. `--json` is a **global flag** (it goes before the subcommand, like `--target`), and it makes `resolve` / `run` / `list` emit **exactly one JSON document to stdout and nothing else** — no colour, no human prose. Any human or progress text goes to **stderr**, so a pipeline can parse stdout cleanly:
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
$ chipwright --target htpv68 --json resolve mobilenet-v2 | jq -e '.outcome'
|
|
170
|
+
"USE_WITH_WARN"
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
```json
|
|
174
|
+
{
|
|
175
|
+
"model": "mobilenet-v2", "target": "htpv68", "outcome": "USE_WITH_WARN",
|
|
176
|
+
"variant": "mobilenet-v2-1.0.0", "arch": "v68", "quant": "w8a16", "shape": "img224",
|
|
177
|
+
"warnings": ["board QAIRT unknown (offline) — SDK compatibility unverified"],
|
|
178
|
+
"reason": "artifact matches arch/quant/shape"
|
|
179
|
+
}
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
`run` adds the verification record, so a deploy can gate on `.verified` without scraping text:
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
$ chipwright --json run mobilenet-v2 | jq '{verified, cosine_min, threshold}'
|
|
186
|
+
{ "verified": true, "cosine_min": 0.95600, "threshold": 0.95 }
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
The **outcome is data, not a glyph** — on a `FAIL`, the failing `axis` (`htp_arch`, `qairt`, `quant`, …) is a first-class field, not a free-text reason to regex. `list` emits an array of per-variant records. Colour in the human (non-`--json`) output is gated on `stdout.isatty()` and honours `NO_COLOR` (disable) and `FORCE_COLOR=1` (enable); `--json` is always plain.
|
|
190
|
+
|
|
191
|
+
Each outcome maps to its own **stable, documented exit code** (also in `chipwright --help`), so a gate can branch on the code alone — "build it" vs "never runs here" vs "FAILED the gate" are distinct:
|
|
192
|
+
|
|
193
|
+
| code | meaning | when |
|
|
194
|
+
| ---: | --- | --- |
|
|
195
|
+
| `0` | **USE** | exact match — run it |
|
|
196
|
+
| `11` | **USE_WITH_WARN** | a match, but the board's SDK is off the tested point (still in range) |
|
|
197
|
+
| `10` | **BUILD** | no artifact, but a recipe can build one |
|
|
198
|
+
| `20` | **FAIL** | nothing fits — the message names the axis |
|
|
199
|
+
| `30` | **VERIFY_MISMATCH** | ran, but the NPU-vs-CPU cosine fell below threshold |
|
|
200
|
+
| `40` | **NOT_RUNNABLE** | resolved, but cannot execute here (no run block / no board) |
|
|
201
|
+
| `2` | usage / unknown model | a usage error |
|
|
202
|
+
| `1` | unexpected error | an internal error (in `--json` mode, still emitted as `{"error":…,"exit_code":1}`) |
|
|
203
|
+
|
|
204
|
+
The codes are a **versioned contract** (`chipwright.jsonout.__api__`): a code never changes meaning without a MAJOR bump, and new outcomes take new codes. (This replaces the earlier 0/1 collapse, where BUILD, FAIL and a failed verification all exited `1` — the one deliberate behaviour change in this release.)
|
|
205
|
+
|
|
164
206
|
---
|
|
165
207
|
|
|
166
208
|
## The three verbs
|
|
@@ -125,6 +125,48 @@ That `0.95600` is not a claim in a README. It is the number Chipwright measured
|
|
|
125
125
|
- `--target <arch[,sdk]>` — resolve **offline** against a hand-named target instead of a probed board, e.g. `--target htpv68` or `--target htpv68,qnn2.37`. The arch is required; the SDK is optional — omit it and the SDK axis is reported unverified rather than guessed. It is a global flag, so it goes before the subcommand (`chipwright --target htpv68 resolve …`).
|
|
126
126
|
- `CW_REGISTRY` — point the client at an alternate index (a local path or a URL) instead of the public index bundled in the wheel. Use it to resolve/run against a dev or private registry; the author's development overlay is `registry/index.dev.yaml`.
|
|
127
127
|
|
|
128
|
+
### Machine interface — `--json` and exit codes
|
|
129
|
+
|
|
130
|
+
Chipwright is meant to be a CI/CD gate, so every outcome is machine-readable. `--json` is a **global flag** (it goes before the subcommand, like `--target`), and it makes `resolve` / `run` / `list` emit **exactly one JSON document to stdout and nothing else** — no colour, no human prose. Any human or progress text goes to **stderr**, so a pipeline can parse stdout cleanly:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
$ chipwright --target htpv68 --json resolve mobilenet-v2 | jq -e '.outcome'
|
|
134
|
+
"USE_WITH_WARN"
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
```json
|
|
138
|
+
{
|
|
139
|
+
"model": "mobilenet-v2", "target": "htpv68", "outcome": "USE_WITH_WARN",
|
|
140
|
+
"variant": "mobilenet-v2-1.0.0", "arch": "v68", "quant": "w8a16", "shape": "img224",
|
|
141
|
+
"warnings": ["board QAIRT unknown (offline) — SDK compatibility unverified"],
|
|
142
|
+
"reason": "artifact matches arch/quant/shape"
|
|
143
|
+
}
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
`run` adds the verification record, so a deploy can gate on `.verified` without scraping text:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
$ chipwright --json run mobilenet-v2 | jq '{verified, cosine_min, threshold}'
|
|
150
|
+
{ "verified": true, "cosine_min": 0.95600, "threshold": 0.95 }
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The **outcome is data, not a glyph** — on a `FAIL`, the failing `axis` (`htp_arch`, `qairt`, `quant`, …) is a first-class field, not a free-text reason to regex. `list` emits an array of per-variant records. Colour in the human (non-`--json`) output is gated on `stdout.isatty()` and honours `NO_COLOR` (disable) and `FORCE_COLOR=1` (enable); `--json` is always plain.
|
|
154
|
+
|
|
155
|
+
Each outcome maps to its own **stable, documented exit code** (also in `chipwright --help`), so a gate can branch on the code alone — "build it" vs "never runs here" vs "FAILED the gate" are distinct:
|
|
156
|
+
|
|
157
|
+
| code | meaning | when |
|
|
158
|
+
| ---: | --- | --- |
|
|
159
|
+
| `0` | **USE** | exact match — run it |
|
|
160
|
+
| `11` | **USE_WITH_WARN** | a match, but the board's SDK is off the tested point (still in range) |
|
|
161
|
+
| `10` | **BUILD** | no artifact, but a recipe can build one |
|
|
162
|
+
| `20` | **FAIL** | nothing fits — the message names the axis |
|
|
163
|
+
| `30` | **VERIFY_MISMATCH** | ran, but the NPU-vs-CPU cosine fell below threshold |
|
|
164
|
+
| `40` | **NOT_RUNNABLE** | resolved, but cannot execute here (no run block / no board) |
|
|
165
|
+
| `2` | usage / unknown model | a usage error |
|
|
166
|
+
| `1` | unexpected error | an internal error (in `--json` mode, still emitted as `{"error":…,"exit_code":1}`) |
|
|
167
|
+
|
|
168
|
+
The codes are a **versioned contract** (`chipwright.jsonout.__api__`): a code never changes meaning without a MAJOR bump, and new outcomes take new codes. (This replaces the earlier 0/1 collapse, where BUILD, FAIL and a failed verification all exited `1` — the one deliberate behaviour change in this release.)
|
|
169
|
+
|
|
128
170
|
---
|
|
129
171
|
|
|
130
172
|
## The three verbs
|
|
@@ -109,3 +109,34 @@ def fetch_named(url: str, sha256: str, name: str, sidecars=None) -> str:
|
|
|
109
109
|
_require_sha(s_sha, "reference sidecar")
|
|
110
110
|
_ensure(s_url, s_sha, os.path.join(d, _safe_name(s_name)))
|
|
111
111
|
return main
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _looks_url(s) -> bool:
|
|
115
|
+
return isinstance(s, str) and s.startswith(("http://", "https://"))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def locate_ref(r: dict, key: str, override=None):
|
|
119
|
+
"""Locate a run-block reference (cpu_onnx / input / meta): a local path, or a HOSTED url fetched into
|
|
120
|
+
the cache so a stranger's `run` works, not only the author's.
|
|
121
|
+
|
|
122
|
+
A url value is sha-verified via a companion `<key>_sha256` (refusing by name if absent — never an
|
|
123
|
+
unverified reference) and fetched with its original filename so an ONNX's external-data sidecar
|
|
124
|
+
resolves; declare sidecars under `<key>_sidecars: [{url, sha256, name}]`.
|
|
125
|
+
|
|
126
|
+
Lives here (device-free) rather than in `run.py` so both the live gate (`run.run_variant`) and the
|
|
127
|
+
offline replay (`replay.rederive`) share one reference locator without either importing the board
|
|
128
|
+
transport. `run._locate_ref` is an alias of this.
|
|
129
|
+
"""
|
|
130
|
+
val = override if override is not None else r.get(key)
|
|
131
|
+
if val is None:
|
|
132
|
+
return None
|
|
133
|
+
if _looks_url(val):
|
|
134
|
+
sha = r.get(f"{key}_sha256")
|
|
135
|
+
if not sha:
|
|
136
|
+
raise RuntimeError(
|
|
137
|
+
f"run block {key} is a url but carries no {key}_sha256 — refusing to fetch an unverified "
|
|
138
|
+
f"reference (a reference the gate can't trust is worse than none).")
|
|
139
|
+
sidecars = [(s["url"], s["sha256"], s["name"]) for s in (r.get(f"{key}_sidecars") or [])]
|
|
140
|
+
name = val.rsplit("/", 1)[-1]
|
|
141
|
+
return fetch_named(val, sha, name, sidecars)
|
|
142
|
+
return os.path.expanduser(val)
|
|
@@ -11,18 +11,70 @@ plainly rather than guessing.
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
13
|
import argparse
|
|
14
|
+
import contextlib
|
|
14
15
|
import importlib.util
|
|
16
|
+
import json
|
|
15
17
|
import os
|
|
16
18
|
import re
|
|
17
19
|
import sys
|
|
18
20
|
|
|
19
|
-
from . import cache
|
|
21
|
+
from . import cache, jsonout
|
|
20
22
|
from .env import TargetKey, probe
|
|
21
23
|
from .registry import Registry
|
|
22
24
|
from .resolver import Outcome, resolve
|
|
23
25
|
|
|
26
|
+
# ANSI palette. These are module globals so the whole file can keep printing with them; `_set_palette`
|
|
27
|
+
# blanks them when colour is off (not a TTY / NO_COLOR / --json). The literals below are the coloured
|
|
28
|
+
# default; the actual values for a given invocation are chosen in `main()`.
|
|
24
29
|
GREEN, YELLOW, RED, DIM, RST = "\033[32m", "\033[33m", "\033[31m", "\033[2m", "\033[0m"
|
|
25
30
|
|
|
31
|
+
|
|
32
|
+
def _color_enabled(json_mode: bool) -> bool:
|
|
33
|
+
"""Should colour be emitted? (spec 006 FR-J05)
|
|
34
|
+
|
|
35
|
+
`--json` is always plain. Otherwise honour `NO_COLOR` (set → off, any value, per the no-color.org
|
|
36
|
+
convention), then `FORCE_COLOR=1` (on), else gate on whether stdout is a real TTY.
|
|
37
|
+
"""
|
|
38
|
+
if json_mode:
|
|
39
|
+
return False
|
|
40
|
+
if os.environ.get("NO_COLOR") is not None:
|
|
41
|
+
return False
|
|
42
|
+
if os.environ.get("FORCE_COLOR") == "1":
|
|
43
|
+
return True
|
|
44
|
+
try:
|
|
45
|
+
return sys.stdout.isatty()
|
|
46
|
+
except Exception:
|
|
47
|
+
return False
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _set_palette(enabled: bool):
|
|
51
|
+
"""Point the module colour globals at either the ANSI codes or empty strings."""
|
|
52
|
+
global GREEN, YELLOW, RED, DIM, RST
|
|
53
|
+
if enabled:
|
|
54
|
+
GREEN, YELLOW, RED, DIM, RST = "\033[32m", "\033[33m", "\033[31m", "\033[2m", "\033[0m"
|
|
55
|
+
else:
|
|
56
|
+
GREEN = YELLOW = RED = DIM = RST = ""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _emit_json(obj):
|
|
60
|
+
"""Write exactly one JSON document to the real stdout, and nothing else (FR-J01).
|
|
61
|
+
|
|
62
|
+
`default=str` keeps a verification record portable: YAML parses `date: 2026-10-08` into a
|
|
63
|
+
`datetime.date`, which `json` can't serialize — stringify such leaves rather than crash.
|
|
64
|
+
"""
|
|
65
|
+
sys.stdout.write(json.dumps(obj, default=str))
|
|
66
|
+
sys.stdout.write("\n")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _unknown_model(args, reg) -> int:
|
|
70
|
+
"""Report an unknown model consistently in both modes; exit code 2 (usage/unknown-model)."""
|
|
71
|
+
if getattr(args, "json", False):
|
|
72
|
+
_emit_json({"error": f"unknown model '{args.model}'. known: {', '.join(reg.models())}",
|
|
73
|
+
"exit_code": jsonout.EXIT_USAGE})
|
|
74
|
+
else:
|
|
75
|
+
print(f"{RED}unknown model{RST} '{args.model}'. known: {', '.join(reg.models())}")
|
|
76
|
+
return jsonout.EXIT_USAGE
|
|
77
|
+
|
|
26
78
|
# A bare Hexagon arch (what `list` and FAIL messages print, e.g. "v68") looks like a target but is
|
|
27
79
|
# missing the mandatory `htp` prefix. Used to turn a malformed --target into a helpful suggestion.
|
|
28
80
|
_ARCH_LIKE = re.compile(r"^v\d+$", re.IGNORECASE)
|
|
@@ -134,6 +186,19 @@ def cmd_doctor(args):
|
|
|
134
186
|
return rc
|
|
135
187
|
|
|
136
188
|
|
|
189
|
+
def _rederivable_note(v):
|
|
190
|
+
"""A small, non-breaking annotation (spec 005 FR-RR04): is this record's headline number
|
|
191
|
+
independently re-derivable offline (NPU output pinned), or does it predate reproducible-record?
|
|
192
|
+
Returns a colored line, or None when the variant carries no verification record at all.
|
|
193
|
+
"""
|
|
194
|
+
from . import replay
|
|
195
|
+
if not isinstance(v.verified, dict) or not isinstance(v.verified.get("vs_reference"), dict):
|
|
196
|
+
return None
|
|
197
|
+
if replay.is_rederivable(v):
|
|
198
|
+
return f"{GREEN}↺ independently re-derivable{RST} — chipwright run {v.model} --replay"
|
|
199
|
+
return f"{DIM}not independently re-derivable yet (record predates reproducible-record){RST}"
|
|
200
|
+
|
|
201
|
+
|
|
137
202
|
def _print_decision(model, target, d):
|
|
138
203
|
print(f"model : {model}")
|
|
139
204
|
print(f"target : {target}")
|
|
@@ -143,6 +208,9 @@ def _print_decision(model, target, d):
|
|
|
143
208
|
f"qnn{v.sdk_tested} ({d.reason})")
|
|
144
209
|
if v.verified:
|
|
145
210
|
print(f" {DIM}verified: {v.verified}{RST}")
|
|
211
|
+
note = _rederivable_note(v)
|
|
212
|
+
if note:
|
|
213
|
+
print(f" {note}")
|
|
146
214
|
elif d.outcome == Outcome.USE_WITH_WARN:
|
|
147
215
|
v = d.variant
|
|
148
216
|
print(f"{YELLOW}▲ USE (warn){RST} {v.model}-{v.version} htp{v.arch}/{v.quant}/{v.shape}")
|
|
@@ -153,6 +221,9 @@ def _print_decision(model, target, d):
|
|
|
153
221
|
print(f" {DIM}{d.reason}{RST}")
|
|
154
222
|
if v.verified:
|
|
155
223
|
print(f" {DIM}prior verification on record: {v.verified}{RST}")
|
|
224
|
+
note = _rederivable_note(v)
|
|
225
|
+
if note:
|
|
226
|
+
print(f" {note}")
|
|
156
227
|
else:
|
|
157
228
|
print(f"{RED}✗ FAIL{RST} axis={d.axis}: {d.reason}")
|
|
158
229
|
return d
|
|
@@ -161,26 +232,92 @@ def _print_decision(model, target, d):
|
|
|
161
232
|
def cmd_resolve(args):
|
|
162
233
|
reg = Registry(args.registry)
|
|
163
234
|
if args.model not in reg.models():
|
|
164
|
-
|
|
165
|
-
return 2
|
|
235
|
+
return _unknown_model(args, reg)
|
|
166
236
|
target = _get_target(args)
|
|
167
237
|
d = resolve(target, reg.variants(args.model), args.quant, args.shape)
|
|
168
|
-
|
|
169
|
-
|
|
238
|
+
if getattr(args, "json", False):
|
|
239
|
+
_emit_json(jsonout.decision_to_dict(args.model, target, d))
|
|
240
|
+
else:
|
|
241
|
+
_print_decision(args.model, target, d)
|
|
242
|
+
return jsonout.exit_code_for(d)
|
|
170
243
|
|
|
171
244
|
|
|
172
|
-
def
|
|
245
|
+
def _replay(reg, model):
|
|
246
|
+
"""Offline replay of a recorded on-device verification (spec 005 US2 / FR-RR03).
|
|
247
|
+
|
|
248
|
+
Picks a variant whose record pins the NPU output, re-derives the cosine from the pinned operands with
|
|
249
|
+
the shared gate, and prints the result LABELLED as a replay of the recorded result (board + date) —
|
|
250
|
+
never as a fresh on-silicon verification (Principle I). Exits 0 on a reproduced pass, non-zero on a
|
|
251
|
+
fail / a recomputed≠recorded mismatch / a missing pin.
|
|
252
|
+
"""
|
|
253
|
+
from . import replay as rp
|
|
254
|
+
variants = reg.variants(model)
|
|
255
|
+
cand = [v for v in variants if rp.is_rederivable(v)]
|
|
256
|
+
if not cand:
|
|
257
|
+
print(f"{RED}✗ cannot replay{RST} {model}: no pinned on-device output to replay; this record "
|
|
258
|
+
f"predates reproducible-record or was not captured.")
|
|
259
|
+
# a record exists but was never captured for replay — point at the live gate as the way to verify.
|
|
260
|
+
if any(v.verified for v in variants):
|
|
261
|
+
print(f" {DIM}the recorded number is still shown by `chipwright list` / `resolve`; to get a "
|
|
262
|
+
f"fresh VERIFIED line, run it on a board (chipwright run {model}).{RST}")
|
|
263
|
+
return 2
|
|
264
|
+
|
|
265
|
+
v = cand[0]
|
|
266
|
+
try:
|
|
267
|
+
res = rp.rederive(v)
|
|
268
|
+
except Exception as e:
|
|
269
|
+
print(f"{RED}✗ replay failed:{RST} {e}")
|
|
270
|
+
return 1
|
|
271
|
+
|
|
272
|
+
rec = v.verified or {}
|
|
273
|
+
board = rec.get("on", "?")
|
|
274
|
+
date = rec.get("date", "?")
|
|
275
|
+
cmin = res["cosine_min"]
|
|
276
|
+
tag = f"{DIM}↺ REPLAY{RST} (recorded on {board}, {date})"
|
|
277
|
+
|
|
278
|
+
if not res["matches"]:
|
|
279
|
+
# recomputed ≠ recorded → refuse, never silently pass (FR-RR04 / Principle II).
|
|
280
|
+
print(f"{tag} recomputed cosine_min={cmin:.5f} {RED}≠{RST} recorded {res['recorded']} "
|
|
281
|
+
f"(±{rp.TOLERANCE}) {RED}✗ REFUSING — record not reproducible{RST}")
|
|
282
|
+
return 1
|
|
283
|
+
if res["pass"]:
|
|
284
|
+
print(f"{tag} cosine_min={cmin:.5f} {GREEN}VERIFIED{RST}")
|
|
285
|
+
for name, c in res["outputs"].items():
|
|
286
|
+
print(f" {name}: {c:.5f}")
|
|
287
|
+
return 0
|
|
288
|
+
# reproduced faithfully, but the recorded number is itself below threshold → a recorded FAIL.
|
|
289
|
+
print(f"{tag} cosine_min={cmin:.5f} {RED}FAILED{RST} (threshold {res['threshold']})")
|
|
290
|
+
return 1
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def cmd_verify(args):
|
|
294
|
+
"""`chipwright verify <model> --offline` — the replay under its own verb (spec 005 US2)."""
|
|
173
295
|
reg = Registry(args.registry)
|
|
174
296
|
if args.model not in reg.models():
|
|
175
297
|
print(f"{RED}unknown model{RST} '{args.model}'. known: {', '.join(reg.models())}")
|
|
176
298
|
return 2
|
|
299
|
+
if not getattr(args, "offline", False):
|
|
300
|
+
print(f"{YELLOW}verify currently supports only offline replay{RST} — pass --offline "
|
|
301
|
+
f"(a live on-device verify is `chipwright run {args.model}`).")
|
|
302
|
+
return 2
|
|
303
|
+
return _replay(reg, args.model)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def cmd_run(args):
|
|
307
|
+
reg = Registry(args.registry)
|
|
308
|
+
if args.model not in reg.models():
|
|
309
|
+
return _unknown_model(args, reg)
|
|
310
|
+
if getattr(args, "replay", False):
|
|
311
|
+
return _replay(reg, args.model)
|
|
177
312
|
target = _get_target(args)
|
|
313
|
+
if getattr(args, "json", False):
|
|
314
|
+
return _run_json(args, reg, target)
|
|
178
315
|
d = resolve(target, reg.variants(args.model), args.quant, args.shape)
|
|
179
316
|
_print_decision(args.model, target, d)
|
|
180
317
|
if not d.ok:
|
|
181
318
|
if d.outcome == Outcome.BUILD:
|
|
182
319
|
print(f"{DIM}milestone 1 is resolve-only — building is out of scope; stopping.{RST}")
|
|
183
|
-
return
|
|
320
|
+
return jsonout.exit_code_for(d)
|
|
184
321
|
# USE / USE_WITH_WARN: execute on the board and verify against a CPU reference.
|
|
185
322
|
v = d.variant
|
|
186
323
|
if not (v.run and v.run.get("modality")):
|
|
@@ -189,19 +326,46 @@ def cmd_run(args):
|
|
|
189
326
|
if v.url:
|
|
190
327
|
print(f"fetching → cache ({cache.CACHE_DIR}) …")
|
|
191
328
|
print(f"{GREEN}✔ cached + hash-verified{RST} {cache.fetch(v.url, v.sha256)}")
|
|
192
|
-
return
|
|
329
|
+
return jsonout.EXIT_NOT_RUNNABLE
|
|
193
330
|
from . import run as runner
|
|
194
331
|
try:
|
|
195
|
-
rep = runner.run_variant(v, args.input)
|
|
332
|
+
rep = runner.run_variant(v, args.input, emit_record=getattr(args, "emit_record", False))
|
|
196
333
|
except Exception as e:
|
|
197
334
|
print(f"{RED}run failed:{RST} {e}")
|
|
198
|
-
return
|
|
335
|
+
return jsonout.EXIT_UNEXPECTED
|
|
199
336
|
mark = f"{GREEN}✔ VERIFIED{RST}" if rep["pass"] else f"{RED}✗ MISMATCH{RST}"
|
|
200
337
|
print(f"\n{mark} NPU vs CPU-reference cosine_min = {rep['cosine_min']:.5f} "
|
|
201
338
|
f"(threshold {rep['threshold']})")
|
|
202
339
|
for name, c in rep["per_output"].items():
|
|
203
340
|
print(f" {name}: {c:.5f}")
|
|
204
|
-
return
|
|
341
|
+
return jsonout.exit_code_for(d) if rep["pass"] else jsonout.EXIT_VERIFY_MISMATCH
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _run_json(args, reg, target) -> int:
|
|
345
|
+
"""`run` in `--json` mode: all human/progress text (including the device runner's own prints) goes to
|
|
346
|
+
stderr, then exactly one JSON document goes to stdout. The resolve/run/verify logic is untouched —
|
|
347
|
+
this only redirects the human stream and projects the result (Principle IV).
|
|
348
|
+
"""
|
|
349
|
+
# Everything human — the decision render and the runner's progress lines — to stderr.
|
|
350
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
351
|
+
d = resolve(target, reg.variants(args.model), args.quant, args.shape)
|
|
352
|
+
_print_decision(args.model, target, d)
|
|
353
|
+
if not d.ok:
|
|
354
|
+
doc, code = jsonout.decision_to_dict(args.model, target, d), jsonout.exit_code_for(d)
|
|
355
|
+
else:
|
|
356
|
+
v = d.variant
|
|
357
|
+
if not (v.run and v.run.get("modality")):
|
|
358
|
+
print(f"artifact resolves, but this variant has no run block — resolve-only.")
|
|
359
|
+
doc = jsonout.decision_to_dict(args.model, target, d)
|
|
360
|
+
doc["verified"] = None
|
|
361
|
+
code = jsonout.EXIT_NOT_RUNNABLE
|
|
362
|
+
else:
|
|
363
|
+
from . import run as runner
|
|
364
|
+
rep = runner.run_variant(v, args.input) # may raise → caught by the --json wrapper in main
|
|
365
|
+
doc = jsonout.run_to_dict(args.model, target, d, rep)
|
|
366
|
+
code = jsonout.exit_code_for(d) if rep.get("pass") else jsonout.EXIT_VERIFY_MISMATCH
|
|
367
|
+
_emit_json(doc)
|
|
368
|
+
return code
|
|
205
369
|
|
|
206
370
|
|
|
207
371
|
def cmd_list(args):
|
|
@@ -216,10 +380,15 @@ def cmd_list(args):
|
|
|
216
380
|
if target is None and args.target:
|
|
217
381
|
target = _manual_target(args.target)
|
|
218
382
|
|
|
383
|
+
if getattr(args, "json", False):
|
|
384
|
+
_emit_json(jsonout.list_to_dict(reg, target))
|
|
385
|
+
return 0
|
|
386
|
+
|
|
219
387
|
head = f"{target}" if target else "no target — pass --target or configure a board to resolve"
|
|
220
388
|
print(f"{DIM}catalogue · {len(reg.models())} models · {head}{RST}")
|
|
221
389
|
badge = {Outcome.USE: f"{GREEN}USE{RST}", Outcome.USE_WITH_WARN: f"{YELLOW}USE*{RST}",
|
|
222
390
|
Outcome.BUILD: f"{YELLOW}BUILD{RST}", Outcome.FAIL: f"{RED}FAIL{RST}"}
|
|
391
|
+
from . import replay as rp
|
|
223
392
|
for m in reg.models():
|
|
224
393
|
vs = reg.variants(m)
|
|
225
394
|
archs = ",".join(sorted({v.arch for v in vs}))
|
|
@@ -230,13 +399,19 @@ def cmd_list(args):
|
|
|
230
399
|
out = badge.get(d.outcome, "")
|
|
231
400
|
if d.outcome == Outcome.FAIL and d.axis:
|
|
232
401
|
out += f"{DIM}({d.axis}){RST}"
|
|
233
|
-
|
|
402
|
+
rec_v = next((v for v in vs if v.verified), None)
|
|
403
|
+
rec = rec_v.verified if rec_v else None
|
|
234
404
|
vr = ""
|
|
235
405
|
if rec and isinstance(rec.get("vs_reference"), dict):
|
|
236
406
|
val = rec["vs_reference"].get("value")
|
|
237
407
|
met = rec["vs_reference"].get("metric", "")
|
|
238
408
|
if val is not None:
|
|
239
409
|
vr = f"{DIM}{met}={val}{RST}"
|
|
410
|
+
# annotate re-derivability (spec 005 FR-RR04) — a small, non-breaking tag.
|
|
411
|
+
if rp.is_rederivable(rec_v):
|
|
412
|
+
vr += f" {GREEN}↺replay{RST}"
|
|
413
|
+
else:
|
|
414
|
+
vr += f" {DIM}(not independently re-derivable yet){RST}"
|
|
240
415
|
print(f" {m:20s} {archs:9s} {quants:9s} {out:16s} {vr}")
|
|
241
416
|
return 0
|
|
242
417
|
|
|
@@ -245,16 +420,15 @@ def cmd_build(args):
|
|
|
245
420
|
import os
|
|
246
421
|
reg = Registry(args.registry)
|
|
247
422
|
if args.model not in reg.models():
|
|
248
|
-
|
|
249
|
-
return 2
|
|
423
|
+
return _unknown_model(args, reg)
|
|
250
424
|
target = _get_target(args)
|
|
251
425
|
d = resolve(target, reg.variants(args.model), args.quant, args.shape)
|
|
252
426
|
_print_decision(args.model, target, d)
|
|
253
427
|
if d.ok:
|
|
254
428
|
print(f"{DIM}already resolves to USE — nothing to build.{RST}")
|
|
255
|
-
return
|
|
429
|
+
return jsonout.exit_code_for(d)
|
|
256
430
|
if d.outcome != Outcome.BUILD:
|
|
257
|
-
return
|
|
431
|
+
return jsonout.exit_code_for(d)
|
|
258
432
|
v = d.variant
|
|
259
433
|
from . import config, build as builder, errors, recipe as recipe_mod
|
|
260
434
|
|
|
@@ -290,22 +464,59 @@ def cmd_build(args):
|
|
|
290
464
|
return 1
|
|
291
465
|
|
|
292
466
|
|
|
467
|
+
# The outcome→exit-code contract, shown in `--help` so a pipeline author can read it without the README
|
|
468
|
+
# (spec 006 FR-J04). Kept in lockstep with chipwright.jsonout (the single source of the numbers).
|
|
469
|
+
_EXIT_CODE_HELP = f"""exit codes (stable contract, chipwright.jsonout.__api__={jsonout.__api__}):
|
|
470
|
+
{jsonout.EXIT_USE:<3} USE exact match — run it
|
|
471
|
+
{jsonout.EXIT_USE_WITH_WARN:<3} USE_WITH_WARN in range, outside the tested SDK point
|
|
472
|
+
{jsonout.EXIT_BUILD:<3} BUILD no artifact, but a recipe can build one
|
|
473
|
+
{jsonout.EXIT_FAIL:<3} FAIL nothing fits — the message names the axis
|
|
474
|
+
{jsonout.EXIT_VERIFY_MISMATCH:<3} VERIFY_MISMATCH ran, but the NPU vs CPU cosine fell below threshold
|
|
475
|
+
{jsonout.EXIT_NOT_RUNNABLE:<3} NOT_RUNNABLE resolved, but cannot execute here (no run block / no board)
|
|
476
|
+
{jsonout.EXIT_USAGE:<3} usage usage error / unknown model
|
|
477
|
+
{jsonout.EXIT_UNEXPECTED:<3} unexpected an unexpected internal error
|
|
478
|
+
"""
|
|
479
|
+
|
|
480
|
+
|
|
293
481
|
def main(argv=None):
|
|
294
|
-
ap = argparse.ArgumentParser(prog="chipwright", description="Chipwright — package manager for Qualcomm NPUs"
|
|
482
|
+
ap = argparse.ArgumentParser(prog="chipwright", description="Chipwright — package manager for Qualcomm NPUs",
|
|
483
|
+
epilog=_EXIT_CODE_HELP, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
295
484
|
ap.add_argument("--registry", default=None, help="index path/URL (default: bundled registry)")
|
|
296
485
|
ap.add_argument("--host"); ap.add_argument("--user"); ap.add_argument("--pw"); ap.add_argument("--port")
|
|
297
486
|
ap.add_argument("--target", help="offline target, e.g. 'htpv68,qnn2.37'")
|
|
298
487
|
ap.add_argument("--quant", help="narrow to a quant, e.g. w8a16")
|
|
299
488
|
ap.add_argument("--shape", help="narrow to a shape profile, e.g. win30s")
|
|
489
|
+
ap.add_argument("--json", action="store_true",
|
|
490
|
+
help="emit one JSON document to stdout (resolve/run/list); human text → stderr, "
|
|
491
|
+
"always plain. Exit codes follow the table below.")
|
|
300
492
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
301
493
|
sub.add_parser("doctor", help="probe the board")
|
|
302
494
|
sub.add_parser("list", help="browse the catalogue (with resolve outcome for your board)")
|
|
303
495
|
rp = sub.add_parser("resolve", help="resolve a model against the board"); rp.add_argument("model")
|
|
304
496
|
rnp = sub.add_parser("run", help="resolve, fetch, run on the board, verify vs CPU"); rnp.add_argument("model"); rnp.add_argument("--input")
|
|
497
|
+
rnp.add_argument("--replay", action="store_true",
|
|
498
|
+
help="no board: replay the recorded on-device verification from its pinned tensors")
|
|
499
|
+
rnp.add_argument("--emit-record", action="store_true",
|
|
500
|
+
help="capture the on-device NPU output tensor(s) + print a ready-to-host record fragment (spec 005 FR-RR05)")
|
|
501
|
+
vp = sub.add_parser("verify", help="offline replay of a recorded verification (no board)")
|
|
502
|
+
vp.add_argument("model"); vp.add_argument("--offline", action="store_true",
|
|
503
|
+
help="re-derive the recorded result from pinned tensors")
|
|
305
504
|
bp = sub.add_parser("build", help="resolve; when it misses, build from a recipe"); bp.add_argument("model")
|
|
306
505
|
args = ap.parse_args(argv)
|
|
307
|
-
|
|
308
|
-
|
|
506
|
+
_set_palette(_color_enabled(args.json))
|
|
507
|
+
handler = {"doctor": cmd_doctor, "list": cmd_list, "resolve": cmd_resolve,
|
|
508
|
+
"run": cmd_run, "verify": cmd_verify, "build": cmd_build}[args.cmd]
|
|
509
|
+
# In --json mode, an unexpected internal error must still leave exactly one JSON document on stdout
|
|
510
|
+
# (never a half-line a parser chokes on) and exit 1 (FR-J01, edge case).
|
|
511
|
+
if args.json and args.cmd in ("resolve", "run", "list"):
|
|
512
|
+
try:
|
|
513
|
+
return handler(args)
|
|
514
|
+
except SystemExit:
|
|
515
|
+
raise # clean usage refusals (exit 2) keep stdout empty — fine for a parser
|
|
516
|
+
except Exception as e:
|
|
517
|
+
_emit_json({"error": str(e), "exit_code": jsonout.EXIT_UNEXPECTED})
|
|
518
|
+
return jsonout.EXIT_UNEXPECTED
|
|
519
|
+
return handler(args)
|
|
309
520
|
|
|
310
521
|
|
|
311
522
|
if __name__ == "__main__":
|
|
@@ -198,6 +198,12 @@ models:
|
|
|
198
198
|
metric: cosine_min
|
|
199
199
|
value: 0.956
|
|
200
200
|
note: real image (ImageNet 'Samoyed'); random-noise input gives 0.776 and the gate correctly FAILs it
|
|
201
|
+
npu_output:
|
|
202
|
+
- name: class_logits
|
|
203
|
+
url: https://huggingface.co/nameissakthi/chipwright-cv/resolve/main/mobilenet-v2/npu_class_logits.npy
|
|
204
|
+
sha256: 7064b5bd22d053fdce8d7f602cdafecc30b01944b3a422da9143e0627bafc3a8
|
|
205
|
+
dtype: float32
|
|
206
|
+
shape: [1, 1000]
|
|
201
207
|
latency_ms:
|
|
202
208
|
note: single 224x224 image; DLC compiled to context on-board
|
|
203
209
|
- model: quicksrnet-small
|