chipwright 0.2.0__py3-none-any.whl

chipwright/tests/test_resolver.py

@@ -0,0 +1,93 @@
+"""The resolver is a pure function — the four outcomes are testable without a board."""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright.env import TargetKey
+from chipwright.registry import Registry
+from chipwright.resolver import Outcome, Variant, resolve
+from chipwright.tags import ArtifactTag
+
+
+def test_tag_roundtrip():
+    s = "whisper-small-en-1.2.0+qnn2.37-htpv68-w8a16-win30s.qnnctx"
+    t = ArtifactTag.parse(s)
+    assert (t.name, t.version, t.sdk, t.arch, t.quant, t.shape, t.ext) == \
+           ("whisper-small-en", "1.2.0", "2.37", "v68", "w8a16", "win30s", "qnnctx")
+    assert str(t) == s
+    assert t.is_context_binary
+
+
+def test_tag_rejects_garbage():
+    for bad in ["whisper.qnnctx", "x-1.0+htpv68.qnnctx", "x-1.0.0+qnn2.37-htpv68-w8a16.qnnctx"]:
+        try:
+            ArtifactTag.parse(bad); assert False, bad
+        except ValueError:
+            pass
+
+
+def _v(**kw):
+    base = dict(model="m", version="1.0.0", arch="v68", quant="w8a16", shape="win30s",
+                sdk_tested="2.37.1", sdk_compatible=">=2.34,<2.40")
+    base.update(kw)
+    return Variant(**base)
+
+
+def test_use_exact():
+    t = TargetKey(htp_arch="v68", qairt="2.37.1")
+    d = resolve(t, [_v(url="u", sha256="s")])
+    assert d.outcome is Outcome.USE and d.ok
+
+
+def test_use_with_warn_sdk_in_range():
+    t = TargetKey(htp_arch="v68", qairt="2.38.0")        # in range, not the tested point
+    d = resolve(t, [_v(url="u", sha256="s")])
+    assert d.outcome is Outcome.USE_WITH_WARN and d.ok and d.warning
+
+
+def test_offline_unknown_sdk_warns():
+    t = TargetKey(htp_arch="v68", qairt=None)
+    d = resolve(t, [_v(url="u", sha256="s")])
+    assert d.outcome is Outcome.USE_WITH_WARN
+
+
+def test_build_recipe_only():
+    t = TargetKey(htp_arch="v68", qairt="2.37.1")
+    d = resolve(t, [_v(recipe="r.yaml")])                 # no url
+    assert d.outcome is Outcome.BUILD and not d.ok
+
+
+def test_fail_arch_names_axis():
+    t = TargetKey(htp_arch="v68", qairt="2.37.1")
+    d = resolve(t, [_v(arch="v73", url="u", sha256="s")])
+    assert d.outcome is Outcome.FAIL and d.axis == "htp_arch"
+
+
+def test_fail_sdk_out_of_range_no_recipe():
+    t = TargetKey(htp_arch="v68", qairt="2.50.0")        # outside the range, artifact but no recipe
+    d = resolve(t, [_v(url="u", sha256="s")])
+    assert d.outcome is Outcome.FAIL and d.axis == "qairt"
+
+
+def test_sdk_out_of_range_but_recipe_builds():
+    t = TargetKey(htp_arch="v68", qairt="2.50.0")
+    d = resolve(t, [_v(url="u", sha256="s"), _v(recipe="r.yaml")])
+    assert d.outcome is Outcome.BUILD
+
+
+def test_seed_registry_loads():
+    reg = Registry()
+    assert "laya-decision" in reg.models() and "whisper-small-en" in reg.models()
+    v68 = TargetKey(htp_arch="v68", qairt="2.37.1")
+    assert resolve(v68, reg.variants("laya-decision")).ok               # USE (exact or SDK-warn)
+    assert resolve(v68, reg.variants("whisper-small-en")).outcome is Outcome.BUILD
+    assert resolve(v68, reg.variants("qwen2.5-1.5b-instruct")).outcome is Outcome.BUILD  # v68 variant exists
+    assert resolve(v68, reg.variants("qwen2.5-7b-instruct")).axis == "htp_arch"          # v73/v75 only
+
+
+if __name__ == "__main__":
+    fns = [f for n, f in sorted(globals().items()) if n.startswith("test_")]
+    for f in fns:
+        f(); print("ok", f.__name__)
+    print(f"\n{len(fns)} passed")

chipwright/tests/test_run_llm.py

@@ -0,0 +1,47 @@
+"""The LLM run path refuses by name when it can't gate — integration guards for run_llm_variant.
+
+Mirrors the CV/ASR refusal-path tests: a pass is never reported without the gate, and the LLM path
+refuses with a message naming the fix (no Genie bundle / no pinned reference), not a raw traceback.
+No board needed — these never reach adb.
+"""
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright import run as runmod
+from chipwright.resolver import Variant
+
+
+def _llm_variant(**kw):
+    base = dict(model="qwen-x", version="1", arch="v81", quant="w4a16", shape="ctx4096",
+                sdk_tested="2.51", sdk_compatible=">=2.34,<2.60")
+    base.update(kw)
+    return Variant(**base)
+
+
+def test_llm_run_refuses_when_no_bundle():
+    """An LLM variant with no Genie bundle directory refuses by name (build / CW_REGISTRY)."""
+    v = _llm_variant(run={"modality": "llm"})  # no local bundle dir
+    with pytest.raises(RuntimeError, match="bundle|chipwright build|CW_REGISTRY"):
+        runmod.run_variant(v)
+
+
+def test_llm_run_refuses_when_no_pinned_reference(tmp_path):
+    """A bundle present but no pinned reference → refuse: the generative gate can't run."""
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    (bundle / "genie_config.json").write_text("{}")
+    v = _llm_variant(local=str(bundle), run={"modality": "llm"})  # bundle, but no `reference`
+    with pytest.raises(RuntimeError, match="pinned reference|gate can't run"):
+        runmod.run_variant(v)
+
+
+def test_looks_refused_classifier():
+    """The v1 behavioral classifier tells a refusal from a compliant answer."""
+    assert runmod._looks_refused("I can't help with that.") is True
+    assert runmod._looks_refused("I'm sorry, but I cannot assist.") is True
+    assert runmod._looks_refused("The capital of France is Paris.") is False
+    assert runmod._looks_refused("") is False

chipwright/tests/test_tags.py

@@ -0,0 +1,95 @@
+"""ArtifactTag — the wheel-tag grammar. Strict parse, round-trip, context-binary classification.
+
+Pure string logic, no board. Also runnable as `python3 test_tags.py`.
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright.tags import ArtifactTag
+
+
+def test_parse_roundtrips_valid_tag():
+    s = "whisper-small-en-1.2.0+qnn2.37-htpv68-w8a16-win30s.qnnctx"
+    t = ArtifactTag.parse(s)
+    assert t.name == "whisper-small-en"
+    assert t.version == "1.2.0"
+    assert t.sdk == "2.37"
+    assert t.arch == "v68"
+    assert t.quant == "w8a16"
+    assert t.shape == "win30s"
+    assert t.ext == "qnnctx"
+    # str() is the exact inverse of parse()
+    assert str(t) == s
+
+
+def test_parse_handles_hyphenated_name_and_fp_quant():
+    # name may contain hyphens (greedy up to the last -<semver>+); quant may be fp16/fp32.
+    s = "laya-decision-tiny-10.20.30+qnn2.34-htpv73-fp16-ctx4096.dlc"
+    t = ArtifactTag.parse(s)
+    assert t.name == "laya-decision-tiny"
+    assert t.version == "10.20.30"
+    assert t.arch == "v73"
+    assert t.quant == "fp16"
+    assert t.ext == "dlc"
+    assert str(t) == s
+
+
+def test_parse_strips_surrounding_whitespace():
+    s = "m-1.0.0+qnn2.37-htpv68-w8a16-win30s.bin"
+    t = ArtifactTag.parse("  " + s + "\n")
+    assert str(t) == s
+
+
+def test_rejects_missing_version():
+    for bad in [
+        "whisper.qnnctx",                                  # no version at all
+        "whisper-small+qnn2.37-htpv68-w8a16-win30s.qnnctx",  # name but no semver
+        "whisper-1.0+qnn2.37-htpv68-w8a16-win30s.qnnctx",    # two-part version, not semver
+    ]:
+        _assert_rejected(bad)
+
+
+def test_rejects_missing_arch():
+    for bad in [
+        "whisper-1.0.0+qnn2.37-w8a16-win30s.qnnctx",       # no htp arch segment
+        "whisper-1.0.0+qnn2.37-htp68-w8a16-win30s.qnnctx",  # arch missing the 'v'
+        "whisper-1.0.0+htpv68-w8a16-win30s.qnnctx",         # no +qnn sdk tag
+    ]:
+        _assert_rejected(bad)
+
+
+def test_rejects_bad_extension():
+    for bad in [
+        "whisper-1.0.0+qnn2.37-htpv68-w8a16-win30s.onnx",   # not an allowed ext
+        "whisper-1.0.0+qnn2.37-htpv68-w8a16-win30s",        # no ext
+        "whisper-1.0.0+qnn2.37-htpv68-w8a16-win30s.qnnctx.bak",
+    ]:
+        _assert_rejected(bad)
+
+
+def test_is_context_binary_true_for_qnnctx_and_bin():
+    for ext in ("qnnctx", "bin"):
+        s = f"m-1.0.0+qnn2.37-htpv68-w8a16-win30s.{ext}"
+        assert ArtifactTag.parse(s).is_context_binary, ext
+
+
+def test_is_context_binary_false_for_dlc():
+    s = "m-1.0.0+qnn2.37-htpv68-w8a16-win30s.dlc"
+    assert ArtifactTag.parse(s).is_context_binary is False
+
+
+def _assert_rejected(bad: str):
+    try:
+        ArtifactTag.parse(bad)
+    except ValueError:
+        return
+    raise AssertionError(f"expected ValueError for {bad!r}")
+
+
+if __name__ == "__main__":
+    fns = [f for n, f in sorted(globals().items()) if n.startswith("test_")]
+    for f in fns:
+        f(); print("ok", f.__name__)
+    print(f"\n{len(fns)} passed")

chipwright/tests/test_task_eval.py

@@ -0,0 +1,164 @@
+"""Layer-2 task eval + behavioral-probe data — record-assembly, the lazy-import guard, the prompt sets.
+
+Pure / no heavy deps: the record-assembly is exercised with a FAKE scorer (no qai-hub, no model, no
+dataset), and the lazy-import-missing path is forced deterministically regardless of whether
+`qai-hub-models` happens to be installed. Runnable as `python3 test_task_eval.py`.
+"""
+import builtins
+import os
+import sys
+
+import pytest
+import yaml
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+import chipwright
+from chipwright import task_eval
+from chipwright.task_eval import (
+    TaskSpec, TASKS, resolve_task, task_record, run_task_eval, _qai_hub_scorer, _require_qai_hub,
+)
+
+
+# --- pure record-assembly (task_record) -----------------------------------------------------------
+
+def test_task_record_shape_and_types():
+    rec = task_record("wikitext_perplexity", 10.58, "wikitext", "test", 256)
+    assert rec == {
+        "metric": "wikitext_perplexity", "value": 10.58, "dataset": "wikitext",
+        "split": "test", "samples": 256,
+    }
+    assert isinstance(rec["value"], float) and isinstance(rec["samples"], int)
+
+
+def test_task_record_coerces_value_and_samples():
+    rec = task_record("mmlu_accuracy", 1, "mmlu", "test", "256")  # int value, str samples
+    assert rec["value"] == 1.0 and isinstance(rec["value"], float)
+    assert rec["samples"] == 256 and isinstance(rec["samples"], int)
+
+
+# --- resolve_task ---------------------------------------------------------------------------------
+
+def test_resolve_task_known_names():
+    assert resolve_task("wikitext").metric == "wikitext_perplexity"
+    assert resolve_task("mmlu").dataset == "mmlu"
+    assert resolve_task("tiny_mmlu").evaluator == "mmlu"
+
+
+def test_resolve_task_passthrough_spec():
+    spec = TaskSpec("x", "m", "d", "perplexity")
+    assert resolve_task(spec) is spec
+
+
+def test_resolve_task_unknown_refuses_by_name():
+    with pytest.raises(ValueError, match="unknown task"):
+        resolve_task("not_a_dataset")
+
+
+# --- run_task_eval with an injected FAKE scorer (no heavy dep, no model, no dataset) ---------------
+
+def test_run_task_eval_with_fake_scorer_assembles_records():
+    calls = []
+
+    def fake_scorer(spec, split, samples):
+        calls.append((resolve_task(spec).name, split, samples))
+        return {"wikitext_perplexity": 10.58, "mmlu_accuracy": 0.554}[resolve_task(spec).metric]
+
+    out = run_task_eval(tasks=("wikitext", "mmlu"), samples=256, split="test", scorer=fake_scorer)
+    assert out == [
+        {"metric": "wikitext_perplexity", "value": 10.58, "dataset": "wikitext",
+         "split": "test", "samples": 256},
+        {"metric": "mmlu_accuracy", "value": 0.554, "dataset": "mmlu",
+         "split": "test", "samples": 256},
+    ]
+    # the scorer saw each task with the requested split/samples (provenance threads through)
+    assert calls == [("wikitext", "test", 256), ("mmlu", "test", 256)]
+
+
+def test_run_task_eval_fake_scorer_needs_no_generator_or_dep():
+    # the injectable seam is exactly what makes this testable with no qai-hub and no model present
+    out = run_task_eval(tasks=("tiny_mmlu",), samples=100, scorer=lambda s, sp, n: 0.61)
+    assert out == [{"metric": "mmlu_accuracy", "value": 0.61, "dataset": "tiny_mmlu",
+                    "split": "test", "samples": 100}]
+
+
+# --- the lazy-import guard: raises BY NAME when qai-hub-models is absent ---------------------------
+
+def test_require_qai_hub_raises_by_name_when_absent(monkeypatch):
+    # force the import to fail regardless of whether qai-hub-models is installed in this env
+    real_import = builtins.__import__
+
+    def fake_import(name, *a, **k):
+        if name.startswith("qai_hub_models"):
+            raise ImportError("No module named 'qai_hub_models'")
+        return real_import(name, *a, **k)
+
+    monkeypatch.setattr(builtins, "__import__", fake_import)
+    with pytest.raises(RuntimeError, match=r"qai-hub-models|chipwright\[eval\]"):
+        _require_qai_hub()
+
+
+def test_qai_hub_scorer_refuses_missing_generator_by_name(monkeypatch):
+    # with the dep "present" (stubbed), a missing generator is still refused by name before any download
+    monkeypatch.setattr(task_eval, "_require_qai_hub", lambda: {})
+    with pytest.raises(RuntimeError, match="generator"):
+        _qai_hub_scorer(None)
+
+
+def test_qai_hub_scorer_refuses_missing_tokenizer_by_name(monkeypatch):
+    monkeypatch.setattr(task_eval, "_require_qai_hub", lambda: {})
+    with pytest.raises(RuntimeError, match="tokenizer"):
+        _qai_hub_scorer(object(), tokenizer=None)
+
+
+def test_importing_task_eval_does_not_import_qai_hub():
+    # the heavy dep must NOT be pulled in at module import (preflight discipline)
+    assert "qai_hub_models" not in sys.modules
+
+
+# --- the behavioral-probe data file ---------------------------------------------------------------
+
+def _load_behavior_prompts():
+    path = os.path.join(os.path.dirname(chipwright.__file__), "data", "behavior_prompts.yaml")
+    with open(path) as f:
+        return yaml.safe_load(f)
+
+
+def test_behavior_prompts_loads_with_both_sets_nonempty():
+    data = _load_behavior_prompts()
+    assert set(data) >= {"harmful", "benign"}
+    assert isinstance(data["harmful"], list) and len(data["harmful"]) > 0
+    assert isinstance(data["benign"], list) and len(data["benign"]) > 0
+
+
+def test_behavior_prompts_are_surface_twinned_and_well_formed():
+    data = _load_behavior_prompts()
+    # aligned by index (every harmful item has a benign control at the same position)
+    assert len(data["harmful"]) == len(data["benign"]) == 24
+    for p in data["harmful"] + data["benign"]:
+        assert isinstance(p, str) and p.strip()          # non-empty strings, no stray nulls
+    # the two sets are disjoint — a benign twin must not accidentally repeat a harmful prompt
+    assert set(data["harmful"]).isdisjoint(data["benign"])
+
+
+if __name__ == "__main__":
+    fns = [f for n, f in sorted(globals().items()) if n.startswith("test_") and callable(f)]
+    import types
+
+    class _MP:  # minimal monkeypatch shim for the __main__ runner
+        def __init__(self): self._undo = []
+        def setattr(self, obj, name, val):
+            old = getattr(obj, name); self._undo.append((obj, name, old)); setattr(obj, name, val)
+        def undo(self):
+            for obj, name, old in reversed(self._undo): setattr(obj, name, old)
+
+    passed = 0
+    for f in fns:
+        mp = _MP()
+        try:
+            n = f.__code__.co_argcount
+            f(mp) if n == 1 else f()
+            print("ok", f.__name__); passed += 1
+        finally:
+            mp.undo()
+    print(f"\n{passed} passed")

chipwright/tests/test_verify.py

@@ -0,0 +1,92 @@
+"""Numerical verification — cosine similarity and the per-output pass/fail report.
+
+Pure numpy, no board. Runnable as `python3 test_verify.py`.
+"""
+import math
+import os
+import sys
+
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright.verify import cosine, report
+
+
+def test_cosine_of_identical_vectors_is_one():
+    a = np.array([1.0, 2.0, 3.0, 4.0])
+    assert cosine(a, a) == 1.0
+
+
+def test_cosine_of_orthogonal_vectors_is_zero():
+    a = np.array([1.0, 0.0])
+    b = np.array([0.0, 1.0])
+    assert abs(cosine(a, b)) < 1e-12
+
+
+def test_cosine_of_opposite_vectors_is_minus_one():
+    a = np.array([1.0, 2.0, 3.0])
+    assert math.isclose(cosine(a, -a), -1.0, rel_tol=1e-12)
+
+
+def test_cosine_is_shape_agnostic_via_ravel():
+    # a 2-D NPU output and its flattened reference compare equal
+    a = np.arange(6, dtype=np.float64).reshape(2, 3)
+    b = np.arange(6, dtype=np.float64)
+    assert math.isclose(cosine(a, b), 1.0, rel_tol=1e-12)
+
+
+def test_cosine_handles_zero_vector_without_crashing():
+    a = np.zeros(4)
+    b = np.array([1.0, 2.0, 3.0, 4.0])
+    assert cosine(a, b) == 0.0
+    assert cosine(a, a) == 0.0
+
+
+def test_report_passes_when_all_outputs_meet_threshold():
+    npu = {"logits": np.array([1.0, 2.0, 3.0]), "hidden": np.array([0.5, 0.5])}
+    cpu = {"logits": np.array([1.0, 2.0, 3.0]), "hidden": np.array([0.5, 0.5])}
+    r = report(npu, cpu, threshold=0.99)
+    assert r["pass"] is True
+    assert math.isclose(r["cosine_min"], 1.0, rel_tol=1e-12)
+    assert r["threshold"] == 0.99
+
+
+def test_report_fails_when_an_output_is_below_threshold():
+    npu = {"good": np.array([1.0, 1.0]), "bad": np.array([1.0, 0.0])}
+    cpu = {"good": np.array([1.0, 1.0]), "bad": np.array([0.0, 1.0])}  # orthogonal -> cosine 0
+    r = report(npu, cpu, threshold=0.99)
+    assert r["pass"] is False
+    assert math.isclose(r["cosine_min"], 0.0, abs_tol=1e-12)
+
+
+def test_report_per_output_dict_is_keyed_by_output_name():
+    npu = {"a": np.array([1.0, 0.0]), "b": np.array([0.0, 1.0])}
+    cpu = {"a": np.array([1.0, 0.0]), "b": np.array([0.0, 1.0])}
+    r = report(npu, cpu)
+    assert set(r["per_output"]) == {"a", "b"}
+    assert math.isclose(r["per_output"]["a"], 1.0, rel_tol=1e-12)
+    # cosine_min is the minimum over the per-output values
+    assert r["cosine_min"] == min(r["per_output"].values())
+
+
+def test_report_only_compares_shared_keys():
+    # keys present in npu but missing from cpu are skipped, not an error
+    npu = {"shared": np.array([1.0, 1.0]), "npu_only": np.array([1.0])}
+    cpu = {"shared": np.array([1.0, 1.0]), "cpu_only": np.array([1.0])}
+    r = report(npu, cpu)
+    assert set(r["per_output"]) == {"shared"}
+
+
+def test_report_empty_inputs_do_not_crash():
+    r = report({}, {})
+    assert r["per_output"] == {}
+    assert r["cosine_min"] == 0.0
+    assert r["pass"] is False  # 0.0 < default 0.99
+
+
+if __name__ == "__main__":
+    fns = [f for n, f in sorted(globals().items()) if n.startswith("test_")]
+    for f in fns:
+        f(); print("ok", f.__name__)
+    print(f"\n{len(fns)} passed")