chipwright 0.2.0__py3-none-any.whl

chipwright/task_eval.py

@@ -0,0 +1,190 @@
+"""Layer-2 task evaluation — WikiText perplexity + MMLU accuracy, reference-side and REPORTED-not-gated.
+
+The generative gate has two layers kept separate (see `verify_llm.py`):
+
+  - **Layer 1** (`verify_llm.tripwire_report`) is the on-chip backend tripwire: the same quantized
+    artifact, run greedily on the chip, must match a pinned CPU reference. It can only rule degradation
+    *out* — near the FP baseline the correlation between its cheap signals and task quality collapses
+    (the "silent zone", arXiv 2606.19558). That is *why* Layer 2 exists.
+  - **Layer 2 — task** (this file) answers the other question: *is the quantized model still good enough?*
+    That is a property of the **artifact**, not of a particular chip, so it is measured **reference-side
+    on the build host** (perplexity and MMLU both need per-token logits over a corpus, which the chip's
+    Genie path does not emit — Genie returns text, not logits). FR-L04: task metrics are
+    necessary-not-sufficient and are REPORTED, never gated. The gate is Layer 1 + the behavioral flag.
+
+This module is deliberately thin. It does NOT re-implement perplexity or MMLU — it wraps qai-hub's own
+eval harness (`qai_hub_models.datasets` + `qai_hub_models.evaluators.{ppl,mmlu}_evaluator`, the same
+`add_from_dataset` / `get_accuracy_score` path the qwen reference test drives). Chipwright's job here is
+the **record shape + named provenance** (dataset / split / samples) so a reader can reproduce the number,
+not the eval kernel.
+
+Two honest constraints shape the design:
+  - `qai-hub-models` is a heavy dependency (torch + transformers + the dataset downloaders). It is
+    **lazy-imported inside the scorer** and, if absent, raised **by name** with the install to run — the
+    same preflight discipline as the sshpass/adb/tokenizers checks elsewhere. There is NO top-level
+    import of it, so `import chipwright.task_eval` stays cheap and the record-assembly stays testable.
+  - Record-assembly is kept **pure and injectable**: `run_task_eval` takes an optional `scorer` seam,
+    so the record shape can be unit-tested with a fake scorer and no model, dataset, or heavy dep.
+
+Honest ceiling: this is a reference-side **quality** characterization of the artifact, not an on-silicon
+measurement — the record must say "build host, ORT-GenAI / FP16 torch," not imply the chip was measured.
+Report WikiText as ΔPPL% vs the model's own FP baseline (PPL can paradoxically *drop* under quantization
+and hide errors); use >=256 WikiText samples for a believable number (tiny_mmlu / 256-512 MMLU for CI vs
+release). It is also why task metrics alone are not enough: a quantized LLM can keep its perplexity
+(1.03x) while silently losing 15.2% of its refusals (arXiv 2606.09864) — that failure is invisible here
+and is caught by the separate behavioral probe (`verify_llm.behavior_report`).
+"""
+from __future__ import annotations
+
+from collections import namedtuple
+
+# A task's identity: the chipwright task name, the metric key written into the record, the qai-hub
+# dataset name, and which evaluator computes it. Pure data — no heavy import to define this.
+TaskSpec = namedtuple("TaskSpec", "name metric dataset evaluator")
+
+# The task registry. Datasets confirmed present in qai_hub_models.datasets (wikitext, mmlu, tiny_mmlu).
+#   - "wikitext"  -> WikiText perplexity (PerplexityEvaluator.get_accuracy_score() == exp(mean NLL))
+#   - "mmlu"      -> MMLU accuracy (MMLUEvaluator.get_accuracy_score() == correct / total)
+#   - "tiny_mmlu" -> the ~100-question CI-grade MMLU subset (same accuracy metric, cheaper)
+TASKS = {
+    "wikitext": TaskSpec("wikitext", "wikitext_perplexity", "wikitext", "perplexity"),
+    "mmlu": TaskSpec("mmlu", "mmlu_accuracy", "mmlu", "mmlu"),
+    "tiny_mmlu": TaskSpec("tiny_mmlu", "mmlu_accuracy", "tiny_mmlu", "mmlu"),
+}
+
+
+def resolve_task(task) -> TaskSpec:
+    """Resolve a task name (or a ready TaskSpec) to its TaskSpec; refuse an unknown name BY NAME.
+
+    An unknown task is refused with the known set rather than silently skipped — a typo must not quietly
+    drop a metric a reader expects in the record.
+    """
+    if isinstance(task, TaskSpec):
+        return task
+    try:
+        return TASKS[task]
+    except KeyError:
+        raise ValueError(f"unknown task {task!r} (known: {', '.join(sorted(TASKS))})") from None
+
+
+def task_record(metric, value, dataset, split, samples) -> dict:
+    """Assemble one `task:` record (the contract block shape). Pure — the unit-testable core.
+
+    Mirrors the contract's Layer-2 `task:` list entry: `{metric, value, dataset, split, samples}`. The
+    value is stored as a float so the record serializes cleanly; everything a reader needs to reproduce
+    the number (which dataset, which split, how many samples) travels with it (FR-L04 / FR-L09).
+    """
+    return {
+        "metric": metric,
+        "value": float(value),
+        "dataset": dataset,
+        "split": split,
+        "samples": int(samples),
+    }
+
+
+def run_task_eval(generator=None, tasks=("wikitext",), samples=256, split="test",
+                  *, scorer=None, tokenizer=None, context_length=4096, device="cpu") -> list[dict]:
+    """Reference-side Layer-2 task metrics for an LLM artifact (build host, NOT on-device).
+
+    Returns one `task:` record per requested metric — each naming dataset/split/samples so a reader can
+    reproduce it (FR-L04, contract `task:` block). Perplexity/MMLU are REPORTED here, never gated; the
+    gate is Layer 1 (`verify_llm.tripwire_report`) plus the behavioral flag. Call this at build/publish
+    time to populate the record's `task:` block — not on the hot `run` path that drives the board.
+
+    The scoring seam is injectable for testing: pass `scorer(spec, split, samples) -> float` to compute
+    the metric without the heavy qai-hub dependency, a model, or a dataset. When `scorer` is None, the
+    real qai-hub harness is wired against `generator` (an `LLM_Generator` wrapping the quantized/FP model)
+    + `tokenizer`, lazy-importing qai-hub and raising BY NAME if it is absent.
+
+    Example return:
+      [{"metric": "wikitext_perplexity", "value": 10.58, "dataset": "wikitext",
+        "split": "test", "samples": 256},
+       {"metric": "mmlu_accuracy", "value": 0.554, "dataset": "mmlu", "split": "test", "samples": 256}]
+    """
+    specs = [resolve_task(t) for t in tasks]
+    score = scorer if scorer is not None else _qai_hub_scorer(
+        generator, tokenizer=tokenizer, context_length=context_length, device=device)
+    return [task_record(s.metric, score(s, split, samples), s.dataset, split, samples) for s in specs]
+
+
+# --- the qai-hub harness scorer (lazy, heavy; raises BY NAME if the dep is absent) ----------------
+
+def _require_qai_hub():
+    """Import the qai-hub eval harness lazily, or raise BY NAME naming the install (preflight style).
+
+    `qai-hub-models` pulls in torch + transformers + the dataset downloaders — far too heavy to import at
+    module load. Like the sshpass/adb/tokenizers preflights, its absence is reported by name with the
+    exact install to run, never as a bare ImportError deep in a traceback. Returns the handful of symbols
+    the real scorer needs.
+    """
+    try:
+        from qai_hub_models.datasets import get_dataset_from_name
+        from qai_hub_models.datasets.common import DatasetSplit
+        from qai_hub_models.evaluators.ppl_evaluator import PerplexityEvaluator
+        from qai_hub_models.evaluators.mmlu_evaluator import MMLUEvaluator
+    except ImportError as e:  # fail by name, not silently — mirrors the adb/sshpass/tokenizers preflights
+        raise RuntimeError(
+            "task_eval: the 'qai-hub-models' package is required to compute reference-side WikiText "
+            "perplexity / MMLU accuracy (it supplies the datasets and the eval kernels) but could not be "
+            "imported. Install the 'eval' extra:  pip install 'chipwright[eval]'") from e
+    return {
+        "get_dataset_from_name": get_dataset_from_name,
+        "DatasetSplit": DatasetSplit,
+        "PerplexityEvaluator": PerplexityEvaluator,
+        "MMLUEvaluator": MMLUEvaluator,
+    }
+
+
+def _split_enum(DatasetSplit, split):
+    """Map a split string ('test'/'train') to the qai-hub `DatasetSplit` enum; refuse an unknown split."""
+    key = str(split).upper()
+    try:
+        return getattr(DatasetSplit, key)
+    except AttributeError:
+        raise ValueError(f"unknown split {split!r} (known: test, train)") from None
+
+
+def _qai_hub_scorer(generator, *, tokenizer=None, context_length=4096, device="cpu"):
+    """Build the real reference-side scorer over qai-hub's eval harness.
+
+    Returns `score(spec, split, samples) -> float`. For each task it builds the named dataset
+    (`get_dataset_from_name(spec.dataset, split=<enum>, num_samples=samples)`), constructs the matching
+    evaluator (`PerplexityEvaluator` / `MMLUEvaluator`, each `(context_length, device, tokenizer)`),
+    drives it over the generator (`evaluator.add_from_dataset(generator, dataset, eval_iterations=samples)`
+    — the same path qai-hub's own model tests use), and reads `evaluator.get_accuracy_score()`
+    (PPL == exp(mean NLL); MMLU == correct / total). Exactly the qai-hub harness, no re-implemented math.
+
+    `generator` is an `LLM_Generator` wrapping the quantized (ORT-GenAI) or FP16-torch model, built on the
+    build host (model/recipe-specific, so it is passed in, not constructed here). It and `tokenizer` are
+    required for the real path; their absence is refused by name before any dataset is downloaded.
+    """
+    mods = _require_qai_hub()
+
+    if generator is None:
+        raise RuntimeError(
+            "task_eval: run_task_eval needs an `LLM_Generator` (the quantized/FP model wrapped for the "
+            "qai-hub harness, built on the build host) to score tasks, but none was passed. Pass "
+            "`generator=` (and `tokenizer=`), or inject a `scorer=` for a dry run. Never on-device — "
+            "perplexity/MMLU need logits the chip's Genie path does not emit.")
+    if tokenizer is None:
+        raise RuntimeError(
+            "task_eval: the qai-hub PerplexityEvaluator/MMLUEvaluator require the model's `tokenizer`; "
+            "pass `tokenizer=` alongside `generator=`.")
+
+    def _evaluator(kind):
+        if kind == "perplexity":
+            return mods["PerplexityEvaluator"](context_length, device, tokenizer)
+        if kind == "mmlu":
+            return mods["MMLUEvaluator"](context_length, device, tokenizer)
+        raise ValueError(f"unknown evaluator {kind!r}")
+
+    def score(spec, split, samples):
+        spec = resolve_task(spec)
+        dataset = mods["get_dataset_from_name"](
+            spec.dataset, split=_split_enum(mods["DatasetSplit"], split), num_samples=int(samples))
+        ev = _evaluator(spec.evaluator)
+        ev.add_from_dataset(generator, dataset, eval_iterations=int(samples))
+        return float(ev.get_accuracy_score())
+
+    return score

chipwright/tests/conftest.py

@@ -0,0 +1,12 @@
+"""Make `import chipwright` work under pytest regardless of where pytest is invoked.
+
+The repo root (the directory that *contains* the `chipwright/` package) is put on sys.path so the
+tests import the package being developed, not an installed copy. Mirrors the sys.path shim each test
+file carries for standalone `python3 <file>` runs.
+"""
+import os
+import sys
+
+_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if _REPO_ROOT not in sys.path:
+    sys.path.insert(0, _REPO_ROOT)

chipwright/tests/test_adapt.py

@@ -0,0 +1,61 @@
+"""Graph-adaptation tests — pure, no board/QAIRT; skip cleanly if `onnx` isn't installed."""
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright import adapt
+from chipwright.recipe import Adaptation
+
+try:
+    import onnx
+    from onnx import TensorProto, helper
+    HAVE_ONNX = True
+except ImportError:
+    HAVE_ONNX = False
+
+
+def _reshape_onnx(path, allowzero):
+    inp = helper.make_tensor_value_info("x", TensorProto.FLOAT, [2, 3])
+    out = helper.make_tensor_value_info("y", TensorProto.FLOAT, [6])
+    shape = helper.make_tensor("shape", TensorProto.INT64, [1], [6])
+    node = helper.make_node("Reshape", ["x", "shape"], ["y"], allowzero=allowzero)
+    g = helper.make_graph([node], "g", [inp], [out], [shape])
+    onnx.save(helper.make_model(g, opset_imports=[helper.make_opsetid("", 13)]), path)
+
+
+def _allowzero_count(path):
+    m = onnx.load(path)
+    return sum(1 for n in m.graph.node if n.op_type == "Reshape"
+               and any(a.name == "allowzero" and a.i == 1 for a in n.attribute))
+
+
+def test_reshape_allowzero_cleared():
+    if not HAVE_ONNX:
+        return
+    with tempfile.TemporaryDirectory() as d:
+        src = os.path.join(d, "m.onnx")
+        _reshape_onnx(src, allowzero=1)
+        assert _allowzero_count(src) == 1
+        out, logs = adapt.apply(src, [Adaptation(kind="reshape_allowzero", params={})], d)
+        assert _allowzero_count(out) == 0
+        assert any("reshape_allowzero" in m for m in logs)
+
+
+def test_noop_when_not_declared():
+    # no supported adaptation declared -> original path returned unchanged, no logs
+    out, logs = adapt.apply("/nonexistent.onnx", [], None)
+    assert out == "/nonexistent.onnx" and logs == []
+
+
+def test_missing_file_is_graceful():
+    out, logs = adapt.apply("/nope.onnx", [Adaptation(kind="reshape_allowzero", params={})], None)
+    assert out == "/nope.onnx" and logs and "skipped" in logs[0]
+
+
+if __name__ == "__main__":
+    fns = [f for n, f in sorted(globals().items()) if n.startswith("test_")]
+    for f in fns:
+        f(); print("ok", f.__name__)
+    print(f"\n{len(fns)} passed" + ("" if HAVE_ONNX else " (onnx-dependent test was a no-op)"))

chipwright/tests/test_build.py

@@ -0,0 +1,279 @@
+"""The build verb is testable with no QAIRT and no board: the recipe loader and the op-check rule
+table are pure, and the executors are exercised with the external tools MOCKED (shutil.which +
+subprocess.run monkeypatched). So this suite runs green on a laptop with nothing installed.
+"""
+import os
+import sys
+import tempfile
+import types
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from chipwright import build, config, recipe, sdk
+from chipwright.errors import (ContextGenError, FidelityBelowThreshold,
+                               OpUnsupported, PreflightError)
+
+# --- a minimal valid recipe document -----------------------------------------------------------
+RECIPE_YAML = """
+name: whisper-small-en
+version: 1.2.0
+route: classic
+source:
+  format: onnx
+  sha256: deadbeef
+  url: hf://openai/whisper-small.en/encoder.onnx
+target:
+  arch: v68
+  quant: w8a16
+  shape: win30s
+  sdk_range: ">=2.34,<2.40"
+adaptations:
+  - static_shape: {bucket: win30s, dims: {audio: [1, 80, 3000]}}
+  - erf_to_tanh: {ops: [Erf]}
+  - mask_clamp: {from: -3.4e38, to: -1e4}
+quant:
+  weights: 8
+  activations: 16
+  bias: 32
+  per_channel: true
+  calibration_ref: calib/whisper_encoder_30s.npz
+  algorithm: min_max
+host:
+  pre: modalities.asr.frontend:log_mel
+  post: modalities.asr.decoders.ctc:greedy
+  deps: [numpy]
+graphs:
+  - {name: encoder, runs_on: npu}
+  - {name: decode, runs_on: host}
+"""
+
+
+def _write(text, suffix=".yaml"):
+    fd, path = tempfile.mkstemp(suffix=suffix)
+    os.write(fd, text.encode())
+    os.close(fd)
+    return path
+
+
+def _recipe():
+    return recipe.load(_write(RECIPE_YAML))
+
+
+# --- recipe loader -----------------------------------------------------------------------------
+def test_recipe_loads_full_schema():
+    r = _recipe()
+    assert r.name == "whisper-small-en" and r.version == "1.2.0" and r.route == "classic"
+    assert r.source.format == "onnx" and r.source.sha256 == "deadbeef"
+    assert r.target.arch == "v68" and r.target.sdk_range == ">=2.34,<2.40"
+    assert r.quant.activations == 16 and r.quant.bias == 32 and r.quant.per_channel is True
+    assert r.quant_tag == "w8a16"
+    assert r.host.pre == "modalities.asr.frontend:log_mel"
+    assert [g.runs_on for g in r.graphs] == ["npu", "host"]
+    assert len(r.npu_graphs) == 1
+    assert r.adaptation("erf_to_tanh") is not None and r.adaptation("nope") is None
+
+
+def test_recipe_rejects_bad_runs_on():
+    bad = RECIPE_YAML.replace("runs_on: host", "runs_on: gpu")
+    try:
+        recipe.load(_write(bad)); assert False, "expected ValueError"
+    except ValueError:
+        pass
+
+
+def test_recipe_rejects_unknown_route_and_missing_field():
+    for mut in [RECIPE_YAML.replace("route: classic", "route: magic"),
+                RECIPE_YAML.replace("  arch: v68\n", "")]:
+        try:
+            recipe.load(_write(mut)); assert False, "expected ValueError"
+        except ValueError:
+            pass
+
+
+# --- op-check rule table (fed a fake op list, no onnx file needed) ------------------------------
+class _FakeTarget:
+    def __init__(self, arch): self.htp_arch = arch
+
+
+def _patch_op_scan(monkey_ops, fp16=False):
+    """Patch sdk._onnx_op_types to return a fixed op list so the rule table is tested directly."""
+    orig = sdk._onnx_op_types
+    sdk._onnx_op_types = lambda path: (monkey_ops, {"fp16": fp16}, None)
+    return orig
+
+
+def test_will_it_run_flags_erf_and_gather():
+    orig = _patch_op_scan(["Conv", "Erf", "Gather", "MatMul"])
+    try:
+        # point at a real existing file so the os.path.exists guard passes; content is ignored (scan patched)
+        rep = sdk.will_it_run(__file__, _FakeTarget("v68"))
+    finally:
+        sdk._onnx_op_types = orig
+    assert rep.supported is True                        # Erf/Gather are adaptable, not unsupported
+    flagged = {f["op"] for f in rep.flags}
+    assert "Erf" in flagged and "Gather" in flagged
+
+
+def test_will_it_run_marks_truly_unsupported_op():
+    orig = _patch_op_scan(["Conv", "NonMaxSuppression"])
+    try:
+        rep = sdk.will_it_run(__file__, _FakeTarget("v68"))
+    finally:
+        sdk._onnx_op_types = orig
+    assert rep.supported is False and "NonMaxSuppression" in rep.unsupported
+
+
+def test_will_it_run_flags_fp16_on_v6x():
+    orig = _patch_op_scan(["Conv"], fp16=True)
+    try:
+        rep = sdk.will_it_run(__file__, _FakeTarget("v68"))
+    finally:
+        sdk._onnx_op_types = orig
+    assert any(f["op"] == "<fp16-tensor>" for f in rep.flags)
+
+
+def test_will_it_run_degrades_without_onnx_file():
+    rep = sdk.will_it_run("/no/such/model.onnx", _FakeTarget("v68"))
+    assert rep.supported is True and rep.note                 # graceful, not a crash
+
+
+# --- tool mocking ------------------------------------------------------------------------------
+class _FakeCompleted:
+    def __init__(self, returncode=0, stdout="ok", stderr=""):
+        self.returncode, self.stdout, self.stderr = returncode, stdout, stderr
+
+
+def _mock_tools(present=True, returncode=0):
+    """Return (which, run) replacements. present=False ⇒ tool missing (PreflightError path)."""
+    def which(name, path=None):
+        return f"/opt/qairt/bin/{name}" if present else None
+    def run(cmd, capture_output=True, text=True, timeout=None):
+        return _FakeCompleted(returncode=returncode)
+    return which, run
+
+
+def _env(tmp):
+    return config.Env(cache_dir=tmp)
+
+
+# --- executor guards: missing tool raises PreflightError ---------------------------------------
+def test_missing_tool_raises_preflight():
+    which, run = _mock_tools(present=False)
+    orig_which, orig_run = build.shutil.which, build.subprocess.run
+    build.shutil.which, build.subprocess.run = which, run
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            try:
+                build.convert(_recipe(), _env(tmp)); assert False, "expected PreflightError"
+            except PreflightError:
+                pass
+    finally:
+        build.shutil.which, build.subprocess.run = orig_which, orig_run
+
+
+def test_classic_route_reports_preflight_axis_when_tool_missing():
+    which, run = _mock_tools(present=False)
+    orig_which, orig_run = build.shutil.which, build.subprocess.run
+    build.shutil.which, build.subprocess.run = which, run
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            res = build.classic(_recipe(), _env(tmp))
+    finally:
+        build.shutil.which, build.subprocess.run = orig_which, orig_run
+    assert res.ok is False and res.failed_axis == "preflight" and res.artifact_path is None
+
+
+# --- mocked successful chain returns a BuildResult(ok=True) ------------------------------------
+def test_classic_chain_succeeds_mocked():
+    which, run = _mock_tools(present=True, returncode=0)
+    orig_which, orig_run = build.shutil.which, build.subprocess.run
+    build.shutil.which, build.subprocess.run = which, run
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            res = build.classic(_recipe(), _env(tmp))
+    finally:
+        build.shutil.which, build.subprocess.run = orig_which, orig_run
+    assert res.ok is True and res.failed_axis is None
+    assert res.artifact_path and res.artifact_path.endswith(".bin")
+    assert res.tag == "whisper-small-en-1.2.0+qnn2.34-htpv68-w8a16-win30s.bin"
+    assert any("qairt-converter" in line for line in res.logs)
+    assert any("qnn-context-binary-generator" in line for line in res.logs)
+
+
+def test_dlc_route_ships_dlc_mocked():
+    which, run = _mock_tools(present=True, returncode=0)
+    orig_which, orig_run = build.shutil.which, build.subprocess.run
+    build.shutil.which, build.subprocess.run = which, run
+    try:
+        r = _recipe(); r.route = "dlc_route"
+        with tempfile.TemporaryDirectory() as tmp:
+            res = build.build(r, _env(tmp))      # dispatch by recipe.route
+    finally:
+        build.shutil.which, build.subprocess.run = orig_which, orig_run
+    assert res.ok is True and res.artifact_path.endswith(".dlc")
+    assert res.tag.endswith(".dlc")
+    assert any("on-board" in line for line in res.logs)
+
+
+def test_context_tool_failure_raises_contextgen_axis():
+    # converter/quantizer succeed, context generator returns nonzero → ContextGenError, failed_axis=context
+    def which(name, path=None):
+        return f"/opt/qairt/bin/{name}"
+    def run(cmd, capture_output=True, text=True, timeout=None):
+        rc = 1 if "context-binary-generator" in cmd[0] else 0
+        return _FakeCompleted(returncode=rc, stderr="VTCM overflow")
+    orig_which, orig_run = build.shutil.which, build.subprocess.run
+    build.shutil.which, build.subprocess.run = which, run
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            res = build.classic(_recipe(), _env(tmp))
+            # and the executor raises the typed error directly
+            dlc = os.path.join(tmp, "x.dlc")
+            raised = False
+            try:
+                build.context(dlc, _FakeTarget("v68"), _env(tmp))
+            except ContextGenError:
+                raised = True
+    finally:
+        build.shutil.which, build.subprocess.run = orig_which, orig_run
+    assert res.ok is False and res.failed_axis == "context"
+    assert raised
+
+
+# --- host preflight + config roundtrip ---------------------------------------------------------
+def test_host_preflight_reports_missing_tools():
+    orig = config.shutil.which
+    config.shutil.which = lambda name, path=None: None
+    try:
+        pf = config.host_preflight(config.Env())
+    finally:
+        config.shutil.which = orig
+    assert pf.ok is False and set(config.REQUIRED_TOOLS).issubset(set(pf.missing))
+
+
+def test_env_write_read_roundtrip():
+    with tempfile.TemporaryDirectory() as tmp:
+        path = os.path.join(tmp, ".qualcomm-env")
+        e = config.Env(hosts=[{"name": "rb3", "host": "10.0.0.5"}], sdk_roots=["/opt/qairt/2.37"],
+                       keys=["~/.ssh/id_rsa"], cache_dir=os.path.join(tmp, "cache"))
+        config.write_env(e, path)
+        back = config.read_env(path)
+    assert back.hosts[0]["name"] == "rb3" and back.sdk_root == "/opt/qairt/2.37"
+    assert back.cache_dir.endswith("cache")
+
+
+# --- fidelity gate (the publish gate, pure) ----------------------------------------------------
+def test_fidelity_gate_rejects_low_cosine():
+    try:
+        build.fidelity_gate({"loads": True, "cosine_min": 0.80, "threshold": 0.99}); assert False
+    except FidelityBelowThreshold:
+        pass
+    # a passing report does not raise
+    build.fidelity_gate({"loads": True, "cosine_min": 0.995, "threshold": 0.99})
+
+
+if __name__ == "__main__":
+    fns = [f for n, f in sorted(globals().items()) if n.startswith("test_") and isinstance(f, types.FunctionType)]
+    for f in fns:
+        f(); print("ok", f.__name__)
+    print(f"\n{len(fns)} passed")