tan-cli 0.7.0__py3-none-any.whl

tan/model/measure.py

@@ -0,0 +1,216 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Host reference run + A/B compare (tan-cli#1287, host tier only).
+
+Ported from alp-sdk `integration/model-edge-ai` (`scripts/alp_model/measure.py`,
+alp-sdk#933). The host run (`backend="cpu-host"`) is a FUNCTIONAL + host-latency
+reference, NOT the target SoM's performance. The on-device tier (target
+latency, peak arena SRAM, per-rail energy via the on-board monitor IC) is the
+bench-gated follow-on that fills the SAME `RunResult` schema with real numbers;
+the host run leaves `power_mj`/`peak_sram_kib` `None` and nothing here
+fabricates them; the device tier (`tan.core.model_device`) fills `peak_sram_kib`
+from the app's ENERGY-CFG `sram_peak_bytes` and energy from its capture. The energy-integration math (`integrate_energy`, `windowed_delta`,
+`EnergyMeasurement`) is the on-device tier's, ported unchanged and PURE; the
+estimate-vs-measured feedback loop is not ported yet.
+
+`numpy`/`onnxruntime` (the optional `model` extra) are imported inside the
+functions that use them.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from statistics import median
+from time import perf_counter
+from typing import Any, Sequence
+
+
+class MeasureError(Exception):
+    """The model could not be loaded or run for measurement."""
+
+
+# (t_seconds, power_watts) — power_watts is an on-board monitor Power-register (3h) read.
+PowerSample = tuple[float, float]
+
+
+def integrate_energy(samples: Sequence[PowerSample]) -> float:
+    """Trapezoidal time-integration of power samples into millijoules:
+    E = Σ 0.5*(P_i + P_i+1)*Δt over consecutive samples (Δt in seconds,
+    P in watts -> joules, *1000 -> mJ). Samples must be in ascending t order
+    (the on-board monitor has no on-chip energy accumulator, so this is the whole
+    point of taking repeated Power-register samples host-side).
+    <2 samples -> 0.0 (no interval to integrate; not an error — a single
+    reading carries no energy information, so returning 0.0 lets a caller
+    that hasn't collected a real window yet compose without special-casing)."""
+    if len(samples) < 2:
+        return 0.0
+    energy_j = 0.0
+    for (t0, p0), (t1, p1) in zip(samples, samples[1:]):
+        energy_j += 0.5 * (p0 + p1) * (t1 - t0)
+    return energy_j * 1000.0
+
+
+def windowed_delta(active_samples: Sequence[PowerSample],
+                    idle_samples: Sequence[PowerSample], n_inferences: int) -> float:
+    """Idle-baseline-subtracted energy per inference, in mJ: the delta between
+    an active (inferring) window and an idle window, divided by how many
+    inferences ran in the active window.
+
+    Precondition: active_samples and idle_samples MUST cover equal wall-clock
+    duration (same integration window length). This is a subtraction of two
+    integrals, not a rate comparison -- unequal durations bias the baseline
+    (e.g. a longer idle window subtracts more idle energy than the active
+    window actually contains). The caller/firmware is responsible for taking
+    symmetric windows; this function does not (and should not) assert it,
+    since real hardware timing has jitter that would make a hard equality
+    check false-trip."""
+    if n_inferences < 1:
+        raise ValueError(f"n_inferences must be >= 1, got {n_inferences}")
+    return (integrate_energy(active_samples) - integrate_energy(idle_samples)) / n_inferences
+
+
+@dataclass(frozen=True)
+class EnergyMeasurement:
+    """A real bench-measured energy result — never fabricate one; `run_host()`
+    leaves `RunResult.energy` as None instead. `source`/`scope` default to (and
+    are validated as) the honest labels: this is a board-level carrier-rail
+    delta, NEVER an isolated NPU/U85/U55/M55 silicon energy figure."""
+    value_mj_per_inference: float
+    rails: list[str]
+    n_inferences: int
+    window_ms: float
+    sample_count: int
+    spread_mj: float | None = None      # std/spread across M repeated windows;
+                                         # None for a single window
+    source: str = "measured"
+    scope: str = "carrier-rail-delta"
+
+    def __post_init__(self) -> None:
+        if self.source != "measured" or self.scope != "carrier-rail-delta":
+            raise ValueError(
+                "EnergyMeasurement is board-level-delta only: source must be "
+                "'measured' and scope 'carrier-rail-delta' — never label this "
+                "NPU/U85/U55/M55 silicon energy.")
+
+
+@dataclass(frozen=True)
+class RunResult:
+    backend: str                 # "cpu-host" (reference) | on-device backend later
+    latency_ms: float            # median wall-clock per inference
+    output_argmax: int | None
+    peak_sram_kib: float | None    # None on host — on-device only
+    power_mj: float | None       # None on host — on-board monitor read (HW-gated)
+    runs: int
+    energy: EnergyMeasurement | None = None  # None until a real bench run populates it
+    # Set when the caller's sample was cast to the model input dtype (tan-cli#1497).
+    input_cast: str | None = None
+
+
+_ORT_DTYPES = {
+    "float": "float32", "float16": "float16", "double": "float64",
+    "int64": "int64", "int32": "int32", "int16": "int16", "int8": "int8",
+    "uint64": "uint64", "uint32": "uint32", "uint16": "uint16", "uint8": "uint8",
+    "bool": "bool",
+}
+
+
+def _np_dtype(ort_type: str) -> str:
+    """`tensor(int64)` -> `int64`; anything unrecognised (sequences, strings)
+    falls back to float32, the previous behaviour (tan-cli#1486)."""
+    inner = ort_type[ort_type.find("(") + 1:ort_type.rfind(")")] if "(" in ort_type else ort_type
+    return _ORT_DTYPES.get(inner, "float32")
+
+
+def _sample(shape: list, ort_type: str, seed: int) -> Any:
+    import numpy as np
+    dtype = _np_dtype(ort_type)
+    dims = [(1 if not isinstance(d, int) else d) for d in shape]
+    if np.dtype(dtype).kind == "f":
+        return np.random.default_rng(seed).standard_normal(dims).astype(dtype)
+    return np.zeros(dims, dtype=dtype)   # token ids / pixels / flags: a valid, deterministic value
+
+
+def describe_input_cast(sample_dtype: Any, want: str) -> str | None:
+    """A sentence when casting a `sample_dtype` sample to @want changes its
+    dtype (tan-cli#1497), naming the silent value loss when float becomes
+    integer/bool; None when no cast happens."""
+    import numpy as np
+    have = np.dtype(sample_dtype)
+    if have == np.dtype(want):
+        return None
+    msg = f"input sample dtype {have} was cast to the model input dtype {want}"
+    wd = np.dtype(want)
+    if wd.kind == "b":
+        msg += "; every nonzero value became True"
+    elif have.kind == "f" and wd.kind in "iu":
+        msg += "; fractional values were truncated toward zero"
+    elif not np.can_cast(have, wd, "safe"):
+        msg += "; precision or range may be lost"
+    return msg
+
+
+def default_input(onnx_path: Path, *, seed: int = 0) -> Any:
+    """Deterministic random sample matching the model's first input shape.
+    Dynamic dims collapse to 1 (intent: a dynamic BATCH dim; a symbolic H/W
+    would also become 1). Wrapped in MeasureError so a bad model fails clean."""
+    import numpy as np
+    import onnxruntime as ort
+    try:
+        sess = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
+        first = sess.get_inputs()[0]
+        return _sample(first.shape, first.type, seed)
+    except Exception as exc:
+        raise MeasureError(f"could not load model {Path(onnx_path).name}: {exc}") from exc
+
+
+def run_host(onnx_path: Path, input_array: Any, *, runs: int = 20) -> RunResult:
+    import numpy as np
+    import onnxruntime as ort
+    try:
+        sess = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
+        inputs = sess.get_inputs()
+        # The sample feeds the FIRST input in that input's own dtype; any further
+        # inputs get a deterministic default sample (tan-cli#1486).
+        raw = np.asarray(input_array)
+        want = _np_dtype(inputs[0].type)
+        input_cast = describe_input_cast(raw.dtype, want)
+        feed = {inputs[0].name: raw.astype(want)}
+        for extra in inputs[1:]:
+            feed[extra.name] = _sample(extra.shape, extra.type, 0)
+        out = sess.run(None, feed)[0]          # warm-up + a real output
+        times = []
+        for _ in range(max(1, runs)):
+            t0 = perf_counter()
+            sess.run(None, feed)
+            times.append((perf_counter() - t0) * 1000.0)
+    except Exception as exc:
+        raise MeasureError(f"host run failed: {exc}") from exc
+    return RunResult(backend="cpu-host", latency_ms=round(median(times), 3),
+                     output_argmax=int(np.asarray(out).ravel().argmax()),
+                     peak_sram_kib=None, power_mj=None, runs=len(times),
+                     input_cast=input_cast)
+
+
+@dataclass(frozen=True)
+class ABComparison:
+    faster: str                  # "a" | "b" | "tie"
+    latency_ratio: float | None  # b.latency_ms / a.latency_ms; None if a.latency_ms == 0
+                                  # (avoids emitting non-finite JSON `Infinity`)
+    a_latency_ms: float
+    b_latency_ms: float
+    size_delta_bytes: int | None  # size_b - size_a
+
+
+def compare(a: RunResult, b: RunResult, *, size_a: int | None = None,
+            size_b: int | None = None) -> ABComparison:
+    if a.latency_ms < b.latency_ms:
+        faster = "a"
+    elif b.latency_ms < a.latency_ms:
+        faster = "b"
+    else:
+        faster = "tie"
+    ratio = (b.latency_ms / a.latency_ms) if a.latency_ms else None
+    delta = (size_b - size_a) if (size_a is not None and size_b is not None) else None
+    return ABComparison(faster=faster,
+                        latency_ratio=round(ratio, 4) if ratio is not None else None,
+                        a_latency_ms=a.latency_ms, b_latency_ms=b.latency_ms,
+                        size_delta_bytes=delta)

tan/model/npu_ops_validate.py

@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Schema validation for the per-NPU op-support tables `tan model check`
+reads (tan-cli#1298).
+
+`metadata/npu_ops/**` used to have no schema, so `analyze.py` carried only
+`isinstance` guards. `metadata/schemas/npu-ops-v1.schema.json` now exists on
+alp-sdk `dev` (alp-sdk#1801 closed, landed with alp-sdk#1470); a bound SDK may
+or may not ship it. This module is the read-path half of tan-cli#964's rule
+for that table family: validate on read, always report, and skip-but-disclose
+when the schema cannot be used -- the policy lives here, the validator stays
+the one in `tan.core.metadata_schema`.
+
+Three outcomes for a table that is about to be used:
+
+* schema usable, table valid -> used.
+* schema usable, table violates it -> NOT used (skipped like a malformed
+  file, so the backend reads `undetermined` unless another table covers the
+  variant), with one note naming the file, JSON pointer and what was found.
+* schema absent, unreadable, not valid JSON, or not a valid JSON Schema ->
+  the table IS used under `analyze.py`'s isinstance guards, with ONE note per
+  backend saying the tables were not validated. A broken schema is no
+  evidence against a table, so it is never blamed on one.
+
+Only a table that matches the requested variant is validated; a table for
+another variant is neither validated nor mentioned. Selection reads
+`applies_to.variant` off a not-yet-validated document, which is safe because
+`_table_variant` is isinstance-guarded to return `""` for any wrong shape --
+and a table whose variant matches is then validated before it is accepted, so
+an unvalidated document can only ever fail to match, never be scored.
+
+The notes ride the static-screen report only. The `--exact` compiled report
+and the bench-point report build their own `notes` and do not carry them.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from tan.core.metadata_schema import (
+    missing_schema_note,
+    npu_ops_schema_path,
+    schema_unusable_reason,
+    validate_document,
+)
+
+# A badly broken table can violate the schema dozens of times; the first few
+# name the problem, the rest just bloat the envelope.
+_MAX_VIOLATIONS_SHOWN = 5
+
+
+def _posix(value: object) -> str:
+    return str(value).replace("\\", "/")
+
+
+class TableValidator:
+    """Validates the tables of ONE backend directory against the npu-ops
+    schema, working out whether the schema is usable lazily -- on the first
+    table that is actually about to be used -- and disclosing "not validated"
+    at most once.
+
+    Every path in a note is written relative to the metadata root, with
+    forward slashes (`npu_ops/ethos_u/u85@vela-5.2.0.json`,
+    `schemas/npu-ops-v1.schema.json`): the note rides the envelope, and an
+    absolute host path in it is both noise and a leak. (It also made a test
+    whose tmp dir name contained a word the note-scanning tests forbid fail
+    on CI, tan-cli#1298.)"""
+
+    def __init__(self, metadata_root: Path | str, table_dir: Path) -> None:
+        self._root = Path(metadata_root)
+        self._schema_path = npu_ops_schema_path(metadata_root)
+        self._table_dir = table_dir
+        self._probed = False
+        self._usable = False
+
+    def _rel(self, path: Path | str) -> str:
+        try:
+            return Path(path).relative_to(self._root).as_posix()
+        except ValueError:
+            return Path(path).name
+
+    def _scrub(self, text: str) -> str:
+        """@text with the metadata root's absolute spellings replaced -- a
+        defence for a reason string that carries a path despite
+        `schema_unusable_reason` already dropping the OS filename."""
+        for spelling in {str(self._root), self._root.as_posix(), _posix(self._root)}:
+            text = text.replace(spelling, "<metadata>")
+        return text
+
+    def _probe(self, notes: list[str]) -> None:
+        self._probed = True
+        where = f"{self._rel(self._table_dir)}: not validated -- "
+        schema = self._rel(self._schema_path)
+        reason = schema_unusable_reason(self._schema_path)
+        if reason is not None:
+            notes.append(f"{where}the schema at {schema} could not be loaded "
+                         f"({self._scrub(reason)}); the tables are used under "
+                         "the built-in shape checks only")
+            return
+        try:
+            absent = missing_schema_note(self._schema_path, source=self._table_dir) is not None
+        except OSError as exc:
+            # `Path.is_file()` raises on a permission-denied ancestor on
+            # Python 3.12/3.13 (see the comment above `_is_table_file` in
+            # analyze.py): "cannot tell" is not "absent".
+            notes.append(f"{where}the schema at {schema} could not be inspected "
+                         f"({exc.strerror or 'OS error'}); the tables are used "
+                         "under the built-in shape checks only")
+            return
+        if absent:
+            notes.append(f"{where}no schema at {schema} in this checkout")
+            return
+        self._usable = True
+
+    def accepts(self, doc: object, table_path: Path, notes: list[str]) -> bool:
+        """True when *table_path* (already parsed to *doc*) may be used.
+        Appends to *notes* the not-validated disclosure (once) or this
+        table's violation note."""
+        if not self._probed:
+            self._probe(notes)
+        if not self._usable:
+            return True
+        errors = validate_document(doc, self._schema_path, self._rel(table_path))
+        if not errors:
+            return True
+        shown = "; ".join(errors[:_MAX_VIOLATIONS_SHOWN])
+        extra = len(errors) - _MAX_VIOLATIONS_SHOWN
+        more = f" (+{extra} more)" if extra > 0 else ""
+        notes.append(f"npu-ops table skipped, it fails npu-ops-v1.schema.json: "
+                     f"{shown}{more}. This table is not used.")
+        return False

tan/model/package.py

@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Assemble/parse the .alpmodel binary container (header + CBOR manifest + blobs).
+
+`CONTAINER_VERSION` describes the BINARY FRAME below -- the 24-byte header's
+field layout and the 8-byte blob-table entries -- and nothing about which keys
+the CBOR manifest carries. Adding a manifest key is therefore NOT a container
+change: alp-sdk's on-device reader skips keys it does not know
+(`src/common/alp_model.c`, see `tan.model.manifest`'s module docstring for the
+measured evidence), while a CONTAINER_VERSION it does not know is a hard
+ALP_ERR_VERSION refusal for every fielded device. Do not move this constant to
+announce a manifest field.
+"""
+from __future__ import annotations
+import os
+import struct
+from pathlib import Path
+from .manifest import Manifest
+
+MAGIC = b"ALPM"
+CONTAINER_VERSION = 1
+_HEADER = struct.Struct("<4sHHIIII")   # magic, ver, flags, mft_off, mft_len, tbl_off, blob_count
+_HEADER_SIZE = _HEADER.size            # 24
+_TBL_ENTRY = struct.Struct("<II")      # blob off, blob len
+
+
+def write_package(mft: Manifest, blobs: list[bytes]) -> bytes:
+    mft_bytes = mft.to_cbor()
+    mft_off = _HEADER_SIZE
+    tbl_off = mft_off + len(mft_bytes)
+    blobs_off = tbl_off + _TBL_ENTRY.size * len(blobs)
+
+    table, blob_region, cursor = b"", b"", blobs_off
+    for b in blobs:
+        table += _TBL_ENTRY.pack(cursor, len(b))
+        blob_region += b
+        cursor += len(b)
+
+    header = _HEADER.pack(MAGIC, CONTAINER_VERSION, 0,
+                          mft_off, len(mft_bytes), tbl_off, len(blobs))
+    return header + mft_bytes + table + blob_region
+
+
+def _unpack_header(head: bytes) -> tuple[int, int, int, int]:
+    """(mft_off, mft_len, tbl_off, blob_count) from a validated 24-byte header."""
+    magic, ver, _flags, mft_off, mft_len, tbl_off, n = _HEADER.unpack_from(head, 0)
+    if magic != MAGIC:
+        raise ValueError(f"bad magic {magic!r}")
+    if ver != CONTAINER_VERSION:
+        raise ValueError(f"unsupported container version {ver}")
+    return mft_off, mft_len, tbl_off, n
+
+
+def read_manifest_file(path: Path) -> Manifest:
+    """The manifest of an on-disk package, WITHOUT reading its blobs.
+
+    `read_package` needs the whole container in memory; a caller that only
+    wants to read back what a package says about itself (`tan model build`
+    reporting a shipped target's caveats) would then pay a full copy of every
+    compiled blob -- megabytes for a real vision model -- to read a few
+    hundred bytes of CBOR. This seeks to the manifest region instead, so the
+    cost is O(manifest), not O(package).
+
+    Reading it back off disk, rather than reporting the in-memory object that
+    was just written, is deliberate: it reports what the ARTIFACT says. A
+    caveat lost by the encoder is then visibly missing from the build output
+    instead of being narrated from a manifest that never reached the file."""
+    with open(path, "rb") as f:
+        head = f.read(_HEADER_SIZE)
+        if len(head) < _HEADER_SIZE:
+            raise ValueError(f"truncated container: {len(head)} bytes, "
+                             f"header is {_HEADER_SIZE}")
+        mft_off, mft_len, _tbl_off, _n = _unpack_header(head)
+        # Bound the read against the real file size BEFORE issuing it.
+        # `mft_off`/`mft_len` are untrusted u32s off the wire, and unlike
+        # `read_package`'s slice of an already-loaded buffer, `f.read(n)`
+        # allocates n bytes -- an `mft_len` of 0xFFFFFFFF would ask for 4 GiB
+        # from a 5 KiB file. Compared subtraction-style, the same shape
+        # alp-sdk's reader uses for the same fields (`src/common/alp_model.c`:
+        # "adding the untrusted u32 offset and length first can wrap").
+        size = os.fstat(f.fileno()).st_size
+        if mft_off > size or mft_len > size - mft_off:
+            raise ValueError(f"manifest region {mft_off}+{mft_len} runs past "
+                             f"the end of a {size}-byte container")
+        f.seek(mft_off)
+        blob = f.read(mft_len)
+    if len(blob) != mft_len:
+        raise ValueError(f"truncated manifest: {len(blob)} of {mft_len} bytes")
+    return Manifest.from_cbor(blob)
+
+
+def read_package(raw: bytes) -> tuple[Manifest, list[bytes]]:
+    """The manifest AND blobs of an in-memory `.alpmodel` buffer.
+
+    Every region this reads (`mft_off`/`mft_len`, the blob table itself, and
+    each table entry's `off`/`length`) is bounds-checked against `len(raw)`
+    before the matching slice, using the same subtraction-style compare
+    `read_manifest_file` uses (`off > size or length > size - off`, chosen
+    there because adding the untrusted u32 offset and length first can wrap).
+    The motivation for checking at all differs, though: `read_manifest_file`
+    checks to avoid an over-allocation risk (`f.read(n)` allocates n bytes up
+    front), while slicing `bytes` cannot over-allocate but also does not
+    raise -- `b"AB"[0:999999]` is `b"AB"`, silently truncated, not a crash
+    (tan-cli#1045). A corrupted or adversarial header/table entry must be
+    refused here, the same way `read_manifest_file`'s file-based sibling
+    already refuses one."""
+    size = len(raw)
+    if size < _HEADER_SIZE:
+        raise ValueError(f"truncated container: {size} bytes, header is {_HEADER_SIZE}")
+    mft_off, mft_len, tbl_off, n = _unpack_header(raw)
+    if mft_off > size or mft_len > size - mft_off:
+        raise ValueError(f"manifest region {mft_off}+{mft_len} runs past "
+                         f"the end of a {size}-byte container")
+    mft = Manifest.from_cbor(raw[mft_off:mft_off + mft_len])
+    tbl_len = n * _TBL_ENTRY.size
+    if tbl_off > size or tbl_len > size - tbl_off:
+        raise ValueError(f"blob table region {tbl_off}+{tbl_len} (for {n} "
+                         f"entries) runs past the end of a {size}-byte container")
+    blobs = []
+    for i in range(n):
+        off, length = _TBL_ENTRY.unpack_from(raw, tbl_off + i * _TBL_ENTRY.size)
+        if off > size or length > size - off:
+            raise ValueError(f"blob {i} region {off}+{length} runs past "
+                             f"the end of a {size}-byte container")
+        blobs.append(raw[off:off + length])
+    return mft, blobs