modelport-cli 0.1.0__py3-none-any.whl

modelport/golden.py

@@ -0,0 +1,108 @@
+"""Golden data: saved inputs and expected outputs that prove a device matches Python."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from PIL import Image
+
+from .bundle import Bundle
+from .errors import MissingDependencyError, ModelPortError
+from .manifest import DType, FileRef, Golden, Manifest, TensorSpec
+from .preprocess import preprocess_image, to_le_bytes
+from .sources import SourceModel
+
+NUMPY_DTYPES: dict[DType, str] = {
+    DType.FLOAT32: "<f4",
+    DType.FLOAT16: "<f2",
+    DType.INT64: "<i8",
+    DType.INT32: "<i4",
+    DType.INT8: "i1",
+    DType.UINT8: "u1",
+    DType.BOOL: "?",
+}
+
+
+def dtype_of(array: np.ndarray) -> DType:
+    for dtype, code in NUMPY_DTYPES.items():
+        if np.dtype(code) == array.dtype.newbyteorder("<"):
+            return dtype
+    raise ModelPortError(f"unsupported tensor dtype {array.dtype}")
+
+
+def sample_image(width: int = 320, height: int = 240) -> Image.Image:
+    """A fixed, colourful test picture with gradients, a ring, and fine detail."""
+    y, x = np.mgrid[0:height, 0:width].astype(np.float64)
+    red = x * 255 / (width - 1)
+    green = y * 255 / (height - 1)
+    blue = 128 + 127 * np.sin(x / 9.0) * np.cos(y / 7.0)
+    distance = np.hypot(x - width * 0.6, y - height * 0.45)
+    ring = (distance > 40) & (distance < 60)
+    pixels = np.stack([red, green, blue], axis=-1)
+    pixels[ring] = (250, 240, 20)
+    return Image.fromarray(np.clip(np.rint(pixels), 0, 255).astype(np.uint8), "RGB")
+
+
+@dataclass(frozen=True)
+class GoldenData:
+    golden: Golden
+    inputs: dict[str, np.ndarray]
+    outputs: dict[str, np.ndarray]
+
+
+def make_golden(source: SourceModel, bundle: Bundle, image: Image.Image) -> GoldenData:
+    """Preprocess `image`, run the PyTorch model, and save both sides under golden/."""
+    try:
+        import torch
+    except ImportError as error:
+        raise MissingDependencyError("Creating golden data", "onnx") from error
+
+    inputs: dict[str, np.ndarray] = {}
+    for spec in source.inputs:
+        if spec.preprocess is None:
+            raise ModelPortError(f"input '{spec.name}' needs image preprocessing for golden data")
+        inputs[spec.name] = preprocess_image(image, spec)
+
+    with torch.no_grad():
+        result = source.module(*(torch.from_numpy(inputs[s.name]) for s in source.inputs))
+    results = result if isinstance(result, (tuple, list)) else (result,)
+    if len(results) != len(source.output_names):
+        raise ModelPortError(
+            f"model returned {len(results)} outputs, expected {len(source.output_names)}"
+        )
+    outputs = {
+        name: tensor.detach().cpu().numpy()
+        for name, tensor in zip(source.output_names, results, strict=True)
+    }
+
+    input_refs = {
+        name: bundle.write_bytes(f"golden/{name}.bin", to_le_bytes(a)) for name, a in inputs.items()
+    }
+    output_refs = {
+        name: bundle.write_bytes(f"golden/{name}.bin", to_le_bytes(a))
+        for name, a in outputs.items()
+    }
+    return GoldenData(Golden(inputs=input_refs, outputs=output_refs), inputs, outputs)
+
+
+def read_tensor(bundle: Bundle, ref: FileRef, spec: TensorSpec) -> np.ndarray:
+    """Load a golden file as an array with the tensor's dtype and shape."""
+    if ref.path is None:
+        raise ModelPortError("golden files must be inside the bundle")
+    data = bundle.path(ref.path).read_bytes()
+    # Copy so the array is writable; torch warns when wrapping read-only buffers.
+    array = np.frombuffer(data, dtype=NUMPY_DTYPES[spec.dtype]).copy()
+    if spec.shape.count(-1) <= 1:
+        array = array.reshape(spec.shape)
+    return array
+
+
+def golden_arrays(bundle: Bundle, manifest: Manifest) -> tuple[dict, dict]:
+    """Golden inputs and outputs of a manifest, keyed by tensor name."""
+    if manifest.golden is None:
+        raise ModelPortError(f"{manifest.id} has no golden data")
+    specs = {s.name: s for s in [*manifest.inputs, *manifest.outputs]}
+    inputs = {n: read_tensor(bundle, r, specs[n]) for n, r in manifest.golden.inputs.items()}
+    outputs = {n: read_tensor(bundle, r, specs[n]) for n, r in manifest.golden.outputs.items()}
+    return inputs, outputs

modelport/hashing.py

@@ -0,0 +1,17 @@
+"""File hashing helpers."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+_CHUNK = 1 << 20
+
+
+def sha256_file(path: str | Path) -> str:
+    """Lowercase hex SHA-256 of a file, read in 1 MiB chunks."""
+    digest = hashlib.sha256()
+    with Path(path).open("rb") as handle:
+        while chunk := handle.read(_CHUNK):
+            digest.update(chunk)
+    return digest.hexdigest()

modelport/inspection/__init__.py

@@ -0,0 +1,87 @@
+"""Read inputs, outputs, and metadata from model files without running them."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+from ..errors import MissingDependencyError, ModelPortError
+from ..hashing import sha256_file
+
+ModelFormat = Literal["onnx", "executorch", "gguf"]
+
+
+class InspectError(ModelPortError):
+    """A model file could not be inspected."""
+
+
+@dataclass(frozen=True)
+class TensorInfo:
+    name: str
+    dtype: str
+    shape: list[int | str]
+
+
+@dataclass(frozen=True)
+class ModelInfo:
+    format: ModelFormat
+    path: Path
+    size: int
+    sha256: str
+    inputs: list[TensorInfo] = field(default_factory=list)
+    outputs: list[TensorInfo] = field(default_factory=list)
+    metadata: dict[str, str] = field(default_factory=dict)
+
+
+def detect_format(path: Path) -> ModelFormat:
+    """Detect the format from magic bytes, falling back to the file extension."""
+    with path.open("rb") as handle:
+        head = handle.read(8)
+    if head[:4] == b"GGUF":
+        return "gguf"
+    if head[4:8] == b"ET12":
+        return "executorch"
+    if path.suffix.lower() == ".onnx":
+        return "onnx"
+    raise InspectError(f"{path}: not a recognized model file (.onnx, .pte, or .gguf)")
+
+
+def inspect_model(path: str | Path) -> ModelInfo:
+    """Describe a model file. Raises InspectError if it cannot be read."""
+    path = Path(path)
+    if not path.is_file():
+        raise InspectError(f"{path}: file not found")
+    fmt = detect_format(path)
+    if fmt == "onnx":
+        from .onnx import read_onnx
+
+        inputs, outputs, metadata = read_onnx(path)
+    elif fmt == "executorch":
+        from .executorch import read_executorch
+
+        inputs, outputs, metadata = read_executorch(path)
+    else:
+        from .gguf import read_gguf
+
+        inputs, outputs, metadata = read_gguf(path)
+    return ModelInfo(
+        format=fmt,
+        path=path,
+        size=path.stat().st_size,
+        sha256=sha256_file(path),
+        inputs=inputs,
+        outputs=outputs,
+        metadata=metadata,
+    )
+
+
+__all__ = [
+    "InspectError",
+    "MissingDependencyError",
+    "ModelFormat",
+    "ModelInfo",
+    "TensorInfo",
+    "detect_format",
+    "inspect_model",
+]

modelport/inspection/executorch.py

@@ -0,0 +1,32 @@
+"""ExecuTorch (.pte) program inspection."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from . import MissingDependencyError, TensorInfo
+
+# c10::ScalarType codes that map to manifest dtypes.
+_DTYPES = {0: "uint8", 1: "int8", 3: "int32", 4: "int64", 5: "float16", 6: "float32", 11: "bool"}
+
+
+def read_executorch(path: Path) -> tuple[list[TensorInfo], list[TensorInfo], dict[str, str]]:
+    try:
+        from executorch.runtime import Runtime
+    except ImportError as error:
+        raise MissingDependencyError("Inspecting .pte files", "executorch") from error
+
+    program = Runtime.get().load_program(str(path))
+    methods = sorted(program.method_names)
+    method = "forward" if "forward" in methods else methods[0]
+    meta = program.metadata(method)
+
+    def describe(prefix: str, info: object, index: int) -> TensorInfo:
+        dtype = int(info.dtype())  # type: ignore[attr-defined]
+        sizes = list(info.sizes())  # type: ignore[attr-defined]
+        return TensorInfo(f"{prefix}_{index}", _DTYPES.get(dtype, f"scalar_type_{dtype}"), sizes)
+
+    inputs = [describe("input", meta.input_tensor_meta(i), i) for i in range(meta.num_inputs())]
+    outputs = [describe("output", meta.output_tensor_meta(i), i) for i in range(meta.num_outputs())]
+    metadata = {"methods": ", ".join(methods), "inspected_method": method}
+    return inputs, outputs, metadata

modelport/inspection/gguf.py

@@ -0,0 +1,42 @@
+"""GGUF (llama.cpp) model inspection."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from . import MissingDependencyError, TensorInfo
+
+
+def read_gguf(path: Path) -> tuple[list[TensorInfo], list[TensorInfo], dict[str, str]]:
+    try:
+        import gguf
+    except ImportError as error:
+        raise MissingDependencyError("Inspecting .gguf files", "gguf") from error
+
+    reader = gguf.GGUFReader(str(path))
+
+    def value(key: str) -> Any:
+        field = reader.fields.get(key)
+        return field.contents() if field is not None else None
+
+    arch = value("general.architecture") or "unknown"
+    file_type = value("general.file_type")
+    if file_type is not None:
+        try:
+            quant = gguf.LlamaFileType(file_type).name.removeprefix("MOSTLY_").lower()
+        except ValueError:
+            quant = str(file_type)
+    else:
+        quant = "unknown"
+
+    metadata = {
+        "architecture": str(arch),
+        "name": str(value("general.name") or "unknown"),
+        "quantization": quant,
+        "context_length": str(value(f"{arch}.context_length") or "unknown"),
+        "tensors": str(len(reader.tensors)),
+        "chat_template": "yes" if "tokenizer.chat_template" in reader.fields else "no",
+        "gguf_version": str(value("GGUF.version") or "unknown"),
+    }
+    return [], [], metadata

modelport/inspection/onnx.py

@@ -0,0 +1,44 @@
+"""ONNX model inspection."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from . import MissingDependencyError, TensorInfo
+
+# onnx.TensorProto element types that map to manifest dtypes.
+_DTYPES = {1: "float32", 10: "float16", 7: "int64", 6: "int32", 3: "int8", 2: "uint8", 9: "bool"}
+
+
+def read_onnx(path: Path) -> tuple[list[TensorInfo], list[TensorInfo], dict[str, str]]:
+    try:
+        import onnx
+    except ImportError as error:
+        raise MissingDependencyError("Inspecting .onnx files", "onnx") from error
+
+    model = onnx.load(str(path), load_external_data=False)
+    graph = model.graph
+    initializers = {init.name for init in graph.initializer}
+
+    def describe(value: Any) -> TensorInfo:
+        tensor_type = value.type.tensor_type
+        elem = tensor_type.elem_type
+        dtype = _DTYPES.get(elem, onnx.TensorProto.DataType.Name(elem).lower())
+        shape: list[int | str] = [
+            dim.dim_value if dim.HasField("dim_value") else (dim.dim_param or "?")
+            for dim in tensor_type.shape.dim
+        ]
+        return TensorInfo(name=value.name, dtype=dtype, shape=shape)
+
+    inputs = [describe(v) for v in graph.input if v.name not in initializers]
+    outputs = [describe(v) for v in graph.output]
+    opset = next((o.version for o in model.opset_import if o.domain in ("", "ai.onnx")), None)
+    external = any(init.data_location == onnx.TensorProto.EXTERNAL for init in graph.initializer)
+    metadata = {
+        "ir_version": str(model.ir_version),
+        "opset": str(opset) if opset is not None else "unknown",
+        "producer": f"{model.producer_name} {model.producer_version}".strip() or "unknown",
+        "external_data": "yes" if external else "no",
+    }
+    return inputs, outputs, metadata

modelport/manifest/__init__.py

@@ -0,0 +1,39 @@
+"""The `modelport.json` manifest: the contract between the CLI and the Dart packages."""
+
+from .files import FileRef
+from .models import (
+    MANIFEST_FILENAME,
+    SCHEMA_VERSION,
+    GenerationDefaults,
+    Golden,
+    LlmConfig,
+    Manifest,
+    Runtime,
+    Task,
+    Tolerance,
+    Variant,
+)
+from .postprocess import ClassificationPostprocess, DetectionPostprocess
+from .tensors import DType, ImagePreprocess, InputSpec, OutputSpec, ResizeSpec, TensorSpec
+
+__all__ = [
+    "MANIFEST_FILENAME",
+    "SCHEMA_VERSION",
+    "ClassificationPostprocess",
+    "DType",
+    "DetectionPostprocess",
+    "FileRef",
+    "GenerationDefaults",
+    "Golden",
+    "ImagePreprocess",
+    "InputSpec",
+    "LlmConfig",
+    "Manifest",
+    "OutputSpec",
+    "ResizeSpec",
+    "Runtime",
+    "Task",
+    "TensorSpec",
+    "Tolerance",
+    "Variant",
+]

modelport/manifest/base.py

@@ -0,0 +1,22 @@
+"""Shared building blocks for manifest models."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from pydantic import BaseModel, ConfigDict, Field
+
+Sha256 = Annotated[
+    str,
+    Field(pattern=r"^[0-9a-f]{64}$", description="Lowercase hex SHA-256 digest of the file."),
+]
+
+
+class StrictModel(BaseModel):
+    """Base for all manifest models.
+
+    Unknown fields are rejected so that typos in hand-written manifests are caught early.
+    Models are immutable once validated.
+    """
+
+    model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True)

modelport/manifest/files.py

@@ -0,0 +1,59 @@
+"""File references inside a model bundle."""
+
+from __future__ import annotations
+
+import re
+
+from pydantic import Field, field_validator, model_validator
+
+from .base import Sha256, StrictModel
+
+_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:")
+
+# Same rules as `_check_relative_path`, for JSON Schema users such as editors and Dart.
+# Python's validator does not use it because pydantic-core's regex engine has no lookahead.
+SAFE_PATH_PATTERN = r"^(?!/)(?![A-Za-z]:)(?!.*\\)(?!.*//)(?!.*/$)(?!(?:.*/)?\.\.?(?:/|$)).+$"
+
+
+class FileRef(StrictModel):
+    """A file that belongs to a model, such as weights, labels, or golden test data.
+
+    Exactly one of `path` or `url` must be set.
+    """
+
+    model_config = StrictModel.model_config | {
+        "json_schema_extra": {"oneOf": [{"required": ["path"]}, {"required": ["url"]}]}
+    }
+
+    path: str | None = Field(
+        default=None,
+        description="POSIX path relative to the manifest file. Use for files inside the bundle.",
+        examples=["onnx-fp32/model.onnx", "labels.txt"],
+        json_schema_extra={"pattern": SAFE_PATH_PATTERN},
+    )
+    url: str | None = Field(
+        default=None,
+        pattern=r"^https://\S+$",
+        description=(
+            "Absolute HTTPS URL. Use for files hosted elsewhere, such as an original GGUF repo."
+        ),
+    )
+    size: int = Field(gt=0, description="File size in bytes.")
+    sha256: Sha256
+
+    @field_validator("path")
+    @classmethod
+    def _check_relative_path(cls, value: str | None) -> str | None:
+        if value is None:
+            return value
+        if value.startswith("/") or "\\" in value or _WINDOWS_DRIVE.match(value):
+            raise ValueError("path must be a relative POSIX path, like 'onnx-fp32/model.onnx'")
+        if any(part in ("", ".", "..") for part in value.split("/")):
+            raise ValueError("path must not contain empty, '.' or '..' segments")
+        return value
+
+    @model_validator(mode="after")
+    def _check_exactly_one_location(self) -> FileRef:
+        if (self.path is None) == (self.url is None):
+            raise ValueError("set exactly one of 'path' or 'url'")
+        return self