metrik-cli 0.1.0__py3-none-any.whl

metrik/cli/__init__.py

@@ -0,0 +1,36 @@
+"""The ``metrik`` command-line interface.
+
+ADR 012: Typer for type-hints-to-CLI, Rich for output quality. ``plan.md`` §10 correction 1
+moves the CLI early -- it grows with each module from Phase 2 rather than arriving at Phase 9,
+so there is a usable surface (and therefore real feedback) while the design can still change
+cheaply.
+
+The CLI is designed dual-mode (``plan.md`` §3.1): direct and in-process against the store, or
+``--remote`` against a daemon. Only the direct mode exists today; the daemon arrives in
+Phase 10. The split is worth keeping visible now because it is what lets the CLI ship without
+the API existing.
+"""
+
+from __future__ import annotations
+
+
+def _distribution_version(name: str) -> str:
+    from importlib.metadata import PackageNotFoundError, version
+
+    try:
+        return version(name)
+    except PackageNotFoundError:  # pragma: no cover - only in an uninstalled source tree
+        return "0+unknown"
+
+
+#: Read from the installed distribution rather than hardcoded.
+#:
+#: Both existed for a while and drifted: a 0.1.0 build reported 0.0.1 from `metrik --version`,
+#: found by installing the actual wheel rather than by any test. A version string is exactly
+#: the kind of duplicate that stays wrong quietly, because nothing imports it in anger.
+#:
+#: The fallback covers a source checkout with nothing installed, where there is no metadata to
+#: read and no released version to name.
+__version__ = _distribution_version("metrik-cli")
+
+__all__ = ["__version__"]

metrik/cli/commands/__init__.py

@@ -0,0 +1,7 @@
+"""Command groups.
+
+One module per group. Each stays a thin shell over the library: parse, call, render. Anything
+worth testing belongs below the CLI, where it can be tested without a subprocess.
+"""
+
+from __future__ import annotations

metrik/cli/commands/bench.py

@@ -0,0 +1,119 @@
+"""``metrik bench`` -- run a benchmark, or explain why it will not.
+
+ADR 019. HumanEval and MBPP ask a model to write code and then execute it, so this command
+establishes whether the machine can contain that **before** anything is generated. A refusal
+after an hour of decoding is a refusal that arrives too late to be a safety feature.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import typer
+from rich.table import Table
+from rich.text import Text
+
+from metrik.bench import TASKS, detect_runtimes, require_runtime, resolve_task
+from metrik.cli.console import out
+from metrik.core.errors import UnsupportedError
+
+__all__ = ["register"]
+
+_STYLE = {"ready": "measured", "not_running": "warn", "not_installed": "muted"}
+
+
+def register(app: typer.Typer) -> None:
+    @app.command("bench")
+    def bench(
+        model: Annotated[
+            str | None, typer.Argument(help="Model to evaluate. Omit with --check.")
+        ] = None,
+        task: Annotated[str | None, typer.Option("--task", help="Which benchmark to run.")] = None,
+        check: Annotated[
+            bool,
+            typer.Option("--check", help="Report what isolation is available and stop."),
+        ] = False,
+    ) -> None:
+        """Evaluate a model under container isolation -- or refuse, and say why.
+
+        There is no unsandboxed mode. A task that executes model-generated code runs in a
+        container or does not run (ADR 019), and no flag changes that.
+        """
+        if check:
+            _report_capability()
+            return
+
+        if model is None or task is None:
+            raise UnsupportedError(
+                "a model and a --task are required",
+                remediation="Run `metrik bench --check` to see the tasks and what can run here.",
+            )
+
+        spec = resolve_task(task)
+        if spec.executes_code:
+            # Before generation, deliberately. This raises when nothing can contain the run,
+            # and there is no branch that continues past it.
+            runtime = require_runtime(task_id=spec.id)
+            out.print(
+                f"[measured]{runtime.name} {runtime.version} ready[/measured] "
+                f"[muted]- {spec.id} will run under container isolation[/muted]"
+            )
+
+        raise UnsupportedError(
+            f"the {spec.id} runner is not implemented yet",
+            remediation=(
+                "Phase 5 ships the sandbox gate before the runner it gates. `metrik bench "
+                "--check` reports what isolation this machine has."
+            ),
+        )
+
+
+def _report_capability() -> None:
+    """What was probed, what each said, and what that permits."""
+    runtimes = detect_runtimes()
+
+    table = Table(box=None, pad_edge=False)
+    table.add_column("runtime", no_wrap=True)
+    table.add_column("state", no_wrap=True)
+    table.add_column("detail", overflow="fold")
+    for runtime in runtimes:
+        table.add_row(
+            Text(runtime.name),
+            Text(runtime.availability.value, style=_STYLE.get(runtime.availability.value, "")),
+            Text(runtime.version or runtime.detail),
+        )
+    out.print(table)
+    out.print()
+
+    usable = next((r for r in runtimes if r.ready), None)
+    tasks = Table(box=None, pad_edge=False)
+    tasks.add_column("task", style="type", no_wrap=True)
+    tasks.add_column("executes code", no_wrap=True)
+    tasks.add_column("can run here", no_wrap=True)
+    tasks.add_column("what it is", overflow="fold")
+    for spec in sorted(TASKS.values(), key=lambda s: s.id):
+        # The distinction the whole command exists to make visible: a task that executes code
+        # needs a runtime, and one that does not never did. Requiring a container for MMLU
+        # would push people to disable the check they need for HumanEval.
+        runnable = usable is not None or not spec.executes_code
+        tasks.add_row(
+            Text(spec.id),
+            Text("yes" if spec.executes_code else "no", style="warn" if spec.executes_code else ""),
+            Text("yes" if runnable else "no", style="measured" if runnable else "err"),
+            Text(spec.summary),
+        )
+    out.print(tasks)
+    out.print()
+
+    if usable is not None:
+        out.print(f"[measured]container isolation available via {usable.name}[/measured]")
+        return
+
+    out.print("[warn]no container runtime is available[/warn]")
+    for runtime in runtimes:
+        if runtime.remediation:
+            out.print(f"  [muted]{runtime.remediation}[/muted]")
+    out.print(
+        "[muted]Tasks that do not execute code still run. There is no unsandboxed mode "
+        "(ADR 019).[/muted]"
+    )

metrik/cli/commands/explore.py

@@ -0,0 +1,370 @@
+"""``metrik explore`` -- what is in this model?"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from metrik.cli.config import resolve_store_root
+from metrik.cli.console import out
+from metrik.core.errors import UserError
+from metrik.core.plugins import PluginRegistry
+from metrik.core.runs import RunRecorder
+from metrik.core.store import ArtifactStore
+from metrik.sdk import API_VERSION, LocalSpine
+
+__all__ = ["register"]
+
+#: The loader that drives the full chain when the chosen one only supplies a graph. A name
+#: resolved through the registry, not an import: the CLI must not depend on a plugin package.
+_DRIVER = "explorer"
+
+
+def _human(n: float | None) -> str:
+    if n is None:
+        return "-"
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+        if abs(n) < 1024 or unit == "TiB":
+            return f"{n:.1f} {unit}" if unit != "B" else f"{int(n)} B"
+        n /= 1024
+    return f"{n:.1f} TiB"  # pragma: no cover - loop always returns
+
+
+def _count(n: int | None) -> str:
+    if n is None:
+        return "-"
+    for threshold, suffix in ((1e9, "B"), (1e6, "M"), (1e3, "K")):
+        if abs(n) >= threshold:
+            return f"{n / threshold:.2f}{suffix}"
+    return str(n)
+
+
+def register(app: typer.Typer) -> None:
+    @app.command("explore")
+    def explore(
+        model: Annotated[
+            str, typer.Argument(help="Path to a model directory or .safetensors file.")
+        ],
+        store: Annotated[
+            Path | None, typer.Option("--store", help="Where to write artifacts.")
+        ] = None,
+        batch_size: Annotated[int, typer.Option(help="Memory estimate: batch size.")] = 1,
+        sequence_length: Annotated[
+            int, typer.Option(help="Memory estimate: prompt length in tokens.")
+        ] = 2048,
+        generated_tokens: Annotated[
+            int, typer.Option(help="Memory estimate: tokens to generate.")
+        ] = 256,
+        loader: Annotated[
+            str, typer.Option("--loader", help="Which model loader plugin to use.")
+        ] = "explorer",
+        allow_plugin: Annotated[
+            list[str] | None,
+            typer.Option("--allow-plugin", help="Opt into a third-party plugin by name."),
+        ] = None,
+        as_json: Annotated[
+            bool, typer.Option("--json", help="Emit JSON instead of a report.")
+        ] = False,
+    ) -> None:
+        """Analyse a model's architecture, parameters, and memory -- with no forward pass.
+
+        Reads safetensors headers and config.json only. No torch, no GPU, no weights loaded,
+        so this is cheap enough to run on anything and cannot accidentally be a benchmark.
+        """
+        location = resolve_store_root(store)
+        handle = ArtifactStore(location.path)
+
+        # Resolved through the registry, never imported by name. This module does not mention
+        # `metrik.explorer` anywhere, which is plan.md Verification #2: the built-in Explorer
+        # loads the way any third-party plugin would. If that stopped working we would find
+        # out here, in Phase 2, rather than in Phase 9 when the SDK is supposed to ship.
+        registry = PluginRegistry.discover(api_version=API_VERSION, allow=allow_plugin or ())
+        chosen = registry.get(loader, kind="model_loader").load()()
+
+        # Two roles, and most loaders fill only one. A *driver* runs the whole
+        # fingerprint/graph/summary/memory chain; a *graph provider* supplies just the middle
+        # step, which is all plan.md sec. 6.1 asks a ModelLoader for (URI -> ModelGraph).
+        #
+        # When the chosen loader is only a provider, the chain is driven by whichever loader
+        # can drive one -- resolved through the registry, never imported by name, so this
+        # module still mentions no plugin package anywhere (plan.md Verification #2).
+        driver, provider = chosen, None
+        if not hasattr(chosen, "run"):
+            if not hasattr(chosen, "provide_graph"):
+                raise UserError(
+                    f"the {loader!r} loader can neither run an explore chain nor supply a graph",
+                    remediation=f"Run `metrik plugins show {loader}` to see what it does.",
+                )
+            driver = registry.get(_DRIVER, kind="model_loader").load()()
+            provider = chosen.provide_graph
+
+        params = _params_for(
+            driver,
+            loader,
+            {
+                "batch_size": (batch_size, 1),
+                "sequence_length": (sequence_length, 2048),
+                "generated_tokens": (generated_tokens, 256),
+            },
+        )
+
+        # The recorder commits a Run whether this succeeds or raises. A failed explore that
+        # left no trace would be the one case with nothing to go back to -- a success can be
+        # re-derived from its artifacts, a failure cannot be re-derived from anything.
+        with RunRecorder(
+            handle,
+            job={"command": "explore", "model": model, "loader": loader, **params.model_dump()},
+        ) as run:
+            result = driver.run(lambda: LocalSpine(handle), model, params, graph_provider=provider)
+            run.add_outputs(result.all)
+
+        if as_json:
+            print(
+                json.dumps(
+                    {
+                        "run": str(run.committed.artifact_id),
+                        "fingerprint": result.fingerprint.payload,
+                        "summary": result.summary.payload,
+                        "memory": result.memory.payload,
+                        "artifact_ids": {
+                            "fingerprint": str(result.fingerprint.artifact_id),
+                            "graph": str(result.graph.artifact_id),
+                            "summary": str(result.summary.artifact_id),
+                            "memory": str(result.memory.artifact_id),
+                        },
+                    },
+                    indent=2,
+                )
+            )
+            return
+
+        _render(result)
+        out.print(f"[muted]metrik runs show {run.committed.artifact_id.short(8)}[/muted]")
+
+
+def _params_for(implementation: Any, loader: str, requested: dict[str, tuple[Any, Any]]) -> Any:
+    """Build a loader's parameter object from the flags it actually declares.
+
+    The CLI cannot assume every model_loader takes the Explorer's options. It did, and with a
+    single loader installed that was invisible; the second one turned `--loader torch` into a
+    raw pydantic ValidationError about extra fields. `Params` is `extra="forbid"` precisely so
+    this surfaces rather than being silently swallowed (sdk-contracts.md sec. 4).
+
+    Unsupported flags are dropped only when they are at their default. A flag the user
+    actually typed is never quietly ignored -- reporting "batch size 4" while the loader
+    computed nothing of the sort is exactly the sort of confident wrong number the artifact
+    rules exist to prevent.
+    """
+    model = implementation.params_model
+    accepted = set(getattr(model, "model_fields", {}))
+    kwargs = {name: value for name, (value, _) in requested.items() if name in accepted}
+
+    ignored = sorted(
+        name
+        for name, (value, default) in requested.items()
+        if name not in accepted and value != default
+    )
+    if ignored:
+        flags = ", ".join(f"--{name.replace('_', '-')}" for name in ignored)
+        raise UserError(
+            f"the {loader!r} loader does not accept {flags}",
+            remediation=(
+                f"Drop the option, or use a loader that supports it -- `metrik plugins show "
+                f"{loader}` lists what this one does."
+            ),
+        )
+    return model(**kwargs)
+
+
+def _render(result: Any) -> None:
+    fingerprint = result.fingerprint.payload
+    summary = result.summary.payload
+    memory = result.memory.payload
+    headline = summary["headline"]
+    graph = result.graph.payload
+
+    facts = Table.grid(padding=(0, 2))
+    facts.add_column(style="muted")
+    facts.add_column()
+    facts.add_row("model", Text(str(fingerprint["uri"])))
+    facts.add_row("architecture", Text(str(fingerprint["architecture"] or "unknown")))
+    facts.add_row(
+        "parameters",
+        Text(f"{headline['param_count_total']:,}  ({_count(headline['param_count_total'])})"),
+    )
+    facts.add_row("layers", Text(str(headline["n_layers"])))
+    facts.add_row("on disk", Text(_human(headline["size_on_disk_bytes"])))
+    facts.add_row("dtypes", Text(", ".join(fingerprint["dtype_on_disk"])))
+    facts.add_row("weight digest", Text(str(fingerprint["weight_digest"])[:22], style="hash"))
+    out.print(Panel(facts, title="model", title_align="left", border_style="muted"))
+
+    _render_topology(graph)
+
+    roles = Table(box=None, pad_edge=False)
+    roles.add_column("role", style="type", no_wrap=True)
+    roles.add_column("tensors", justify="right", no_wrap=True)
+    roles.add_column("parameters", justify="right", no_wrap=True)
+    roles.add_column("share", justify="right", no_wrap=True)
+    roles.add_column("bytes", justify="right", no_wrap=True)
+    for row in summary["by_role"]:
+        # A role whose parameters are all tied would otherwise render as "0" across the board,
+        # which reads as "this part of the model is empty". It is not -- it shares storage
+        # with another tensor, and saying so is the difference between an explanation and an
+        # alarming zero.
+        tied = row.get("param_count_tied")
+        if tied and not row["param_count"]:
+            roles.add_row(
+                Text(row["role"]),
+                Text(str(row["n_tensors"])),
+                Text(f"{_count(tied)} tied", style="muted"),
+                Text("-", style="muted"),
+                Text("shared", style="muted"),
+            )
+            continue
+        roles.add_row(
+            Text(row["role"]),
+            Text(str(row["n_tensors"])),
+            Text(_count(row["param_count"])),
+            Text(f"{row['param_fraction']:.1%}"),
+            Text(_human(row["bytes"])),
+        )
+    out.print(Panel(roles, title="parameters by role", title_align="left", border_style="muted"))
+
+    # Analytic, and labelled as such in the panel title. artifacts.md sec. 5 requires estimated
+    # and measured to be visually distinguishable; the theme reserves a colour for each.
+    mem = Table.grid(padding=(0, 2))
+    mem.add_column(style="muted")
+    mem.add_column(justify="right")
+    assumptions = memory["assumptions"]
+    mem.add_row("weights", Text(_human(memory["weights"]["bytes_total"]), style="analytic"))
+    mem.add_row("KV cache", Text(_human(memory["kv_cache"]["bytes_total"]), style="analytic"))
+    mem.add_row("", Text(""))
+    mem.add_row("min (weights only)", Text(_human(memory["total_bytes"]["min"]), style="analytic"))
+    mem.add_row("typical", Text(_human(memory["total_bytes"]["typical"]), style="analytic"))
+    mem.add_row("peak (with slack)", Text(_human(memory["total_bytes"]["peak"]), style="analytic"))
+    mem.add_row("", Text(""))
+    mem.add_row(
+        "at",
+        Text(
+            f"batch {assumptions['batch_size']}, "
+            f"{assumptions['sequence_length']}+{assumptions['generated_tokens']} tokens, "
+            f"{assumptions['kv_dtype']} KV"
+        ),
+    )
+    # Title via Text, not a markup string: "[analytic - not measured]" would be parsed as a
+    # style tag -- and `analytic` IS a style in this theme, so it vanished silently rather
+    # than erroring. Same trap as the lineage role labels.
+    out.print(
+        Panel(
+            mem,
+            title=Text("memory estimate (analytic - not measured)"),
+            title_align="left",
+            border_style="analytic",
+        )
+    )
+
+    if summary["notable"]:
+        notes = Table(box=None, pad_edge=False, show_header=False)
+        notes.add_column("", no_wrap=True)
+        notes.add_column("")
+        for note in summary["notable"]:
+            marker = {"warn": "[warn]![/warn]", "note": "[muted]*[/muted]"}.get(
+                note["severity"], "[muted]i[/muted]"
+            )
+            notes.add_row(marker, Text(note["message"]))
+        out.print(Panel(notes, title="notable", title_align="left", border_style="muted"))
+
+    # Omissions last but never absent. Every gap in this report is named, with a reason.
+    gaps = [
+        (artifact.envelope.artifact_type, omission)
+        for artifact in result.all
+        if artifact.envelope.partial is not None
+        for omission in artifact.envelope.partial.omissions
+    ]
+    if gaps:
+        table = Table(box=None, pad_edge=False)
+        table.add_column("field", style="warn", no_wrap=True)
+        table.add_column("why")
+        for artifact_type, omission in gaps:
+            table.add_row(
+                Text(f"{artifact_type.removeprefix('metrik.')}.{omission.field_path}"),
+                Text(omission.detail),
+            )
+        out.print(Panel(table, title="not determined", title_align="left", border_style="warn"))
+
+    out.print(
+        f"[muted]artifacts written to {result.summary.artifact_id.short()} and 3 others[/muted]"
+    )
+    out.print(f"[muted]metrik store lineage {result.memory.artifact_id.short(8)}[/muted]")
+
+
+def _render_topology(graph: dict[str, Any]) -> None:
+    """The architecture's shape, labelled with where it came from.
+
+    The panel title says "declared" because that is what it is: config.json is the model
+    author's statement about their own architecture, not something Metrik observed. The same
+    distinction the memory panel draws between analytic and measured (artifacts.md sec. 5) --
+    a reader who cannot tell a declaration from an observation will eventually trust the wrong
+    one.
+    """
+    topology = graph.get("topology") or {}
+    if not topology:
+        return
+
+    rows: list[tuple[str, str]] = []
+    if topology.get("n_layers") is not None:
+        rows.append(("layers", str(topology["n_layers"])))
+
+    heads, kv = topology.get("n_heads"), topology.get("n_kv_heads")
+    if heads is not None:
+        attention = str(topology.get("attention_kind") or "")
+        detail = f"{heads}"
+        if kv is not None and kv != heads:
+            # The ratio is the number that matters: it is the factor by which the KV cache
+            # shrinks, and it is why a GQA model fits where an MHA one does not.
+            detail += f" -> {kv} kv  ({attention}, {heads // kv}x smaller KV)"
+        elif attention:
+            detail += f"  ({attention})"
+        rows.append(("heads", detail))
+
+    if topology.get("head_dim") is not None:
+        rows.append(("head dim", str(topology["head_dim"])))
+    hidden, inter = topology.get("hidden_size"), topology.get("intermediate_size")
+    if hidden is not None:
+        rows.append(("hidden / inter", f"{hidden} / {inter}" if inter else str(hidden)))
+    if topology.get("vocab_size") is not None:
+        rows.append(("vocab", f"{topology['vocab_size']:,}"))
+
+    kinds = [topology.get("norm_kind"), topology.get("ffn_kind")]
+    position = topology.get("position_kind")
+    if position and (rope := topology.get("rope")):
+        position = f"{position}(theta={rope['theta']:g})"
+    kinds.append(position)
+    if any(kinds):
+        rows.append(("norm / ffn / pos", " . ".join(str(k) for k in kinds if k)))
+
+    if topology.get("tied_embeddings") is not None:
+        rows.append(("tied embeddings", "yes" if topology["tied_embeddings"] else "no"))
+
+    if not rows:
+        return
+
+    grid = Table.grid(padding=(0, 2))
+    grid.add_column(style="muted")
+    grid.add_column()
+    for label, value in rows:
+        grid.add_row(label, Text(value))
+    out.print(
+        Panel(
+            grid,
+            title=Text("topology (declared by config.json - not observed)"),
+            title_align="left",
+            border_style="muted",
+        )
+    )