inferencekit-cli 0.1.0__py3-none-any.whl

inferencekit/ops/nsys.py

@@ -0,0 +1,224 @@
+"""nsys capture / nsys analyze: a timeline of every kernel, copy and CUDA API call.
+
+Capture command (environment reduced to an allowlist, see proc.clean_env):
+
+    nsys profile -t cuda,nvtx --cuda-graph-trace=node -s none --cpuctxsw=none -o <out> <argv>
+    nsys export --type sqlite -o <out>.sqlite <out>.nsys-rep
+
+  -t cuda,nvtx                trace CUDA API + GPU activity, and NVTX annotation ranges
+  --cuda-graph-trace=node     record each kernel inside a CUDA graph, not one opaque block
+                              (without it, a decode step under graphs is a single bar)
+  -s none / --cpuctxsw=none   no CPU sampling or context-switch tracing: less overhead,
+                              smaller files; this capture is about the GPU timeline
+
+What the report file holds: nsys stores the profiled program's full command line,
+working directory, user name and NAME=value of every environment variable it received,
+plus the host name and the host's process list. The evidence record keeps only a digest
+of the command and the variable names. The exported .sqlite is scrubbed of those values
+(nsys_sqlite.scrub_metadata); the .nsys-rep cannot be edited and keeps them, so treat it
+as sensitive. That is also why secret-looking names are refused for --keep-env.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from inferencekit import evidence as ev
+from inferencekit.analysis import nsys_sqlite
+from inferencekit.gpuinfo import gpu_identity
+from inferencekit.proc import clean_env, dropped_config_env, require, run, tool_version
+from inferencekit.registry import Arg, OpError, operation
+
+REP_IS_SENSITIVE = ("the .nsys-rep stores the profiled command line, working directory, user and host name, and "
+                    "NAME=value of every passed variable ({names}): treat it as sensitive and do not share it; the "
+                    "exported .sqlite has these values blanked")
+
+
+def export_sqlite(nsys: str, rep: Path, out: Path) -> None:
+    r = run([nsys, "export", "--type", "sqlite", "--force-overwrite", "true", "-o", str(out), str(rep)],
+            timeout=3600)
+    if not r.ok or not out.is_file():
+        raise OpError(f"nsys export failed (exit {r.returncode}): {r.stderr[-800:]}")
+
+
+def redact_outputs(staging: Path) -> None:
+    """The program's own output may echo secrets: redact it as soon as the run ends."""
+    for name in ("stdout.txt", "stderr.txt"):
+        p = staging / name
+        if p.is_file():
+            p.write_text(ev.redact_text(p.read_text(errors="replace")))
+
+
+@operation("capture_nsys", ("nsys", "capture"), "Run a command under 'nsys profile' with an allowlisted "
+           "environment (env -i equivalent) and export the trace to sqlite.",
+           [Arg("keep_env", "array", "Extra environment variable NAMES to pass through. The evidence record stores "
+                "names only, but nsys writes NAME=value of every passed variable (and the command line) into the "
+                ".nsys-rep, so secret-looking names (*TOKEN*, *KEY*, *SECRET*...) are refused.", default=[]),
+            Arg("trace", "string", "nsys -t value.", default="cuda,nvtx"),
+            Arg("capture_range", "string", "Limit capture to cudaProfilerStart/Stop or an NVTX range.",
+                default="none", choices=("none", "cudaProfilerApi", "nvtx")),
+            Arg("nvtx_capture", "string", "NVTX range name for capture_range=nvtx."),
+            Arg("delay", "number", "Seconds to wait before collecting."),
+            Arg("duration", "number", "Seconds to collect."),
+            Arg("timeout", "integer", "Seconds before the run (and every process it started) is killed.",
+                default=3600)],
+           passthrough=True)
+def capture_nsys(call, keep_env, trace, capture_range, nvtx_capture, delay, duration, timeout, argv):
+    nsys = require("nsys", "Install Nsight Systems (it ships with the CUDA toolkit).")
+    if capture_range == "nvtx" and not nvtx_capture:
+        raise OpError("capture_range=nvtx needs nvtx_capture=<range name>")
+    env, env_names = clean_env(keep_env)  # raises on secret-looking names, before anything runs
+    dropped = dropped_config_env(env)
+    store = ev.Store()
+    with store.staging_dir() as staging:  # removed again if anything below fails
+        base = staging / "profile"
+        cmd = [nsys, "profile", "-t", trace, "--cuda-graph-trace=node", "-s", "none", "--cpuctxsw=none",
+               "--force-overwrite", "true", "-o", str(base)]
+        if capture_range != "none":
+            cmd += ["--capture-range", capture_range]
+        if nvtx_capture:
+            cmd += ["--nvtx-capture", nvtx_capture]
+        if delay:
+            cmd += ["--delay", str(delay)]
+        if duration:
+            cmd += ["--duration", str(duration)]
+        r = run(cmd + list(argv), timeout=timeout, env=env, stdout_path=staging / "stdout.txt",
+                stderr_path=staging / "stderr.txt")
+        redact_outputs(staging)
+        rep = base.with_suffix(".nsys-rep")
+        if not rep.is_file():
+            raise OpError(f"nsys produced no report (exit {r.returncode}): {ev.redact_text(r.stderr[-800:])}")
+        sqlite = base.with_suffix(".sqlite")
+        export_sqlite(nsys, rep, sqlite)
+        scrubbed = nsys_sqlite.scrub_metadata(str(sqlite))
+        limitations, missing = [], []
+        if r.returncode != 0:
+            limitations.append(f"profiled command exited with code {r.returncode}: the trace may be partial")
+            missing.append("clean exit")
+        limitations.append(REP_IS_SENSITIVE.format(names=", ".join(env_names)))
+        if dropped:
+            limitations.append("set here but not passed to the profiled run, so the configuration may differ from a "
+                               "benchmark run with the full environment: " + ", ".join(dropped)
+                               + " (pass each one that matters with --keep-env NAME)")
+        summary = f"nsys trace ({rep.stat().st_size / 1e6:.1f} MB), exit {r.returncode}; run 'nsys analyze' on it"
+        record = ev.build(call, kind="nsys_capture", claim_class="observed", summary=summary,
+                          identity_=ev.identity(gpu=gpu_identity()),
+                          artifacts=[ev.artifact(rep, staging, "nsys_rep"), ev.artifact(sqlite, staging, "sqlite"),
+                                     ev.artifact(staging / "stderr.txt", staging, "stderr")],
+                          missing=missing, limitations=limitations,
+                          data={"nsys_version": tool_version("nsys"), "exit_code": r.returncode,
+                                "env_names": env_names, "env_dropped": dropped, "sqlite_scrubbed": scrubbed,
+                                "trace": trace, "cuda_graph_trace": "node", "capture_range": capture_range})
+        return store.save(record, staging)
+
+
+def _resolve_input(store: ev.Store, source: str, staging: Path) -> tuple[Path, bool, dict | None]:
+    """(sqlite path, whether it was exported into ``staging``, source record if an evidence id was given)."""
+    p = Path(source)
+    if p.suffix == ".sqlite" and p.is_file():
+        return p, False, None
+    if p.suffix == ".nsys-rep" and p.is_file():
+        out = staging / (p.stem + ".sqlite")
+        export_sqlite(require("nsys", "nsys is needed to export .nsys-rep files"), p, out)
+        nsys_sqlite.scrub_metadata(str(out))  # this copy is stored as an artifact
+        return out, True, None
+    rec = store.load(source)  # an evidence id of an nsys capture
+    if rec.get("kind") != "nsys_capture":
+        raise OpError(f"{source}: expected a .sqlite, a .nsys-rep or an nsys_capture evidence id")
+    art = next((a for a in rec["artifacts"] if a.get("role") == "sqlite"), None)
+    if art is None:
+        raise OpError(f"{source}: capture has no sqlite artifact")
+    return store.run_dir(rec["evidence_id"]) / art["path"], False, rec
+
+
+def timeline_metrics(result: dict) -> tuple[list[dict], list[str]]:
+    """Evidence metrics for an analysed trace, and any limitations they need.
+
+    Families are ranked by critical-path share of the decode step (``critical_path_pct``),
+    with GPU idle inside the step as its own row, so the rows add up to 100%. Summed
+    kernel durations are not used for ranking: when kernels overlap (programmatic
+    dependent launch, several streams) they over-count; ``kernel_sum_over_span`` > 1
+    says by how much.
+    """
+    steps = result["steps"]
+    metrics = [ev.metric("gpu_busy_pct", result["gpu_busy_pct"], "%", "window"),
+               ev.metric("gpu_idle_ms", result["gpu_idle_ms"], "ms", "window"),
+               ev.metric("graph_launches", result["graph_launch"]["launches"], "count", "total")]
+    limitations = []
+    host = result["graph_launch"]["host_us"]
+    if host.get("n"):
+        metrics.append(ev.metric("graph_launch_host_us", host["p50"], "us", "median", n=host["n"]))
+    # step_gap_us is time with *nothing* running on the GPU between two steps (not just the
+    # gap between the steps' kernels, which can contain other GPU work).
+    for key, name in (("span_us", "step_span_us"), ("idle_in_step_us", "step_idle_us"),
+                      ("gap_fully_idle_us", "step_gap_us")):
+        d = steps.get(key) or {}
+        if d.get("n"):
+            metrics.append(ev.metric(name, d["p50"], "us", "median", n=d["n"]))
+    span = (steps.get("span_us") or {}).get("p50")
+    kernel_sum = (steps.get("kernel_sum_us") or {}).get("p50")
+    families = steps.get("families") or {}
+    n = steps.get("steady_steps")
+    if span and kernel_sum:
+        ratio = kernel_sum / span
+        metrics.append(ev.metric("kernel_sum_over_span", round(ratio, 3), "ratio", "median", n=n))
+        if ratio > 1.05:
+            limitations.append(f"kernels overlap: summed kernel time is {ratio:.2f}x the step span, so shares of "
+                               "summed duration over-count; rank by critical_path_pct")
+    if families:
+        metrics.append(ev.metric("critical_path_pct", steps.get("idle_pct_of_step"), "%", "mean_step_share", n=n,
+                                 subject="GPU idle inside the step"))
+        for f, v in list(families.items())[:6]:  # already sorted by critical-path time
+            metrics.append(ev.metric("critical_path_pct", v["critical_pct_of_step"], "%", "mean_step_share", n=n,
+                                     subject=f))
+    else:  # no decode steps found: only whole-window shares of summed kernel time are available
+        for f, v in list(result["families"].items())[:6]:
+            metrics.append(ev.metric("kernel_time_pct", v["pct_kernel_time"], "%", "summed_duration_share",
+                                     subject=f))
+        limitations.append("no decode steps found: family shares are of summed kernel time over the whole window, "
+                           "which over-counts overlapping kernels")
+    return metrics, limitations
+
+
+@operation("analyze_nsys", ("nsys", "analyze"), "Per-step GPU busy/idle, idle-gap histogram, kernel families "
+           "ranked by critical-path share of the step, graph launches and top kernels from an nsys trace.",
+           [Arg("source", "string", "A .nsys-rep, a .sqlite export, or an nsys_capture evidence id.", required=True,
+                positional=True),
+            Arg("step_regex", "string", "NVTX range names that mark one decode step.",
+                default=nsys_sqlite.DEFAULT_STEP_REGEX),
+            Arg("gap_us", "number", "Fallback segmentation: a host gap longer than this starts a new step.",
+                default=20.0),
+            Arg("top", "integer", "How many top kernels to keep.", default=25)])
+def analyze_nsys(call, source, step_regex, gap_us, top):
+    store = ev.Store()
+    with store.staging_dir() as staging:
+        sqlite, exported, src_rec = _resolve_input(store, source, staging)
+        result = nsys_sqlite.analyze(str(sqlite), step_regex=step_regex, gap_us=gap_us, top=top)
+        if "error" in result:
+            raise OpError(result["error"])
+        steps = result["steps"]
+        metrics, limitations = timeline_metrics(result)
+        limitations = list(result["notes"]) + limitations
+        limitations.append("nsys adds a small per-launch overhead; absolute step times run slightly long under the "
+                           "profiler (compare to bench numbers, not across tools)")
+        gpu = {"name": result.get("gpu_name")}
+        if src_rec:
+            gpu = src_rec["identity"]["gpu"]
+            problem = ev.integrity_problem(src_rec)
+            if problem:
+                limitations.append(f"source capture {problem}")
+        span = steps.get("span_us", {})
+        summary = (f"GPU busy {result['gpu_busy_pct']}% of {result['window_ms']:.1f} ms; "
+                   f"{steps.get('steps', 0)} steps ({steps.get('segmentation')})")
+        if span.get("n"):
+            summary += f", step span p50 {span['p50']} us"
+        top_family = next(iter(steps.get("families") or {}), None)
+        if top_family:
+            summary += (f"; top family {top_family} {steps['families'][top_family]['critical_pct_of_step']}% "
+                        f"of the step's critical path")
+        artifacts = [ev.artifact(sqlite, staging if exported else None, "sqlite" if exported else "input")]
+        record = ev.build(call, kind="nsys_timeline", claim_class="observed", summary=summary,
+                          identity_=ev.identity(gpu=gpu), metrics=metrics, artifacts=artifacts,
+                          limitations=limitations,
+                          data={**result, "source_evidence": src_rec["evidence_id"] if src_rec else None})
+        return store.save(record, staging)

inferencekit/ops/prompts.py

@@ -0,0 +1,316 @@
+"""prompts build / seal / verify / show: the versioned prompt sets every engine is run on.
+
+``build`` writes the shipped dev sets (``dev``: 16 prompts per bucket; ``dev-load``:
+1,200+ short/medium prompts for load tests) and ``exclusions.json``. ``seal`` writes the
+sealed sets, built by the same code from disjoint source groups, to a directory *outside*
+any git work tree, plus a ``SHA256SUMS`` file whose hashes go into the pre-registration.
+``verify`` re-hashes a set against its manifest (and optionally an expected sha256) and
+checks every item against the format and bucket rules; with ``--secret-file`` it also
+checks the revealed secret against the manifest's ``secret_sha256`` (``sha256sum`` of the
+file) and rebuilds the sealed set from the sources to compare hashes. ``show`` prints
+per-bucket length statistics.
+
+Numbers printed here are computed from the files (claim class ``derived``).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import statistics
+from collections import Counter
+from pathlib import Path
+
+from inferencekit import prompts as ps
+from inferencekit.registry import Arg, OpError, operation
+
+_DEFAULT_CACHE = os.path.join(os.path.expanduser("~"), ".cache", "inferencekit", "prompt-sources")
+
+
+def _git_root(path: Path) -> Path | None:
+    p = path.resolve()
+    for d in (p, *p.parents):
+        if (d / ".git").exists():
+            return d
+    return None
+
+
+def _cache(sources: str | None, fetch: bool) -> Path:
+    from inferencekit.prompts import build
+
+    cache = Path(sources or _DEFAULT_CACHE).expanduser()
+    if fetch:
+        build.fetch(cache, log=lambda *_: None)
+    return cache
+
+
+def _exclusions(path: str | None) -> dict:
+    from inferencekit.prompts import build
+
+    return build.read_exclusions(Path(path).expanduser() if path else ps.data_dir() / build.EXCLUSIONS_FILE)
+
+
+_EXCL_ARG = Arg("exclusions", "string", "Exclusions file to apply (default: the package's exclusions.json).")
+_BLOCK_ARG = Arg("blocklist", "string", "Private scrub-patterns file whose matches are excluded and recorded in "
+                 "exclusions.json (default: $INFERENCEKIT_PROMPT_BLOCKLIST if set, else none).")
+
+
+def _rules(blocklist: str | None) -> list:
+    from inferencekit.prompts import build
+
+    try:
+        return build.load_rules(build.blocklist_path(blocklist))
+    except FileNotFoundError as e:
+        raise OpError(str(e)) from None
+
+
+@operation("prompts_build", ("prompts", "build"), "Build the shipped dev prompt sets (dev, dev-load) "
+           "deterministically from the downloaded open-licence sources.",
+           [Arg("sources", "string", f"Source cache directory (default {_DEFAULT_CACHE.replace(os.path.expanduser('~'), '~')})."),
+            Arg("out", "string", "Output directory (default: the package's prompts directory). The package's "
+                "prompts directory gets .jsonl.gz like the shipped sets; any other directory plain .jsonl."),
+            Arg("fetch", "boolean", "Download missing source files first.", default=False),
+            Arg("seed", "string", "Split/rank seed (changing it changes every set).", default="inferencekit-prompts-v1"),
+            _EXCL_ARG, _BLOCK_ARG])
+def prompts_build(call, sources, out, fetch, seed, exclusions, blocklist):
+    from inferencekit.prompts import build
+
+    cache = _cache(sources, fetch)
+    out_dir = Path(out).expanduser() if out else ps.data_dir()
+    # The shipped sets stay gzipped however the package directory is named (scripts/build_prompts.py
+    # passes it explicitly); a rebuild elsewhere writes plain JSONL. The manifests hash the JSONL either way.
+    gz = out_dir.resolve() == ps.data_dir().resolve()
+    rules = _rules(blocklist)
+    try:
+        mans, applied = build.build_split(cache, out_dir, "dev", seed, exclusions=_exclusions(exclusions),
+                                          rules=rules, gz=gz)
+        build.write_exclusions(out_dir / build.EXCLUSIONS_FILE, applied, build.source_names(cache))
+    except (FileNotFoundError, ValueError) as e:
+        raise OpError(str(e)) from None
+    return {"claim_class": "derived", "sets": [_brief(m) for m in mans],
+            "excluded_rows": sum(len(v) for v in applied.values()), "private_rules": len(rules)}
+
+
+@operation("prompts_seal", ("prompts", "seal"), "Build the sealed prompt sets (disjoint from dev) into a "
+           "directory outside any git work tree and write SHA256SUMS for the pre-registration.",
+           [Arg("out", "string", "Output directory, outside every git work tree.", required=True),
+            Arg("sources", "string", "Source cache directory."),
+            Arg("fetch", "boolean", "Download missing source files first.", default=False),
+            Arg("seed", "string", "Split/rank seed (must match the dev build).", default="inferencekit-prompts-v1"),
+            Arg("secret_file", "string", "File holding the selection secret (default <out>/SEED.secret, created "
+                "with 32 random bytes if missing). Keep it private until the sealed run is over."),
+            Arg("public", "boolean", "Rank by the public seed only (anyone can rebuild the set; tests and demos).",
+                default=False),
+            _EXCL_ARG, _BLOCK_ARG])
+def prompts_seal(call, out, sources, fetch, seed, secret_file, public, exclusions, blocklist):
+    import secrets as _secrets
+
+    from inferencekit.prompts import build
+
+    out_dir = Path(out).expanduser()
+    root = _git_root(out_dir)
+    if root is not None:
+        raise OpError(f"refusing to write sealed prompts inside the git work tree {root}: "
+                      "a sealed set must never be committable")
+    secret = secret_sha = None
+    if not public:
+        sf = Path(secret_file).expanduser() if secret_file else out_dir / "SEED.secret"
+        if _git_root(sf) is not None:
+            raise OpError(f"refusing to keep the sealing secret inside a git work tree ({sf})")
+        if not sf.is_file():
+            sf.parent.mkdir(parents=True, exist_ok=True)
+            sf.write_text(_secrets.token_hex(32) + "\n", encoding="utf-8")
+            sf.chmod(0o600)
+        secret, secret_sha = _read_secret(sf)
+    cache = _cache(sources, fetch)
+    listed = _exclusions(exclusions)
+    rules = _rules(blocklist)
+    try:
+        if rules:  # a public rebuild must exclude the same rows: they have to be in the exclusions file
+            _, by_rules = build.candidates(cache, seed, None, rules)
+            extra = sum(len(refs - listed.get(sha, set())) for sha, refs in by_rules.items())
+            if extra:
+                raise OpError(f"the private rules exclude {extra} row(s) the exclusions file does not list, so the "
+                              "sealed set could not be rebuilt without them: run `prompts build` first (it rewrites "
+                              "exclusions.json) or pass --exclusions")
+        mans, _ = build.build_split(cache, out_dir, "sealed", seed, secret, secret_sha, exclusions=listed,
+                                    rules=rules)
+    except (FileNotFoundError, ValueError) as e:
+        raise OpError(str(e)) from None
+    sums = build.write_sums(out_dir, mans)
+    return {"claim_class": "derived", "sets": [_brief(m) for m in mans], "sha256sums": str(sums),
+            "secret_sha256": secret_sha, "secret_rule": build.SECRET_RULE if secret else None}
+
+
+def _read_secret(sf: Path) -> tuple[str, str]:
+    """(secret text used as the ranking salt, sha256 of the file bytes: the published commitment)."""
+    import hashlib
+
+    raw = sf.read_bytes()
+    secret = raw.decode("utf-8").strip()
+    if len(secret) < 16:
+        raise OpError(f"{sf}: the secret is too short (use at least 16 characters)")
+    return secret, hashlib.sha256(raw).hexdigest()
+
+
+def _brief(m: dict) -> dict:
+    return {k: m[k] for k in ("name", "n", "buckets", "sha256", "sealed")}
+
+
+def _sums_lookup(path: Path) -> dict[str, str]:
+    out = {}
+    for line in path.read_text(encoding="utf-8").splitlines():
+        parts = line.split()
+        if len(parts) == 2:
+            out[parts[1].lstrip("*")] = parts[0]
+    return out
+
+
+def check_items(items: list[dict]) -> list[str]:
+    """Format and bucket-rule problems (empty = fine)."""
+    problems, ids, texts = [], Counter(), Counter()
+    for n, it in enumerate(items, 1):
+        for key in ("id", "bucket", "messages", "max_tokens", "source", "kind"):
+            if key not in it:
+                problems.append(f"line {n}: missing '{key}'")
+        if it.get("bucket") not in ps.BUCKETS:
+            problems.append(f"line {n}: unknown bucket {it.get('bucket')!r}")
+            continue
+        msgs = it.get("messages")
+        if not isinstance(msgs, list) or not msgs or not all(isinstance(m, dict) and m.get("role") and
+                                                             isinstance(m.get("content"), str) for m in msgs):
+            problems.append(f"line {n}: messages must be a non-empty OpenAI chat list")
+            continue
+        if not isinstance(it.get("max_tokens"), int) or it["max_tokens"] < 1:
+            problems.append(f"line {n}: max_tokens must be a positive integer")
+        text = ps.prompt_text(it)
+        if not ps.in_bucket(text, it["bucket"]):
+            problems.append(f"line {n} ({it.get('id')}): {ps.words(text)} words / {ps.proxy_tokens(text)} proxy "
+                            f"tokens is outside the {it['bucket']} rule {ps.BUCKET_RULES[it['bucket']]}")
+        ids[it.get("id")] += 1
+        texts[text] += 1
+    problems += [f"duplicate id {i!r} ({c}x)" for i, c in ids.items() if c > 1]
+    dup_texts = sum(1 for c in texts.values() if c > 1)
+    if dup_texts:
+        problems.append(f"{dup_texts} prompt text(s) appear more than once")
+    return problems
+
+
+@operation("prompts_verify", ("prompts", "verify"), "Check a prompt set: sha256 against its manifest (and an "
+           "expected hash or SHA256SUMS), the item format, bucket rules, unique ids and prompts.",
+           [Arg("set", "string", "'dev', 'dev-load', or a path to a .jsonl / .jsonl.gz prompt set.", default="dev"),
+            Arg("expected_sha256", "string", "Pre-registered sha256 the JSONL bytes must have."),
+            Arg("sums", "string", "A SHA256SUMS file listing the set."),
+            Arg("secret_file", "string", "Sealed sets: the revealed SEED.secret. Checks `sha256sum` of the file "
+                "against the manifest's secret_sha256 and, when the sources are present, rebuilds the set."),
+            Arg("sources", "string", "Source cache directory for the rebuild (default "
+                f"{_DEFAULT_CACHE.replace(os.path.expanduser('~'), '~')})."),
+            Arg("fetch", "boolean", "Download missing source files for the rebuild.", default=False),
+            _EXCL_ARG])
+def prompts_verify(call, set, expected_sha256, sums, secret_file, sources, fetch, exclusions):  # noqa: A002
+    path = ps.resolve(set)
+    if not path.is_file():
+        raise OpError(f"no prompt set at {path}")
+    try:
+        actual = ps.sha256_file(path)
+    except (OSError, ValueError) as e:  # e.g. a truncated or corrupt .jsonl.gz
+        raise OpError(f"{path}: cannot read ({e})") from None
+    checks = []
+
+    def check(name, ok, detail):
+        checks.append({"check": name, "ok": bool(ok), "detail": detail})
+
+    man_path = ps.manifest_path(path)
+    man = None
+    if man_path.is_file():
+        man = json.loads(man_path.read_text(encoding="utf-8"))
+        check("manifest_sha256", man.get("sha256") == actual, f"manifest {man.get('sha256')} vs file {actual}")
+    else:
+        check("manifest_present", False, f"{man_path.name} not found")
+    if expected_sha256:
+        check("expected_sha256", expected_sha256.strip().lower() == actual, f"expected {expected_sha256}")
+    if sums:
+        table = _sums_lookup(Path(sums))
+        key = path.name if path.name in table else ps.set_stem(path) + ".jsonl"
+        listed = table.get(key)
+        check("sha256sums", listed == actual, f"SHA256SUMS lists {listed} for {key}")
+    try:
+        items = [json.loads(line) for line in ps.read_text(path).splitlines() if line.strip()]
+    except (ValueError, OSError) as e:
+        raise OpError(f"{path}: not JSONL ({e})") from None
+    problems = check_items(items)
+    check("items", not problems, problems[:20] or f"{len(items)} items fit the format and bucket rules")
+    if man is not None:
+        counts = {b: sum(1 for it in items if it.get("bucket") == b) for b in ps.BUCKETS}
+        counts = {b: n for b, n in counts.items() if n}
+        check("manifest_counts", man.get("n") == len(items) and man.get("buckets") == counts,
+              f"manifest n={man.get('n')} buckets={man.get('buckets')}; file n={len(items)} buckets={counts}")
+    if secret_file:
+        _verify_secret(man, Path(secret_file).expanduser(), actual, sources, fetch, exclusions, check)
+    ok = all(c["ok"] for c in checks)
+    return {"set": man.get("name") if man else ps.set_stem(path), "path": str(path), "sha256": actual,
+            "verdict": "PASS" if ok else "FAIL", "checks": checks}
+
+
+def _verify_secret(man: dict | None, sf: Path, actual: str, sources, fetch, exclusions, check) -> None:
+    from inferencekit.prompts import build
+
+    if not man or not man.get("secret_sha256"):
+        check("secret_sha256", False, "the manifest commits to no secret (not a secret-sealed set)")
+        return
+    if not sf.is_file():
+        raise OpError(f"no secret file at {sf}")
+    secret, got = _read_secret(sf)
+    check("secret_sha256", got == man["secret_sha256"],
+          f"sha256sum of {sf.name} = {got}; manifest secret_sha256 = {man['secret_sha256']}")
+    if got != man["secret_sha256"]:
+        return
+    cache = _cache(sources, fetch)
+    try:
+        rebuilt = build.rebuild_sha256(cache, man.get("seed") or build.SEED, secret, _exclusions(exclusions))
+    except FileNotFoundError as e:
+        check("rebuild", False, f"cannot rebuild: {e} (pass --sources DIR or --fetch)")
+        return
+    except ValueError as e:
+        check("rebuild", False, f"cannot rebuild: {e}")
+        return
+    want = rebuilt.get(man.get("role") or "single")
+    check("rebuild", want == actual, f"rebuilt from the sources with this secret: {want}; file: {actual}")
+
+
+@operation("prompts_show", ("prompts", "show"), "Per-bucket statistics of a prompt set (counts, word and "
+           "proxy-token lengths, sources).",
+           [Arg("set", "string", "'dev', 'dev-load', or a path to a .jsonl / .jsonl.gz prompt set.", default="dev"),
+            Arg("examples", "integer", "Show the first N prompts of each bucket (truncated).", default=0)])
+def prompts_show(call, set, examples):  # noqa: A002
+    path = ps.resolve(set)
+    if not path.is_file():
+        raise OpError(f"no prompt set at {path}")
+    try:
+        items = ps.load(str(path))
+    except (OSError, ValueError) as e:  # corrupt .jsonl.gz, not JSONL
+        raise OpError(f"{path}: cannot read ({e})") from None
+    try:
+        man = ps.load_manifest(str(path))
+    except (OSError, ValueError):
+        man = {}
+    buckets = {}
+    for b in ps.BUCKETS:
+        sel = [it for it in items if it.get("bucket") == b]
+        if not sel:
+            continue
+        w = [ps.words(ps.prompt_text(it)) for it in sel]
+        t = [ps.proxy_tokens(ps.prompt_text(it)) for it in sel]
+        row = {"n": len(sel), "words": {"min": min(w), "median": statistics.median(w), "max": max(w)},
+               "proxy_tokens": {"min": min(t), "median": statistics.median(t), "max": max(t)},
+               "max_tokens": sorted({it.get("max_tokens") for it in sel}),
+               "sources": dict(Counter(it.get("source", "?").split("/")[0].split("-")[0]
+                                       if it.get("source", "").startswith("gutenberg") else it.get("source", "?")
+                                       for it in sel))}
+        if examples:
+            row["examples"] = [{"id": it["id"], "text": ps.prompt_text(it)[:200]} for it in sel[:examples]]
+        buckets[b] = row
+    return {"set": man.get("name") or ps.set_stem(path), "sha256": ps.sha256_file(path), "sealed": man.get("sealed"),
+            "n": len(items), "claim_class": "derived", "length_proxy": man.get("length_proxy",
+                                                                                "proxy_tokens = words * 1.3"),
+            "buckets": buckets}