inferencekit-cli 0.1.0__py3-none-any.whl

inferencekit/stats.py

@@ -0,0 +1,249 @@
+"""Small, dependency-free statistics for benchmark rounds.
+
+Benchmarks here produce few numbers (one per round, typically 5-30), so plain Python
+is fast enough and keeps the logic easy to read. Randomness comes from
+``random.Random(seed)`` and only through ``rng.random()``, whose sequence Python
+guarantees to be reproducible for a given seed; so a CI computed today and next
+year from the same data is identical.
+
+Contents
+  quantile / nearest_rank   two percentile conventions (interpolated / an actual sample)
+  percentile_ci             95% percentile interval from bootstrap replicates
+  bootstrap_ci              CI of a percentile, resampling whole *units* (clusters)
+  paired_ratio_ci           CI of median(candidate) / median(baseline), resampling pairs
+  unpaired_ratio_ci         same without pairing, for runs that cannot be paired
+  rounds_for_power          how many rounds would make the CI narrow enough for decide()
+  decide                    verdict for a comparison (improved / regressed / ...)
+  combine_claims            PASS / FAIL / UNPROVEN / NOT RUN across several checks
+"""
+
+from __future__ import annotations
+
+import math
+import random
+import statistics
+from typing import Hashable, Iterable, Sequence
+
+Z95 = 1.959964
+DEFAULT_RESAMPLES = 10_000
+# A CI counts as "powered" (narrow enough to support a "no change" call) when its
+# half-width is at most this fraction of the practical threshold.
+POWER_RATIO = 0.75
+# rounds_for_power multiplies its 1/sqrt(n) estimate by this, so that most reruns (not
+# half of them) come out powered. Calibrated by simulation through paired_ratio_ci +
+# decide: 85-100% of reruns powered at most noise/drift mixes (tests/test_stats.py).
+POWER_MARGIN = 2.0
+
+
+# --------------------------------------------------------------------------- percentiles
+
+def quantile(xs: Sequence[float], p: float) -> float | None:
+    """Linear-interpolated percentile (p in 0..100), the numpy default. For summaries."""
+    if not xs:
+        return None
+    v = sorted(xs)
+    k = (len(v) - 1) * p / 100.0
+    lo = int(math.floor(k))
+    hi = min(lo + 1, len(v) - 1)
+    return v[lo] + (v[hi] - v[lo]) * (k - lo)
+
+
+def nearest_rank(sorted_xs: Sequence[float], p: float) -> float | None:
+    """Nearest-rank percentile: the smallest sample with at least p% of samples <= it.
+
+    Unlike interpolation, the answer is always a value that was actually observed,
+    which is what you want inside a bootstrap (no invented in-between values).
+    """
+    n = len(sorted_xs)
+    if n == 0:
+        return None
+    rank = max(1, math.ceil(p / 100.0 * n))
+    return sorted_xs[min(rank, n) - 1]
+
+
+def summarize(xs: Iterable[float]) -> dict:
+    """n / median / p95 / min / max / mean of a list (None-free)."""
+    v = [float(x) for x in xs if x is not None]
+    if not v:
+        return {"n": 0}
+    return {"n": len(v), "median": statistics.median(v), "p95": quantile(v, 95), "min": min(v), "max": max(v),
+            "mean": sum(v) / len(v)}
+
+
+def percentile_ci(replicates: Sequence[float]) -> tuple[float | None, float | None]:
+    """95% percentile interval of bootstrap replicates.
+
+    With B sorted replicates the bounds are element floor(0.025 (B-1)) and
+    ceil(0.975 (B-1)): slightly conservative (never narrower than the exact quantiles).
+    NaN in any replicate means some resample had no defined value: the CI is undefined.
+    """
+    reps = [float(r) for r in replicates]
+    if not reps or any(math.isnan(r) for r in reps):
+        return None, None
+    reps.sort()
+    b = len(reps)
+    return reps[int(0.025 * (b - 1))], reps[int(math.ceil(0.975 * (b - 1)))]
+
+
+def _draw(rng: random.Random, k: int) -> int:
+    # rng.random() is the only stream Python guarantees reproducible across versions.
+    return min(int(rng.random() * k), k - 1)
+
+
+# --------------------------------------------------------------------------- bootstrap CIs
+
+def bootstrap_ci(values: Sequence[float], units: Sequence[Hashable] | None = None, ps: Sequence[float] = (50,),
+                 resamples: int = DEFAULT_RESAMPLES, seed: int = 0) -> dict[float, tuple[float | None, float | None]]:
+    """95% CI of nearest-rank percentiles ``ps`` of ``values``, by cluster bootstrap.
+
+    ``units[i]`` names the resampling unit of ``values[i]``. Values that share a unit
+    (e.g. all per-token latencies of one round) are correlated, so resampling them
+    one by one would understate the uncertainty. Instead each resample draws whole
+    units with replacement (as many draws as there are units) and keeps every value
+    of every drawn unit. With ``units=None`` each value is its own unit (ordinary
+    bootstrap). Returns ``{p: (lo, hi)}``; ``(None, None)`` with < 2 values or units.
+    """
+    if units is None:
+        units = list(range(len(values)))
+    by_unit: dict[Hashable, list[float]] = {}
+    for v, u in zip(values, units):
+        by_unit.setdefault(u, []).append(float(v))
+    groups = list(by_unit.values())
+    k = len(groups)
+    if len(values) < 2 or k < 2 or resamples <= 0:
+        return {p: (None, None) for p in ps}
+    rng = random.Random(seed)
+    reps: dict[float, list[float]] = {p: [] for p in ps}
+    for _ in range(resamples):
+        sample: list[float] = []
+        for _ in range(k):
+            sample.extend(groups[_draw(rng, k)])
+        sample.sort()
+        for p in ps:
+            reps[p].append(nearest_rank(sample, p))
+    return {p: percentile_ci(reps[p]) for p in ps}
+
+
+def paired_ratio_ci(baseline: Sequence[float], candidate: Sequence[float], resamples: int = DEFAULT_RESAMPLES,
+                    seed: int = 0) -> dict:
+    """Ratio of medians, candidate / baseline, with a paired bootstrap 95% CI.
+
+    ``baseline[i]`` and ``candidate[i]`` must come from the same round i (run back to
+    back). Resampling *pairs* keeps that link: anything that slowed round i for both
+    arms (a thermal dip, a noisy neighbour) moves numerator and denominator together
+    and largely cancels, which gives a narrower and more honest CI than resampling
+    the two arms independently.
+    """
+    n = min(len(baseline), len(candidate))
+    base, cand = [float(x) for x in baseline[:n]], [float(x) for x in candidate[:n]]
+    out = {"n_pairs": n, "ratio": None, "ci95": (None, None), "log_ratio_sd": None}
+    if n == 0 or statistics.median(base) == 0:
+        return out
+    out["ratio"] = statistics.median(cand) / statistics.median(base)
+    logs = [math.log(c / b) for b, c in zip(base, cand) if b > 0 and c > 0]
+    out["log_ratio_sd"] = statistics.stdev(logs) if len(logs) > 1 else (0.0 if logs else None)
+    if n < 2:
+        return out
+    rng = random.Random(seed)
+    reps = []
+    for _ in range(resamples):
+        idx = [_draw(rng, n) for _ in range(n)]
+        mb = statistics.median([base[i] for i in idx])
+        mc = statistics.median([cand[i] for i in idx])
+        reps.append(mc / mb if mb > 0 else float("nan"))
+    out["ci95"] = percentile_ci(reps)
+    return out
+
+
+def unpaired_ratio_ci(baseline: Sequence[float], candidate: Sequence[float], resamples: int = DEFAULT_RESAMPLES,
+                      seed: int = 0) -> dict:
+    """Ratio of medians with each arm resampled independently (no pairing available)."""
+    base, cand = [float(x) for x in baseline], [float(x) for x in candidate]
+    out = {"n_baseline": len(base), "n_candidate": len(cand), "ratio": None, "ci95": (None, None)}
+    if not base or not cand or statistics.median(base) == 0:
+        return out
+    out["ratio"] = statistics.median(cand) / statistics.median(base)
+    if len(base) < 2 or len(cand) < 2:
+        return out
+    rng = random.Random(seed)
+    reps = []
+    for _ in range(resamples):
+        mb = statistics.median([base[_draw(rng, len(base))] for _ in base])
+        mc = statistics.median([cand[_draw(rng, len(cand))] for _ in cand])
+        reps.append(mc / mb if mb > 0 else float("nan"))
+    out["ci95"] = percentile_ci(reps)
+    return out
+
+
+# --------------------------------------------------------------------------- power and verdicts
+
+def rounds_for_power(n: int, half_width: float | None, threshold: float, power_ratio: float = POWER_RATIO,
+                     margin: float = POWER_MARGIN) -> int | None:
+    """Rounds per arm that would make decide() call the CI powered, from a pilot comparison.
+
+    ``n`` rounds gave a CI with this ``half_width`` (decide()'s, on the ratio scale); the
+    target is ``power_ratio * threshold``. A CI's half-width shrinks like 1/sqrt(n), so
+    n * (half_width / target)^2 rounds would hit the target on average -- which means only
+    about half of the reruns would. ``margin`` (2x) buys a large majority.
+
+    Why not the textbook 1.96 * SD / sqrt(n) of per-pair log ratios: decide() tests a
+    bootstrap CI of a *ratio of medians*. A median is noisier than a mean (about 1.25x the
+    standard error for normal data), and slow drift cancels only partly when each arm's
+    median is taken on its own. Scaling the half-width of the very CI that decide() uses
+    includes both effects. (Simulated with 2-3% per-round noise: n from the textbook
+    formula came out powered in only 3-15% of reruns.) With fewer than about 5 pilot rounds
+    the estimate is rough.
+    """
+    if half_width is None or n < 1 or not threshold:
+        return None
+    target = power_ratio * abs(threshold)
+    return max(2, int(math.ceil(margin * n * (half_width / target) ** 2 - 1e-9)))
+
+
+def decide(ratio: float | None, lo: float | None, hi: float | None, threshold: float, higher_is_better: bool,
+           power_ratio: float = POWER_RATIO) -> dict:
+    """Turn a ratio CI into ``improved | regressed | within_noise | inconclusive``.
+
+    Work with g = the ratio oriented so that g > 1 is better. The checks run in this
+    order and the first that applies wins:
+      1. no ratio or no CI                                    -> inconclusive (no data)
+      2. CI entirely below 1 and g <= 1 - threshold           -> regressed
+      3. CI entirely above 1 and g >= 1 + threshold           -> improved
+      4. CI inside [1 - threshold, 1 + threshold] and powered -> within_noise
+      5. anything else                                        -> inconclusive
+    "Powered" means the CI half-width is at most power_ratio x threshold: a CI that
+    happens to sit inside the band but is nearly as wide as it is not evidence of
+    "no change"; more rounds are needed.
+    """
+    if ratio is None or lo is None or hi is None or ratio <= 0 or lo <= 0:
+        return {"verdict": "inconclusive", "reason": "no ratio or no confidence interval (need >= 2 rounds per arm)"}
+    if higher_is_better:
+        g, g_lo, g_hi = ratio, lo, hi
+    else:  # invert so that "faster" (smaller latency) is still g > 1
+        g, g_lo, g_hi = 1 / ratio, 1 / hi, 1 / lo
+    half_width = max(g - g_lo, g_hi - g)
+    powered = half_width <= power_ratio * threshold * (1 + 1e-9)
+    base = {"oriented_ratio": g, "oriented_ci95": [g_lo, g_hi], "half_width": half_width, "powered": powered}
+    if g_hi < 1 and g <= 1 - threshold:
+        return {**base, "verdict": "regressed", "reason": f"CI below 1 and change beyond -{threshold:.1%}"}
+    if g_lo > 1 and g >= 1 + threshold:
+        return {**base, "verdict": "improved", "reason": f"CI above 1 and change beyond +{threshold:.1%}"}
+    if g_lo >= 1 - threshold and g_hi <= 1 + threshold and powered:
+        return {**base, "verdict": "within_noise", "reason": f"CI inside +/-{threshold:.1%} and narrow enough"}
+    why = "CI too wide to decide" if not powered else "CI crosses the practical threshold"
+    return {**base, "verdict": "inconclusive", "reason": why}
+
+
+def combine_claims(verdicts: Iterable[str]) -> str:
+    """Combine claim verdicts: any FAIL -> FAIL; else any UNPROVEN -> UNPROVEN; else PASS.
+
+    "NOT RUN" entries are ignored unless nothing else ran. Missing data is never a pass.
+    """
+    vs = [v for v in verdicts if v != "NOT RUN"]
+    if not vs:
+        return "NOT RUN"
+    if "FAIL" in vs:
+        return "FAIL"
+    if "UNPROVEN" in vs:
+        return "UNPROVEN"
+    return "PASS"

inferencekit_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,233 @@
+Metadata-Version: 2.5
+Name: inferencekit-cli
+Version: 0.1.0
+Summary: Make your model serve faster on your GPU, and prove it. Skills for Claude Code, Codex and OpenCode, plus a CLI and MCP server: your agent picks and tunes inference engines, can write GPU kernels, and backs every number with client-timed benchmarks, anti-cheat gates and quality checks.
+Project-URL: Homepage, https://github.com/kittenml/inferencekit
+Project-URL: Repository, https://github.com/kittenml/inferencekit
+Project-URL: Issues, https://github.com/kittenml/inferencekit/issues
+Author: Stellon Labs
+License-Expression: Apache-2.0
+License-File: LICENSE
+License-File: NOTICE
+Keywords: benchmarking,capacity,correctness,cuda,gpu,inference,llama.cpp,llm,mcp,nsight-compute,nsight-systems,nvidia,profiling,quantization,roofline,serving,sglang,tensorrt-llm,vllm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: GPU :: NVIDIA CUDA
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: System :: Benchmark
+Requires-Python: >=3.10
+Requires-Dist: jsonschema>=4.20
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: mcp
+Requires-Dist: mcp<3,>=2.3; extra == 'mcp'
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+# inferencekit
+
+**Turn your coding agent into an inference engineer.**
+
+inferencekit made Gemma 4 run 1.33× faster than tuned vLLM. Your model is next.
+
+[![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
+[![PyPI: inferencekit-cli](https://img.shields.io/pypi/v/inferencekit-cli.svg?label=inferencekit-cli)](https://pypi.org/project/inferencekit-cli/)
+[![Claude Code](https://img.shields.io/badge/Claude%20Code-plugin-d97757.svg)](#install)
+[![Codex](https://img.shields.io/badge/Codex-plugin-111111.svg)](#install)
+[![OpenCode](https://img.shields.io/badge/OpenCode-plugin-4a7bd0.svg)](#install)
+
+[Install](#install) · [How it works](#how-it-works) · [Results](#results) · [Features](#features) · [Engines](https://github.com/kittenml/inferencekit-engines) · [Roadmap](#roadmap) · [Contributing](CONTRIBUTING.md)
+
+![Bar chart of speedups against engine baselines on the same hardware, including tuned vLLM and MLX-Audio: Gemma 4 26B-A4B on RTX 5090, 1.33× faster than tuned vLLM; LTX-2.5 on RTX PRO 6000, 1.25× faster than ComfyUI; Qwen3-8B on RTX 5090, 1.25× faster than default vLLM; Drama3 on Apple M4, 1.11× faster than tuned MLX-Audio; MOSS-Image-SFT on RX 9070 XT, 2.46× with quality secondary (FP8, 24 vs 50 steps; paired 95% CI 2.453–2.465×, lower bound 2.453×)](assets/hero-speedups.png)
+
+</div>
+
+inferencekit is a plugin for Claude Code, Codex and OpenCode, plus a command-line tool (CLI). Give your agent a model, hardware and a goal. It finds, benchmarks and tunes supported inference engines for your workload. You can stop with the best tested configuration or go further with custom optimizations, including GPU kernels.
+
+## How it works
+
+inferencekit gives your agent the playbook and the tools of a senior inference engineer. On every run it works the problem the same way.
+
+1. **Understand the task.** The agent first asks about your goal, deadline, budget and how much the results may change, reusing answers you already gave. It identifies the model, hardware and real inputs, then estimates memory, compute and launch costs per request. Text, image, audio and video use their own metrics and quality checks.
+2. **Find and tune an engine.** It researches current implementations, then tunes and benchmarks supported engines (vLLM, SGLang, llama.cpp, TensorRT-LLM and others) for your workload on your hardware. The fastest tested configuration that meets your quality requirements can be the result, or the tuned baseline for further work.
+3. **Find the bottleneck.** It selects the profiler for your hardware: Nsight on NVIDIA, ROCm tools on AMD, or Instruments and Metal on Apple. It measures the critical path and investigates memory, compute and launch costs, checking current vendor docs when support is uncertain.
+4. **Investigate and implement, if requested.** Search current code, papers, compiler docs and engineering talks; inspect dispatch, IR and assembly when useful. Choose graph execution, fusion, matrix kernels, packing, state reuse or other techniques from measured bottlenecks. Independent implementations can run in parallel; integration and shared-device measurements stay isolated.
+5. **Prove every step.** Every change gets a fast correctness check and a timed screen; milestones get the full proof: interleaved rounds against the tuned baseline, the anti-cheat gates and quality against a reference model. Changes that miss the agreed performance or quality requirements are reverted.
+6. **Ship it.** You get the fastest proven configuration, its launch command, and a report with the commands that reproduce every number.
+
+## When it asks you
+
+The agent works on its own and stops only when it needs you:
+
+- **Missing details.** If you didn't say which model, GPU or workload, it asks before it starts.
+- **Your preferences.** Choose exactly the same results, small differences with the same quality, or speed with an agreed quality tradeoff. Set your goal, spending, deadline and check-ins; it reuses answers you already gave.
+- **Model changes.** Permission to simplify the model is separate from permission to use stronger compression. Plans are optional: ask to see one if you want it.
+- **The budget cap.** When the run reaches the cap you set, it stops and asks.
+- **Things only you can give.** For example, permission to read the GPU's profiler counters, or a Hugging Face token for a gated model.
+
+Everything else runs without asking. For a hands-off run, approve a budget with a cap up front (`non_interactive: true` and `budget.approved: true` in the intake file); the agent then writes its questions and the choices it made to `QUESTIONS.md` instead of waiting for you.
+
+## Results
+
+**Pushing speed on AMD:** with quality secondary, MOSS-Image-SFT reached **2.46×** faster image generation than the tested tuned vLLM-Omni baseline on a 16 GB RX 9070 XT.‡
+
+Single-user decode for the language models; one clip at a time for video; single-user speech generation for Drama3.
+
+| Engine | GPU | Speedup | Agent | Agent time | Tokens | Cost |
+|---|---|---|---|---|---|---|
+| [Gemma 4 26B-A4B (NVFP4)](https://github.com/kittenml/inferencekit-engines/tree/main/gemma-4-26b-a4b-nvfp4_rtx-5090) | RTX 5090 | **1.33×** vs tuned vLLM | Claude Opus 5.5 (high) | 19 h | 630M | $199 |
+| [LTX-2.5 text-to-video](https://github.com/kittenml/inferencekit-engines/tree/main/ltx-2.5_rtx-pro-6000) | RTX PRO 6000 | **1.25×** vs ComfyUI | Claude Opus 5.5 (high) | 3.6 h | 47M | $18 |
+| [Qwen3-8B (FP8)](https://github.com/kittenml/inferencekit-engines/tree/main/qwen3-8b-fp8_rtx-5090)* | RTX 5090 | **1.25×** vs vLLM | Claude Opus 5.5 (xhigh) | about 10 h | about 240M | about $125 |
+| [Drama3 speech](https://github.com/haaziq-sl/inferencekit-engines/tree/main/drama-3_apple-m4-16gb/profiles/main5-tower5)† | Apple M4 (16 GB) | **1.112×** vs tuned MLX-Audio | GPT-6.1 Sol (high) | ~70 h elapsed | ~2.07B | ~$475 standard-rate equivalent |
+| [MOSS-Image-SFT (FP8)](https://github.com/haaziq-sl/inferencekit-engines/tree/main/moss-image-sft_rx-9070-xt-fp8)‡ | RX 9070 XT (16 GB) | **2.46×** vs tuned vLLM-Omni | GPT-6.1 Terra (high) | ~19.6 h elapsed | ~1.80B | ~$206 standard-rate equivalent |
+
+<sub>* Against vLLM with its default settings; neither side uses speculative decoding. † Drama3 used identical quantized weights and matched PCM audio, codes, chunks and completion in 144 fresh output pairs. Four paired timing rounds gave a 95% CI of 1.059×–1.156×. [Comparison details](https://github.com/haaziq-sl/inferencekit-engines/tree/main/drama-3_apple-m4-16gb/profiles/main5-tower5).</sub>
+
+<sub>‡ Lossy speed-first experiment: FP8/24 steps vs BF16/50 steps; paired 95% CI **2.453–2.465×**, conservative lower bound **2.453×**. [Outputs, timing and accounting](https://github.com/haaziq-sl/inferencekit-engines/tree/main/moss-image-sft_rx-9070-xt-fp8/).</sub>
+
+Time and tokens include setup, benchmarking, optimization, validation and reporting. Costs are API list-price estimates; most tokens are cached conversation reads. Drama3’s estimates cover direct agent work ([accounting](https://github.com/haaziq-sl/inferencekit-engines/blob/main/drama-3_apple-m4-16gb/ACCOUNTING_V65.json)). How each engine was built and measured: [inferencekit-engines](https://github.com/kittenml/inferencekit-engines).
+
+## Speed you can trust
+
+Fast is easy. Fast and still correct is the hard part, and it's the part agents skip.
+
+Agents are good at finding speed. They're also good at finding fake speed: a cached answer, a shorter reply, a different video. inferencekit keeps only the speed that's real.
+
+Without these checks, one agent thought its video engine was 2% faster, but it was rendering a completely different video. Another was sure it had a 1.12× speedup; measured again on a clean machine, its engine was 5% slower than where it started.
+
+A speedup counts only when both engines pass every check that applies.
+
+| A way to game a benchmark | What catches it |
+|---|---|
+| Answer repeated prompts from a cache | `caching`: a replayed prompt may not be over 10% faster; a prompt changed by one character may not get the old answer |
+| Report more tokens than were made | `token_count`: the reported count must match the streamed tokens and the re-tokenized text |
+| Stop early or cut answers short | `output_length`: with end-of-text ignored, every answer must be exactly the requested length |
+| Drop rare tokens from the vocabulary | `rare_tokens`: random CJK text, emoji, long numbers and code names must come back exactly |
+| Ignore sampling settings or fake probabilities | `hidden_probes`: temperature must change answers; probabilities must be valid |
+| Use another chat template so prompts get shorter | Render check: every prompt must become the same number of tokens as on the baseline |
+| Speculative decoding that changes answers | `spec_exact`: greedy answers must match the same engine with speculation off |
+| Lower quality for speed | Quality vs a reference, GSM8K, per-step video check |
+| Numbers that only hold on the agent's machine | Every run writes `manifest.json` with each engine's install and launch commands, so anyone can re-run it on a clean machine, on prompts sealed with `inferencekit prompts seal` that the agent never saw |
+
+Each check ends PASS, FAIL, UNPROVEN (not enough evidence) or NOT RUN. A missing check is never a pass.
+
+## Features
+
+- **9 skills, 4 commands, 5 read-only subagents and an MCP server**, the same in every supported agent.
+- **Speed-of-light ceilings:** measured memory bandwidth (`ceiling bw`) and exact bytes per token read from the checkpoint's safetensors headers (`roofline`), including NVFP4, FP8 and mixture-of-experts models.
+- **Engine triage and tuning:** vLLM, SGLang, llama.cpp, TensorRT-LLM, ComfyUI, Diffusers and others. Only settings that the engine's own logs show in effect are counted.
+- **Kernel work:** profiling routed by hardware, CUDA/HIP, Triton and Metal kernels, and torch.compile. The included engine skeleton targets NVIDIA CUDA.
+- **One timer for every engine:** interleaved rounds in a balanced order, medians with 95% confidence intervals.
+- **Anti-cheat gates:** caching, token counts, output length, rare tokens, sampling probes, prompt rendering, speculative-decoding exactness.
+- **Quality checks:** reference quality and GSM8K for language models; latent and final-clip checks for video. Images and speech use client timing and structural checks plus an external semantic quality protocol; see [media benchmarks](docs/media-benchmarks.md).
+- **Budget control:** spending authorized up front, with caps in hardware hours, dollars and elapsed time. No repeated plan approval inside the agreed scope.
+- **Reproducible output:** a ranked table labelled PASS, FAIL, UNPROVEN or NOT RUN, a launch command, a review page and the commands that reproduce every number.
+
+### Tools behind each step
+
+| Step | Tools |
+|---|---|
+| Size the problem | `intake`, `doctor`, `ceiling bw`, `roofline` |
+| Start from the best engine | `bench rounds`, `gates run`, `quality` |
+| Find the bottleneck | `doctor` routes profiling; Nsight commands on NVIDIA, vendor tools on AMD/Apple |
+| Pull the right levers, prove every step | `correctness`, `bench rounds`, `gates run`, `quality` |
+| Ship it | `ledger`, `review`, `report` |
+
+## Install
+
+Requirements: [uv](https://docs.astral.sh/uv/) and Python 3.10 or newer. Install the engine runtime for your hardware. NVIDIA capture commands require CUDA and Nsight; AMD and Apple profiling uses vendor tools with separate evidence files. See [supported hardware](#supported-hardware-and-agents).
+
+### Claude Code
+
+```bash
+claude plugin marketplace add kittenml/inferencekit
+claude plugin install inferencekit@inferencekit
+```
+
+### Codex
+
+```bash
+codex plugin marketplace add kittenml/inferencekit
+codex plugin add inferencekit@inferencekit
+```
+
+### OpenCode
+
+```bash
+git clone --depth 1 https://github.com/kittenml/inferencekit ~/.local/share/inferencekit
+python3 ~/.local/share/inferencekit/plugin/opencode/install.py
+```
+
+| Agent | Commands | Check the install |
+|---|---|---|
+| Claude Code | `/inferencekit:optimize-inference`, `/inferencekit:bench`, `/inferencekit:profile`, `/inferencekit:perf-report` | `claude plugin list`; `/mcp` lists the inferencekit server |
+| Codex | same stages; subagent files are followed one at a time where Codex has no subagents | `codex plugin list`; `codex mcp list` |
+| OpenCode | `/optimize-inference`, `/bench`, `/profile`, `/perf-report`, and each skill as `/<skill-name>` | `opencode mcp list` |
+
+Updating, removing, per-project installs and OpenCode's options: [docs/install.md](docs/install.md).
+
+## Try it
+
+```text
+/inferencekit:optimize-inference Qwen/Qwen3-8B-FP8 on 1x RTX 5090, fastest single-user decode, by 6h
+```
+
+To tune existing engines without custom kernels:
+
+```text
+/inferencekit:optimize-inference Tune vLLM and SGLang for my model and GPU using their built-in settings only; do not write custom kernels.
+```
+
+In OpenCode the command is `/optimize-inference`. It asks only for missing preferences and spending authorization; a plan is optional. For a repeatable setup, start from an intake file: `inferencekit intake template --out intake.yaml`, then `/inferencekit:optimize-inference intake.yaml`. For unattended runs, set `non_interactive: true` and `budget.approved: true` with a cap.
+
+More things to ask for: [docs/examples.md](docs/examples.md).
+
+## CLI and MCP server
+
+```bash
+uv tool install "inferencekit-cli==0.1.0"
+inferencekit doctor                                         # GPU, driver, CUDA, profiler access
+inferencekit gates run --base-url http://127.0.0.1:8000     # anti-cheat gates on any OpenAI-compatible server
+uvx --from 'inferencekit-cli[mcp]==0.1.0' inferencekit mcp serve   # MCP server (stdio) for any MCP client
+```
+
+Every CLI command is also an MCP tool with the same arguments. Text benchmarks, gates and quality checks talk HTTP to OpenAI-compatible servers. Image and streamed speech measurements use the [media adapter contracts](docs/media-benchmarks.md).
+
+## Supported hardware and agents
+
+| | Status | Run end to end |
+|---|---|---|
+| NVIDIA GPUs (CUDA) | Full tool set | Yes: RTX 5090, RTX PRO 6000 |
+| AMD ROCm GPUs, Apple silicon | Hardware discovery and vendor profiling guidance; HTTP benchmarks; vendor bandwidth/captures collected externally | RX 9070 XT and Apple M4 case studies above |
+| x86 CPUs | Skills and HTTP benchmarks; platform profiling collected externally | No |
+| ARM CPUs | Not covered | No |
+| Language models | Benchmarks, gates, GSM8K, quality vs a reference | Yes |
+| Text-to-video | Per-clip timing, per-step check, final-clip PSNR | Yes |
+| Text-to-image | Client-timed decoded images and structural checks; external quality proof required | No |
+| Speech models | Client-timed streamed PCM, first-signal latency, RTF and underruns; external quality proof required | No |
+| Claude Code | Supported | Yes |
+| Codex, OpenCode | Install paths and CI install checks | No |
+
+Profiles and logs can contain environment variables, host names and paths: scan them before you share them.
+
+## Roadmap
+
+- **More hardware:** H100, H200, B200, RTX 4090, Intel GPUs and Jetson, with the hardware tools (`doctor`, `ceiling bw`, profiling) for each.
+- **More models and modalities:** speech (TTS and STT), image, more video models.
+- **Multi-user serving:** published results for many users at once (load sweeps), not just one user at a time.
+- **OpenCode end to end.**
+
+## Contributing
+
+We welcome bug reports, engine settings and new skills. See [CONTRIBUTING.md](CONTRIBUTING.md). Develop with `uv sync --all-extras`, then `uv run --all-extras pytest -q` (no GPU needed). Report security problems privately: [SECURITY.md](SECURITY.md).
+
+## License
+
+Apache-2.0: [LICENSE](LICENSE) and [NOTICE](NOTICE). Copyright 2026 Stellon Labs. inferencekit is an independent project, not affiliated with NVIDIA, any engine project or any model publisher. Product and model names belong to their owners. Every result holds for the GPU, driver, engine version, model and workload it was measured on; change any of them and the numbers can change. The software comes with no warranty, as the Apache-2.0 licence states, and you are responsible for the cost of any GPU time you use with it.

inferencekit_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,104 @@
+inferencekit/__init__.py,sha256=PJmV9jtnKhDfQa9G90COALhzphb0ki25xKjSblnG-qU,378
+inferencekit/__main__.py,sha256=7i0SjFFDvEJVQuG69KjaoiDdQ7GthDgLM5JBaKrPJvQ,149
+inferencekit/canonical.py,sha256=UOKPEblWi--MqN9KYSctu5Y_loWyV9rN8EKcYH-z26g,4187
+inferencekit/checkpoint.py,sha256=mIKBWz_kg4zcQxNt8rLjhkk2kslrbtmfZg6kZt8M4dk,16602
+inferencekit/cli.py,sha256=jt_t1-3udOXWt3-Qz3oFJiPeLxaxmmRuR5bNrcr1X-Q,13336
+inferencekit/evidence.py,sha256=gRhxNJtNinQ0bXr4zteBEqoqikr5Wfs2OrW5tV4gQ8Y,18864
+inferencekit/gpuinfo.py,sha256=RJuv5UpeYW4o0VBnzv9TgfNErF4zOdVLzyGiR2xsXuk,3600
+inferencekit/hardware.py,sha256=fHTq-VmSp3UxTZl4PUlJFhN7ZeDT1SO6vZQ57c9g3pI,6592
+inferencekit/mcp_server.py,sha256=RIp7L4l_AU7qnjTV1ftHpfxGA7X8YitHRFvF5vWmFAg,8944
+inferencekit/proc.py,sha256=llS8tKL7Cgv8zHASmRc879DHUno0kroi-aaNmYd1z5c,8144
+inferencekit/registry.py,sha256=xf3NzzXM0lqnYQQNQLSZTibZt6xUd8hwWzshOAZ8ieI,7587
+inferencekit/roofline_headers.py,sha256=4U7Dv3SBO_mLbTTwSDJAGePEFLDO_ZsagzQZMrvmvEY,25598
+inferencekit/stats.py,sha256=4Ow1aGdSQYdyohaIKtMRySO_A-gRN30VgUkMgMxFuAI,12045
+inferencekit/analysis/__init__.py,sha256=PpsmjGnAkH22GjmMe2bY4BFU56YIW-mWjtJuPUSKPeE,98
+inferencekit/analysis/ncu_csv.py,sha256=MiIduu3QPN4c4dYxqFOJmK-DMFML6ibbCimHuKWm-jg,14417
+inferencekit/analysis/nsys_sqlite.py,sha256=hVGNPfuJOuPSdp-rO6M9QnTyWy1Y2hpXYyywM6Dav-4,25901
+inferencekit/cuda/bw_peak.cu,sha256=LkiVtN5Kawqad4KgiNp_Q_qSuJl8s5FzsOtTsKe2uko,11205
+inferencekit/cuda/torch_bw.py,sha256=8VxbGCZ8jdoY_fBrYn7Kqc6_uXFHdkoDe3-mwiHdYkk,2524
+inferencekit/engines/__init__.py,sha256=mRrlkPZ7CKwD2qsx8XIZxU0QEmWQj3Otq1BjFlXfPCw,353
+inferencekit/engines/adapters.py,sha256=NGSrAKqeBbrfC8EuI8OMs7eOcmE4dygr93xD5bi8aCQ,9844
+inferencekit/engines/gates.py,sha256=t9FDgheJyZiXHqy4i2W5gf4rUo7CRM1-d4T5Sge60t0,4929
+inferencekit/engines/sglang_worker.py,sha256=JsF8qvDIjyBfcQBVl2b6HBbotcM4NuIEmJiVh1wRWrM,3602
+inferencekit/engines/vllm_worker.py,sha256=zg34kpR7klRYjrmY-z__MCBSC0meuKkVhREyb9ilugE,5638
+inferencekit/engines/clip/__init__.py,sha256=Eu9KiTRUoHLYT1Puqyd_i9L7F2WY2Aqu-vls1jVUe2s,879
+inferencekit/engines/clip/comfyui.py,sha256=ts55pn1xF8RmcKsTGpITAh2_BfhNx4k6HCqdGKEhRdw,40169
+inferencekit/engines/clip/diffusers_ltx.py,sha256=ap_bvVu3Ai08SeqyLIHqIdH23uzWI_dXRIxykMSQbfs,34596
+inferencekit/engines/clip/noise.py,sha256=_jIqqfnJDtChjqbopOw7fS8GPGp0DlxGcDho6DfeBI0,2154
+inferencekit/engines/clip/official_ltx.py,sha256=fOfz8_69xgYHsuscm-7uEyv3VZhIyY9OVcimFuCRvas,17747
+inferencekit/engines/clip/server.py,sha256=nc5JCXJTMMALY-9yLzOwTN80LhwCtAEMoSCcQ0kVKzA,24501
+inferencekit/engines/clip/stepdump.py,sha256=L_TQVvv3nolvwUY7W8fx67Dw_9fPoXkI8rMnMFERs_M,8698
+inferencekit/ops/__init__.py,sha256=DAmvBcl1SoTQI-RyTnjgV7XTfmNMayPCLcBOSCFczxA,1140
+inferencekit/ops/bench.py,sha256=89GmgkmtMtkZoUIjntcnDVz3gAOzs4WUNCjtu_ZNKF8,10630
+inferencekit/ops/bench_audio.py,sha256=2nr4DCSQGa8Xdd3ClJKs0Iyuaf_w9xe6e2GLXK9sPls,25787
+inferencekit/ops/bench_clip.py,sha256=Kcfe396Tz0CbQ0U9TCtKZPz3Y2VeajSVwEpPO3QwPbc,49376
+inferencekit/ops/bench_image.py,sha256=x18X8HNpkSRxZi_K-mFGzmBSVLOwMctGsimg1osIRfo,24948
+inferencekit/ops/bench_rounds.py,sha256=JozJWlsWAvt4hB5GFKLqcrrcXhOD3z9f70CwSYXnds4,21028
+inferencekit/ops/bench_serve.py,sha256=6e9gOXYhWQ7JdwEYou3g_LXsfXdKp_-Ak7D_mhUeTss,53226
+inferencekit/ops/ceiling.py,sha256=NHmTxoNWkr0uAy8rbx1xafiY6xByCnSV7WSqHtaBwdQ,7470
+inferencekit/ops/compare.py,sha256=hgA2B4pRhkH6vAnj8aPzOtfK3cOVUrk9P5GLtkC23iY,24787
+inferencekit/ops/correctness.py,sha256=PmTUkDUr1PsZS9IIDM9tZuutEaa8_rSeLFryyAM_eho,7861
+inferencekit/ops/doctor.py,sha256=q-qLNzRMjtOTqoPKB2cgPK3GGIpEYNVZAm_wbauKH1c,25866
+inferencekit/ops/gates.py,sha256=TrL5GL-Z5Wicve3oBlwF6PZBLK0jcufknN9ViOmI_ng,99334
+inferencekit/ops/gates_clip.py,sha256=l-3lgBtn8RSdZgoJiyuSqMDQoASGQNkVcjAFEhbhPjU,55608
+inferencekit/ops/intake.py,sha256=OoFRi_7YTeM9adj7Cz-GNNdTkpvVj8RcHfX625a8y3c,75411
+inferencekit/ops/ledger.py,sha256=1YvpD5nRediCBBa9tSsqb36_u_dPIfIXKkYLz_pQxTg,39492
+inferencekit/ops/manifest.py,sha256=QnKzucYxoXi3iZ2ZOcsu6oIS9kK3A--wI6noELvhMtQ,17621
+inferencekit/ops/ncu.py,sha256=KUQ7ExsEW8pmA5-M8USeG9MgD5Zu5L_C2hhipb6qVvc,10866
+inferencekit/ops/nsys.py,sha256=ocUgmOfT4VgyhpJhWbqu1nHJkY6a64hhx-Ip3-etYa4,13623
+inferencekit/ops/prompts.py,sha256=KZm7ufbLndli-QTtIGO6HLyNqUZ93CTQY8BMoO4Tvxs,16321
+inferencekit/ops/quality.py,sha256=Ib9DQK_ybDFBoLSlKPN81-tJuPEyDw-5SLuCqqFwrnw,129400
+inferencekit/ops/quality_clip.py,sha256=Uj6NTx-HkmqpX992Lx1iEUud6zKxZZtG0utRcI91Y14,42255
+inferencekit/ops/quality_clip_steps.py,sha256=E7CnwAUC3TWmThby_VcALoI5-RkP1T9k7r38hNlQNYg,21885
+inferencekit/ops/report.py,sha256=SXXpxZ_8zhngoBrcXQFhAhEMiwVvdI0-OLGKeEFFOM0,17306
+inferencekit/ops/review.py,sha256=2lYrLVtxcf7pTDuSQbNQ9BGAQMIZIG8lHVrqi3J_d8M,6241
+inferencekit/ops/rocprof.py,sha256=09-UK6_3p9JMJ7ztBD_q0Yzifu8X2zUqNRNalBUgHyA,5356
+inferencekit/ops/roofline.py,sha256=ZU34aygsNkUcBxBKfvzuri52TRfCp_UTq-iEl7aGlCk,22317
+inferencekit/ops/store.py,sha256=BBwvO5qML3Z442DH5SgYEpYRXbjnKzlGlyN4jOpm7fc,3049
+inferencekit/prompts/ATTRIBUTION.md,sha256=zppCL-BM8dXbBvDHLqm2R2NWfavwEidY3yWRnPJhGkc,3900
+inferencekit/prompts/__init__.py,sha256=w66vzdYU-geXKO-22svk6vUmNmPg9ph1VYBYeuiAClk,6347
+inferencekit/prompts/build.py,sha256=VEcpCuxkRRjQOi2FXxlJoqjvSCgaox4-FHKA9WMMNLk,26840
+inferencekit/prompts/dev-load.jsonl.gz,sha256=uHFhk6_e7ijropWthUkciP--itSBjXVC6v7q-9jtsQ8,239038
+inferencekit/prompts/dev-load.manifest.json,sha256=WTGYf-z4o3_Oqx3Dlxi0Z9eHs27R66N4O8XeOZ4r-5Y,8001
+inferencekit/prompts/dev.jsonl.gz,sha256=Cavr1imNrBrNwtvAOHl6ySYJF7aJGSl4lnN7i0WKV9A,87529
+inferencekit/prompts/dev.manifest.json,sha256=C55Un2fKfH1ytaEQFnpqy-z0PJoN1KPPWImk3D-yd64,8031
+inferencekit/prompts/exclusions.json,sha256=HgWDMSd7O5xnuFeamUoHljPOMPK-wc_L6HBSU2kNTYY,2203
+inferencekit/quality/__init__.py,sha256=ELORbnqiu8pkHuFVAauZA80Nc5t-Wptu9qsoZv4vvzw,1463
+inferencekit/quality/client.py,sha256=1sf2dThe3vcwq6pypQlsfrYULvbAy8mdK_5Fbq3xbvg,2597
+inferencekit/quality/fake.py,sha256=CdHlJf3J-LWqiWrehQEq315FfGV3ua0gsQ6BKVIw6So,16319
+inferencekit/quality/gsm8k.py,sha256=HEWkqT6Zq9bvxSnLhj53HLbytysw8zP_c9OGxq0rWiM,9959
+inferencekit/quality/logprobs.py,sha256=0UQfWUkRHV_o0gOrpNYdDrCpNa1SiSAr0vDOhTPJ9ac,4625
+inferencekit/quality/metrics.py,sha256=PFOgzPrxJkEFv82Cc4G3KYZkX4jiEbKnwYk2CrTJpNg,10407
+inferencekit/quality/niah.py,sha256=FAYe76LmDoqYfxzS2qDypZiWs3x7tI_bha9LxL0AR8c,3515
+inferencekit/quality/parquet.py,sha256=HGI88TssEGqEAs6UJ1wpAXCWIntMBa5F4nVwg8QGtaM,11267
+inferencekit/quality/sampling.py,sha256=he-pLZXCIz2bK7YuGj197EXxXo2j4n2XZiBPMg92DgU,14739
+inferencekit/quality/step_replay.py,sha256=T-cPg8s7RKXLrqa0U_ogfipW8m3U-5umKUL_a7EKwFY,9856
+inferencekit/quality/video_metrics.py,sha256=c9EXoOr28Cm8WTe8V0enr8ENMGYhjDFX-JRMP8ESSjs,14356
+inferencekit/quality/data/gsm8k_LICENSE.txt,sha256=hru3PoVYIdfEAZEv1L-C40MT5uO2_W-QnytsyeIJpTs,1062
+inferencekit/review/__init__.py,sha256=OvarRG6lMYdrU4tl9pKEx67nywwKkk9xKhAnHKh5ujc,320
+inferencekit/review/diff.py,sha256=cnR3_ydwL5lom0DN0wqrp1uNkmBO64bzbSBHnQvdwLI,6805
+inferencekit/review/media.py,sha256=Evrbg4ueHOATsovO7Gw4xZf8KiO3ip8MOkVX3VOnfBo,9192
+inferencekit/review/page.py,sha256=Ty_vH5ZjcbaZ4hCDn903KsUMPkqsQyeZYkQVG5zQfNQ,51471
+inferencekit/review/rundir.py,sha256=2T3Dbd_Ee1LTHFGsvhNy6E4-ckpAAvE20sjVtHgejxg,43505
+inferencekit/serve/__init__.py,sha256=El5XiZMZJn89O0k_oHyEmDxFYHt0wiXNi0pYuAySmZg,518
+inferencekit/serve/audio_client.py,sha256=Qqut5YjmtkC38do8NQYwHGK8BEKGKbaLUKo6xCiSX_c,15041
+inferencekit/serve/client.py,sha256=cQbOcLOwBNCuqjjakOJn3k9sZ6Ho16cI9ZET8whgsJ0,18755
+inferencekit/serve/clip_client.py,sha256=cRPWx-Am21wePDAXATEG0rnaYqBKm3ghj3WnmBLen5M,17202
+inferencekit/serve/clip_rounds.py,sha256=uWml18v7aJo89qfNJBqfEv0M_lAMikMs6lXiVetcCVQ,28059
+inferencekit/serve/image_client.py,sha256=2q9zXz0QJdYYcYOYw8ZOPX78vYBnF-9Sx6AO9ddv54s,17635
+inferencekit/serve/loadgen.py,sha256=7Dd5POFeozmPUqJ4gLcqG7NNSom7mDGEVQMALiYlPF8,5787
+inferencekit/serve/metrics.py,sha256=DXnwhWMxbAjmEA0BjsCuBfPfUf9_kzpj_bwmC4XnArE,10755
+inferencekit/serve/profiles.py,sha256=2wWgxDSUVxWzcYD8WNriyimJCg1rWLbWlTaQm9VrmNM,5758
+inferencekit/serve/prompts_io.py,sha256=q5P2YvGQDA5NnGy0uK7_WDer1adL44M5MzK2z3pNZaw,5384
+inferencekit/serve/rounds.py,sha256=Eu2eCxl_qHTcpA6-OUMnHC3q9UM75dWM3Y-dupqOnQY,61352
+inferencekit/serve/servers.py,sha256=D0KspAuZaK-QZAxsgjmdMYrqNSRzFk9t_1YZeQYqgLc,40472
+inferencekit/schemas/evidence.v1.json,sha256=hGSdeJzPFsewHTSvegV2bFuGI6CvUkWHlTObdGXMFTU,4976
+inferencekit/schemas/evidence.v2.json,sha256=QPIf7vnSGEKt5Wpc5niWexmVq9s_GRlmeeGQN_JZKY4,15450
+inferencekit/schemas/intake.v1.json,sha256=msxET5X-4lwJ6NGYYPgIRizBaPIgr1L2d7BAT53Xw0U,23089
+inferencekit/schemas/run-manifest.v1.json,sha256=CujWEgDwYu3MxWYUr37qc354Tu3THyuN4IQxUPMP4BM,8654
+inferencekit_cli-0.1.0.dist-info/METADATA,sha256=n0sJ7WBO993TWIezoX4cMKk_ls7XZKfk8-iOF1nvRVI,17953
+inferencekit_cli-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+inferencekit_cli-0.1.0.dist-info/entry_points.txt,sha256=pl01EvSJ2tIjZ6XJuyABIKTeiS_WCbi02wILbMT6vnw,55
+inferencekit_cli-0.1.0.dist-info/licenses/LICENSE,sha256=_IkNG2bSqucQOHCATYVgvIPqkI8nPQZqx2bXzIjtI80,11343
+inferencekit_cli-0.1.0.dist-info/licenses/NOTICE,sha256=o4B_ehJCo7di6Qoo3ztq-OHrhDlugPZuXmFkrhkJ6Fc,364
+inferencekit_cli-0.1.0.dist-info/RECORD,,

inferencekit_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

inferencekit_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+inferencekit = inferencekit.cli:main