inferencekit-cli 0.1.0__py3-none-any.whl

inferencekit/cuda/bw_peak.cu

@@ -0,0 +1,252 @@
+// Practical DRAM bandwidth of the GPU: the "100%" that decode kernels should be
+// compared against (datasheet bandwidth is a theoretical peak no kernel reaches).
+//
+// Tests, each printed as one JSON object per line on stdout:
+//   device      name, SM count, L2 size, bus width, memory clock, theoretical GB/s
+//   read        read-only stream over N bytes (16-byte non-caching loads, 4 in flight
+//               per thread); L2 is flushed before every timed run so small buffers
+//               measure DRAM, not cache
+//   copy        read N/2 bytes + write N/2 bytes (counts N bytes moved)
+//   graph_read  a CUDA graph of 32 back-to-back launches, each reading its own S-byte
+//               region: the best a kernel of that size can do. The gap to "read" is
+//               the per-launch ramp-up and tail, which is why small decode kernels
+//               rarely reach peak bandwidth. With PDL (programmatic dependent launch,
+//               sm_90+) the next launch starts its prologue while the previous drains.
+//
+// Build: nvcc -O3 -std=c++17 -arch=native -o bw_peak bw_peak.cu
+// Usage: bw_peak [--sizes-mib 8,32,128,512,1024] [--graph-mib 1,4,16] [--iters 30]
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#define CHECK(x)                                                                              \
+  do {                                                                                        \
+    cudaError_t err_ = (x);                                                                   \
+    if (err_ != cudaSuccess) {                                                                \
+      std::fprintf(stderr, "%s:%d %s: %s\n", __FILE__, __LINE__, #x, cudaGetErrorString(err_)); \
+      std::exit(1);                                                                           \
+    }                                                                                         \
+  } while (0)
+
+// 16-byte load that bypasses L1 (we never reuse the data, so caching it only costs).
+__device__ __forceinline__ uint4 load_stream(const uint4* p) {
+#if __CUDA_ARCH__ >= 700
+  uint4 v;
+  asm volatile("ld.global.nc.L1::no_allocate.v4.u32 {%0,%1,%2,%3}, [%4];"
+               : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
+               : "l"(p));
+  return v;
+#else
+  return __ldg(p);
+#endif
+}
+
+// Grid-stride read. UNROLL independent loads are issued before any is used, so each
+// thread keeps several memory requests in flight (memory-level parallelism).
+template <int UNROLL>
+__global__ void read_kernel(const uint4* __restrict__ p, size_t n16, unsigned* sink, int pdl) {
+#if __CUDA_ARCH__ >= 900
+  if (pdl) asm volatile("griddepcontrol.launch_dependents;" ::: "memory");
+#endif
+  unsigned acc = 0;
+  const size_t stride = (size_t)gridDim.x * blockDim.x;
+  size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+  for (; i + (UNROLL - 1) * stride < n16; i += UNROLL * stride) {
+    uint4 v[UNROLL];
+#pragma unroll
+    for (int u = 0; u < UNROLL; ++u) v[u] = load_stream(p + i + u * stride);
+#pragma unroll
+    for (int u = 0; u < UNROLL; ++u) acc ^= v[u].x ^ v[u].y ^ v[u].z ^ v[u].w;
+  }
+  for (; i < n16; i += stride) {
+    const uint4 v = load_stream(p + i);
+    acc ^= v.x ^ v.y ^ v.z ^ v.w;
+  }
+#if __CUDA_ARCH__ >= 900
+  if (pdl) asm volatile("griddepcontrol.wait;" ::: "memory");
+#endif
+  if (acc == 0x9e3779b9u) sink[0] = acc;  // data-dependent store: the compiler cannot drop the loads
+}
+
+__global__ void copy_kernel(const uint4* __restrict__ a, uint4* __restrict__ b, size_t n16) {
+  const size_t stride = (size_t)gridDim.x * blockDim.x;
+  for (size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; i < n16; i += stride) b[i] = a[i];
+}
+
+// Writes a buffer larger than L2 so the next timed kernel starts with a cold cache.
+__global__ void flush_kernel(float* f, size_t n) {
+  const size_t stride = (size_t)gridDim.x * blockDim.x;
+  for (size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; i < n; i += stride) f[i] += 1.0f;
+}
+
+static float median(std::vector<float> v) {
+  std::sort(v.begin(), v.end());
+  return v[v.size() / 2];
+}
+
+static std::vector<size_t> parse_list(const char* s) {
+  std::vector<size_t> out;
+  std::string str(s);
+  size_t pos = 0;
+  while (pos < str.size()) {
+    size_t comma = str.find(',', pos);
+    if (comma == std::string::npos) comma = str.size();
+    out.push_back(std::strtoull(str.substr(pos, comma - pos).c_str(), nullptr, 10));
+    pos = comma + 1;
+  }
+  return out;
+}
+
+int main(int argc, char** argv) {
+  std::vector<size_t> sizes_mib = {8, 32, 128, 512, 1024};
+  std::vector<size_t> graph_mib = {1, 4, 16};
+  int iters = 30;
+  for (int a = 1; a + 1 < argc; a += 2) {
+    if (!std::strcmp(argv[a], "--sizes-mib")) sizes_mib = parse_list(argv[a + 1]);
+    else if (!std::strcmp(argv[a], "--graph-mib")) graph_mib = parse_list(argv[a + 1]);
+    else if (!std::strcmp(argv[a], "--iters")) iters = std::max(5, std::atoi(argv[a + 1]));
+  }
+  const int launches_per_graph = 32;
+
+  int dev = 0, sms = 0, l2 = 0, bus_bits = 0, mem_khz = 0, cc_major = 0, cc_minor = 0;
+  CHECK(cudaSetDevice(dev));
+  CHECK(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev));
+  CHECK(cudaDeviceGetAttribute(&l2, cudaDevAttrL2CacheSize, dev));
+  CHECK(cudaDeviceGetAttribute(&cc_major, cudaDevAttrComputeCapabilityMajor, dev));
+  CHECK(cudaDeviceGetAttribute(&cc_minor, cudaDevAttrComputeCapabilityMinor, dev));
+  // Memory clock and bus width are informational; some toolkits deprecate them, so ignore errors.
+  if (cudaDeviceGetAttribute(&bus_bits, cudaDevAttrGlobalMemoryBusWidth, dev) != cudaSuccess) bus_bits = 0;
+  if (cudaDeviceGetAttribute(&mem_khz, cudaDevAttrMemoryClockRate, dev) != cudaSuccess) mem_khz = 0;
+  cudaGetLastError();
+  cudaDeviceProp prop;
+  CHECK(cudaGetDeviceProperties(&prop, dev));
+  // DDR: two transfers per clock. bytes/s = 2 * clock(Hz) * bus width(bytes).
+  const double theoretical_gbs = 2.0 * mem_khz * 1e3 * (bus_bits / 8.0) / 1e9;
+  std::printf("{\"test\":\"device\",\"name\":\"%s\",\"cc\":\"%d.%d\",\"sms\":%d,\"l2_bytes\":%d,\"bus_bits\":%d,"
+              "\"mem_clock_khz\":%d,\"theoretical_gbs\":%.1f}\n",
+              prop.name, cc_major, cc_minor, sms, l2, bus_bits, mem_khz, theoretical_gbs);
+
+  size_t max_bytes = 0;
+  for (size_t m : sizes_mib) max_bytes = std::max(max_bytes, m << 20);
+  for (size_t m : graph_mib) max_bytes = std::max(max_bytes, (m << 20) * launches_per_graph);
+  size_t free_b = 0, total_b = 0;
+  CHECK(cudaMemGetInfo(&free_b, &total_b));
+  const size_t flush_bytes = std::max<size_t>(size_t(4) * l2, size_t(64) << 20);
+  if (2 * max_bytes + flush_bytes > free_b * 9 / 10) {
+    std::fprintf(stderr, "not enough free GPU memory for %zu MiB buffers\n", max_bytes >> 20);
+    return 2;
+  }
+  uint4 *a, *b;
+  unsigned* sink;
+  float* flush;
+  CHECK(cudaMalloc(&a, max_bytes));
+  CHECK(cudaMalloc(&b, max_bytes));
+  CHECK(cudaMalloc(&sink, 4));
+  CHECK(cudaMalloc(&flush, flush_bytes));
+  CHECK(cudaMemset(a, 1, max_bytes));
+  CHECK(cudaMemset(b, 0, max_bytes));
+  CHECK(cudaMemset(flush, 0, flush_bytes));
+  cudaStream_t s;
+  CHECK(cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking));
+  cudaEvent_t e0, e1;
+  CHECK(cudaEventCreate(&e0));
+  CHECK(cudaEventCreate(&e1));
+  auto flush_l2 = [&]() { flush_kernel<<<sms * 4, 512, 0, s>>>(flush, flush_bytes / 4); };
+
+  // ---- read and copy over several buffer sizes
+  for (size_t mib : sizes_mib) {
+    const size_t bytes = mib << 20;
+    const size_t n16 = bytes / 16;
+    for (int blocks_per_sm : {4, 8}) {
+      std::vector<float> t_read, t_copy;
+      for (int it = 0; it < iters + 3; ++it) {  // first 3 iterations are warm-up
+        float ms = 0;
+        flush_l2();
+        CHECK(cudaEventRecord(e0, s));
+        read_kernel<4><<<sms * blocks_per_sm, 512, 0, s>>>(a, n16, sink, 0);
+        CHECK(cudaEventRecord(e1, s));
+        CHECK(cudaEventSynchronize(e1));
+        CHECK(cudaEventElapsedTime(&ms, e0, e1));
+        if (it >= 3) t_read.push_back(ms);
+        flush_l2();
+        CHECK(cudaEventRecord(e0, s));
+        copy_kernel<<<sms * blocks_per_sm, 512, 0, s>>>(a, b, n16 / 2);
+        CHECK(cudaEventRecord(e1, s));
+        CHECK(cudaEventSynchronize(e1));
+        CHECK(cudaEventElapsedTime(&ms, e0, e1));
+        if (it >= 3) t_copy.push_back(ms);
+      }
+      const float mr = median(t_read), mc = median(t_copy);
+      const float br = *std::min_element(t_read.begin(), t_read.end());
+      std::printf("{\"test\":\"read\",\"bytes\":%zu,\"blocks_per_sm\":%d,\"iters\":%d,\"ms_median\":%.5f,"
+                  "\"gbs\":%.1f,\"gbs_best\":%.1f}\n",
+                  bytes, blocks_per_sm, iters, mr, bytes / mr / 1e6, bytes / br / 1e6);
+      std::printf("{\"test\":\"copy\",\"bytes\":%zu,\"blocks_per_sm\":%d,\"iters\":%d,\"ms_median\":%.5f,"
+                  "\"gbs\":%.1f}\n",
+                  bytes, blocks_per_sm, iters, mc, bytes / mc / 1e6);
+    }
+  }
+
+  // ---- launch-sized reads inside a CUDA graph
+#if CUDART_VERSION >= 12030  // PDL edges inside captured graphs need CUDA 12.3+
+  const bool pdl_ok = cc_major >= 9;
+#else
+  const bool pdl_ok = false;
+#endif
+  for (size_t mib : graph_mib) {
+    const size_t bytes = mib << 20;
+    for (int pdl = 0; pdl <= (pdl_ok ? 1 : 0); ++pdl) {
+      cudaGraph_t graph;
+      cudaGraphExec_t exec;
+      CHECK(cudaStreamBeginCapture(s, cudaStreamCaptureModeGlobal));
+      for (int l = 0; l < launches_per_graph; ++l) {
+        const uint4* src = a + (size_t)l * (bytes / 16);  // each launch reads its own region
+#if CUDART_VERSION >= 11080
+        cudaLaunchConfig_t cfg = {};
+        cfg.gridDim = dim3(sms * 2);
+        cfg.blockDim = dim3(512);
+        cfg.stream = s;
+        cudaLaunchAttribute attr[1];
+        attr[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
+        attr[0].val.programmaticStreamSerializationAllowed = 1;
+        cfg.attrs = attr;
+        cfg.numAttrs = pdl ? 1 : 0;
+        CHECK(cudaLaunchKernelEx(&cfg, read_kernel<1>, src, bytes / 16, sink, pdl));
+#else
+        read_kernel<1><<<sms * 2, 512, 0, s>>>(src, bytes / 16, sink, 0);
+#endif
+      }
+      CHECK(cudaStreamEndCapture(s, &graph));
+      CHECK(cudaGraphInstantiateWithFlags(&exec, graph, 0));
+      std::vector<float> t;
+      for (int it = 0; it < iters + 3; ++it) {
+        float ms = 0;
+        flush_l2();
+        CHECK(cudaEventRecord(e0, s));
+        CHECK(cudaGraphLaunch(exec, s));
+        CHECK(cudaEventRecord(e1, s));
+        CHECK(cudaEventSynchronize(e1));
+        CHECK(cudaEventElapsedTime(&ms, e0, e1));
+        if (it >= 3) t.push_back(ms);
+      }
+      const float ms = median(t);
+      const double total = (double)bytes * launches_per_graph;
+      std::printf("{\"test\":\"graph_read\",\"bytes_per_launch\":%zu,\"launches\":%d,\"pdl\":%d,\"iters\":%d,"
+                  "\"us_per_launch\":%.2f,\"gbs\":%.1f}\n",
+                  bytes, launches_per_graph, pdl, iters, ms * 1e3 / launches_per_graph, total / ms / 1e6);
+      CHECK(cudaGraphExecDestroy(exec));
+      CHECK(cudaGraphDestroy(graph));
+    }
+  }
+  CHECK(cudaFree(a));
+  CHECK(cudaFree(b));
+  CHECK(cudaFree(sink));
+  CHECK(cudaFree(flush));
+  return 0;
+}

inferencekit/cuda/torch_bw.py

@@ -0,0 +1,60 @@
+"""Bandwidth fallback with PyTorch, used when nvcc is not available.
+
+Prints the same JSON lines as bw_peak.cu ("device", "read", "copy"). It runs in its
+own process (``python torch_bw.py --sizes-mib ...``) so the caller never imports
+torch or holds a CUDA context. Expect a few percent below bw_peak.cu: a reduction
+kernel also does arithmetic and a final cross-block step, a pure load loop does not.
+Standard library + torch only; does not import inferencekit.
+"""
+
+import argparse
+import json
+import statistics
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--sizes-mib", default="8,32,128,512,1024")
+    ap.add_argument("--iters", type=int, default=30)
+    args = ap.parse_args()
+    import torch
+
+    if not torch.cuda.is_available():
+        print(json.dumps({"test": "error", "message": "torch.cuda is not available"}))
+        return 2
+    props = torch.cuda.get_device_properties(0)
+    l2 = getattr(props, "L2_cache_size", 0) or (64 << 20)
+    print(json.dumps({"test": "device", "name": props.name, "cc": f"{props.major}.{props.minor}",
+                      "sms": props.multi_processor_count, "l2_bytes": l2, "theoretical_gbs": None}))
+    flush = torch.empty(max(4 * l2, 64 << 20) // 4, dtype=torch.float32, device="cuda")
+    start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+
+    def timed(fn) -> float:
+        times = []
+        for it in range(args.iters + 3):
+            flush.add_(1.0)  # evict L2 so the timed op reads DRAM
+            start.record()
+            fn()
+            end.record()
+            end.synchronize()
+            if it >= 3:
+                times.append(start.elapsed_time(end))
+        return statistics.median(times)
+
+    for mib in [int(x) for x in args.sizes_mib.split(",") if x]:
+        nbytes = mib << 20
+        x = torch.ones(nbytes // 4, dtype=torch.float32, device="cuda")
+        y = torch.empty(nbytes // 8, dtype=torch.float32, device="cuda")
+        ms = timed(lambda: torch.sum(x))
+        print(json.dumps({"test": "read", "bytes": nbytes, "iters": args.iters, "ms_median": ms,
+                          "gbs": nbytes / ms / 1e6, "method": "torch.sum"}))
+        half = x[: nbytes // 8]
+        ms = timed(lambda: y.copy_(half))
+        print(json.dumps({"test": "copy", "bytes": nbytes, "iters": args.iters, "ms_median": ms,
+                          "gbs": nbytes / ms / 1e6, "method": "Tensor.copy_"}))
+        del x, y
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

inferencekit/engines/__init__.py

@@ -0,0 +1,7 @@
+"""Benchmark adapters: one per engine.
+
+vLLM and SGLang run in a *worker script* executed by the engine's own interpreter
+(``--python /path/to/venv/bin/python``), so inferencekit never has to share a virtual
+environment with an engine and the engine's GPU memory is freed when the worker
+exits. llama.cpp and ``cmd`` are external binaries/commands.
+"""

inferencekit/engines/adapters.py

@@ -0,0 +1,190 @@
+"""Run one engine's benchmark and return a common result shape.
+
+Every adapter returns::
+
+    {"engine": {"name", "version", "commit"}, "flags": {...}, "rounds": [round, ...],
+     "tokens": [...] or None, "log_text": str, "limitations": [...], "files": [(path, role)]}
+
+where each round is ``{"index", "t_start", "t_end", "ttft_ms", "decode_tok_s",
+"tpot_ms", "gen_tokens"}`` (times in epoch seconds; None when unknown).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shlex
+import time
+from pathlib import Path
+
+from inferencekit.evidence import redact, redact_text
+from inferencekit.proc import run, which
+from inferencekit.registry import OpError
+
+WORKER_DIR = Path(__file__).resolve().parent
+MARK = "INFERKIT_RESULT "
+
+
+def _redact_file(path: Path) -> None:
+    """Engine logs can echo configuration; scrub secret-looking env values before keeping them."""
+    if path.is_file():
+        path.write_text(redact_text(path.read_text(errors="replace")))
+
+
+def run_python_worker(engine: str, python: str, cfg: dict, staging: Path, timeout: int) -> dict:
+    """vLLM / SGLang: run the worker script under the engine's interpreter."""
+    cfg_path = staging / "worker_config.json"
+    cfg_path.write_text(json.dumps(cfg, indent=2))  # the worker needs the real values while it runs
+    out_path, log_path = staging / "worker_stdout.txt", staging / "engine.log"
+    r = run([python, str(WORKER_DIR / f"{engine}_worker.py"), str(cfg_path)], timeout=timeout,
+            stdout_path=out_path, stderr_path=log_path)
+    # The kept copy must not carry a token typed into --engine-args ({"hf_token": ...}):
+    # redact() blanks secret-looking keys, redact_text secret env values and the home dir.
+    cfg_path.write_text(redact_text(json.dumps(redact(cfg), indent=2)))
+    _redact_file(log_path)
+    _redact_file(out_path)
+    lines = [ln for ln in r.stdout.splitlines() if ln.startswith(MARK)]
+    if not lines:
+        tail = log_path.read_text(errors="replace")[-1500:] if log_path.is_file() else r.stderr[-1500:]
+        raise OpError(f"{engine} worker failed (exit {r.returncode}); log tail:\n{tail}")
+    result = json.loads(lines[-1][len(MARK):])
+    result["log_text"] = log_path.read_text(errors="replace") if log_path.is_file() else ""
+    result["files"] = [(log_path, "engine_log"), (cfg_path, "worker_config")]
+    result.setdefault("limitations", [])
+    return result
+
+
+# --------------------------------------------------------------------------- llama.cpp
+
+def _llama_flag(key: str) -> str:
+    """{'ngl': 99} -> '-ngl'; {'flash-attn': 1} -> '--flash-attn'; keys with dashes pass through."""
+    if key.startswith("-"):
+        return key
+    return ("-" if len(key) <= 3 else "--") + key
+
+
+def run_llamacpp(model: str, prompt_len: int, gen_len: int, rounds: int, engine_args: dict, staging: Path,
+                 timeout: int) -> dict:
+    bench = which("llama-bench")
+    if bench:
+        return _run_llama_bench(bench, model, prompt_len, gen_len, rounds, engine_args, staging, timeout)
+    cli = which("llama-cli")
+    if cli:
+        return _run_llama_cli(cli, model, prompt_len, gen_len, rounds, engine_args, staging, timeout)
+    raise OpError("llama-bench / llama-cli not found on PATH (build llama.cpp with -DGGML_CUDA=ON)")
+
+
+def _run_llama_bench(bench, model, prompt_len, gen_len, rounds, engine_args, staging, timeout) -> dict:
+    extra = []
+    for k, v in engine_args.items():
+        extra += [_llama_flag(k), str(v)]
+    argv = [bench, "-m", model, "-p", str(prompt_len), "-n", str(gen_len), "-r", str(rounds), "-o", "json", *extra]
+    out_path, log_path = staging / "llama_bench.json", staging / "llama_bench.log"
+    t0 = time.time()
+    r = run(argv, timeout=timeout, stdout_path=out_path, stderr_path=log_path)
+    t1 = time.time()
+    _redact_file(log_path)
+    if not r.ok:
+        raise OpError(f"llama-bench failed (exit {r.returncode}): {redact_text(r.stderr[-1000:])}")
+    entries = json.loads(r.stdout)
+    pp = next((e for e in entries if e.get("n_prompt", 0) > 0 and e.get("n_gen", 0) == 0), None)
+    tg = next((e for e in entries if e.get("n_gen", 0) > 0 and e.get("n_prompt", 0) == 0), None)
+    if tg is None:
+        raise OpError("llama-bench output has no generation (tg) test")
+    tg_ts = tg.get("samples_ts") or [tg.get("avg_ts")]
+    pp_ns = (pp or {}).get("samples_ns") or ([pp["avg_ns"]] if pp and pp.get("avg_ns") else [])
+    out_rounds = []
+    for i, ts in enumerate(tg_ts):
+        ttft = pp_ns[i] / 1e6 if i < len(pp_ns) else None
+        out_rounds.append({"index": i, "t_start": None, "t_end": None, "ttft_ms": ttft, "decode_tok_s": ts,
+                           "tpot_ms": 1e3 / ts if ts else None, "gen_tokens": gen_len})
+    keys = ("n_gpu_layers", "flash_attn", "type_k", "type_v", "n_batch", "n_ubatch", "n_threads", "backends",
+            "split_mode", "gpu_info", "model_type")
+    flags = {k: tg.get(k) for k in keys if k in tg}
+    limitations = [
+        "llama-bench runs prefill (pp) and generation (tg) as separate tests: TTFT here is the pp time without "
+        "sampling, and tg decodes from an empty KV cache unless a depth (-d) is passed",
+        f"rounds ran back to back inside one llama-bench call ({t0:.0f}-{t1:.0f} epoch s): no per-round timestamps",
+    ]
+    if not tg.get("samples_ts"):
+        limitations.append("this llama-bench build reports only the average over repetitions (n=1)")
+    return {"engine": {"name": "llama.cpp", "version": str(tg.get("build_number") or "") or None,
+                       "commit": tg.get("build_commit")},
+            "flags": flags, "rounds": out_rounds, "tokens": None, "log_text": r.stderr, "limitations": limitations,
+            "files": [(out_path, "raw_output"), (log_path, "engine_log")]}
+
+
+_PROMPT_EVAL = re.compile(r"prompt eval time\s*=\s*([\d.]+)\s*ms\s*/\s*(\d+)\s*tokens")
+_EVAL = re.compile(r"(?<!prompt )eval time\s*=\s*([\d.]+)\s*ms\s*/\s*(\d+)\s*(?:runs|tokens)")
+
+
+def _run_llama_cli(cli, model, prompt_len, gen_len, rounds, engine_args, staging, timeout) -> dict:
+    extra = []
+    for k, v in engine_args.items():
+        extra += [_llama_flag(k), str(v)]
+    prompt = " ".join(["hello"] * prompt_len)  # roughly one token per word; exact length is not controlled
+    out_rounds, logs = [], []
+    for i in range(rounds):
+        argv = [cli, "-m", model, "-p", prompt, "-n", str(gen_len), "--temp", "0", "--ignore-eos", "-no-cnv", *extra]
+        t_start = time.time()
+        r = run(argv, timeout=timeout)
+        if not r.ok and "no-cnv" in r.stderr:  # newer builds dropped the flag
+            r = run([a for a in argv if a != "-no-cnv"], timeout=timeout)
+        t_end = time.time()
+        logs.append(r.stderr)
+        pe, de = _PROMPT_EVAL.search(r.stderr), _EVAL.search(r.stderr)
+        if not de:
+            raise OpError(f"could not find llama.cpp timing lines in llama-cli output (exit {r.returncode})")
+        ms, runs = float(de.group(1)), int(de.group(2))
+        out_rounds.append({"index": i, "t_start": t_start, "t_end": t_end,
+                           "ttft_ms": float(pe.group(1)) if pe else None,
+                           "decode_tok_s": runs / (ms / 1e3) if ms > 0 else None,
+                           "tpot_ms": ms / runs if runs else None, "gen_tokens": runs + 1})
+    log_path = staging / "llama_cli.log"
+    log_path.write_text(redact_text("\n\n".join(logs)))
+    return {"engine": {"name": "llama.cpp", "version": None, "commit": None}, "flags": dict(engine_args),
+            "rounds": out_rounds, "tokens": None, "log_text": "\n".join(logs),
+            "limitations": ["llama-cli fallback (no llama-bench): prompt length is approximate and TTFT is the "
+                            "reported prompt-eval time"],
+            "files": [(log_path, "engine_log")]}
+
+
+# --------------------------------------------------------------------------- any command
+
+def run_cmd(cmd: str, rounds: int, round_offset: int, timeout: int) -> dict:
+    """Run ``cmd`` once per round; it must print a JSON line containing decode_tok_s.
+
+    Optional keys in that line: ttft_ms, tpot_ms, gen_tokens, tokens (list of ids),
+    flags (dict, e.g. {"cuda_graphs": true, "attention_backend": "FLASH_ATTN"}),
+    engine ({"name", "version", "commit"}).
+    """
+    argv = shlex.split(cmd)
+    out_rounds, last = [], {}
+    for k in range(rounds):
+        t_start = time.time()
+        r = run(argv, timeout=timeout)
+        t_end = time.time()
+        if not r.ok:
+            raise OpError(f"command failed in round {k} (exit {r.returncode}): {redact_text(r.stderr[-800:])}")
+        found = None
+        for line in reversed(r.stdout.splitlines()):
+            line = line.strip()
+            if line.startswith("{") and "decode_tok_s" in line:
+                try:
+                    found = json.loads(line)
+                    break
+                except ValueError:
+                    continue
+        if found is None:
+            raise OpError(f"round {k}: no JSON line with decode_tok_s in the command's stdout")
+        last = found
+        tok_s = found.get("decode_tok_s")
+        out_rounds.append({"index": round_offset + k, "t_start": t_start, "t_end": t_end,
+                           "ttft_ms": found.get("ttft_ms"), "decode_tok_s": tok_s,
+                           "tpot_ms": found.get("tpot_ms") or (1e3 / tok_s if tok_s else None),
+                           "gen_tokens": found.get("gen_tokens")})
+    engine = last.get("engine") if isinstance(last.get("engine"), dict) else {}
+    return {"engine": {"name": engine.get("name") or "cmd", "version": engine.get("version"),
+                       "commit": engine.get("commit")},
+            "flags": last.get("flags") if isinstance(last.get("flags"), dict) else {},
+            "rounds": out_rounds, "tokens": last.get("tokens"), "log_text": "", "limitations": [], "files": []}

inferencekit/engines/clip/__init__.py

@@ -0,0 +1,13 @@
+"""Video clip engines behind the clip HTTP contract, and a ComfyUI client.
+
+  server.py         POST /v1/clips and GET /health around a pipeline object; runs in the pipeline's own
+                    virtual environment (standard library plus whatever the pipeline imports).
+  noise.py          the initial-noise recipe: drawn on the CPU from one seed, so every engine starts from
+                    the same tensors.
+  diffusers_ltx.py  LTX-2.x through Diffusers (text-to-video and image-to-video), BF16 reference mode.
+  official_ltx.py   the official LTX-2 pipelines package, when it is installed (guarded import).
+  comfyui.py        a client for a ComfyUI server (POST /prompt with a pinned API-format workflow), and the
+                    ``comfyui`` backend bench clip drives.
+
+Nothing here is imported by the rest of inferencekit unless a clip command asks for it.
+"""