inferencekit-cli 0.1.0__py3-none-any.whl

inferencekit/engines/clip/stepdump.py

@@ -0,0 +1,182 @@
+"""Per-step dumps: what a clip engine leaves of every request for ``inferencekit quality clip-steps``.
+
+THE CONTRACT (the same for every diffusion clip engine, whatever it is built on):
+
+  The engine process finds a directory in its environment variable ``CLIP_STEP_DUMP_DIR``. For every clip
+  request it serves (timed or not: there is no audit mode, and the requests to check are picked after the
+  clips are made), it writes one file before it answers:
+
+      $CLIP_STEP_DUMP_DIR/<video_sha256>.safetensors
+
+  <video_sha256> is the lowercase hex sha256 of the mp4 bytes it answers with. The file is in the
+  safetensors format and holds, for each latent stream <s> the model denoises ("video", and "audio" for an
+  audio-and-video model) and each denoising step i = 0 .. N-1 in the order the steps ran:
+
+      step.<i>.<s>.x       the latent the step started from (x_t)
+      step.<i>.<s>.out     the guided model output the sampler's update took: every guidance pass of the
+                           step combined (and rescaled, if the pipeline rescales), in the form the sampler
+                           takes it. Flow-matching models: the velocity, so that the Euler update is
+                           x_next = x + (sigma_next - sigma) * out; a sampler that holds the denoised
+                           estimate x0 writes (x - x0) / sigma
+      step.<i>.<s>.x_next  the latent the update produced (x_{t-1})
+      final.<s>            the latent the clip was decoded from
+      sigma                [N] float64: each step's noise level (flow matching: x = (1 - sigma) x0 + sigma noise)
+      sigma_next           [N] float64: the noise level each step's update went to
+
+  Latents have the layout and shape of the initial noise (video [1, C, F, H, W], audio [1, C, L, M]; not a
+  packed token sequence) and the dtype the engine holds them in. The file's metadata (safetensors' string
+  map) holds "format": "clip-steps.v1" and "request": the request body as JSON (the checker compares its prompt,
+  shape, steps, guidance scale, seed and noise seed with the request it sent). Write under a temporary name and
+  rename, so no reader sees a partial file; never delete one.
+
+  Cost on LTX-2.5 (768x512, 121 frames, 30 steps, float32 latents): 292,150,200 bytes per request, written in
+  0.26 s (median of 2 clips, the clip server's step_dump stage, RTX PRO 6000) [measured].
+
+``StepRecorder`` writes this format from torch tensors in a few lines (any engine can copy the idea);
+``DumpReader`` and ``layout_problems`` are the checker's side. Both need torch and safetensors, imported when
+used: the clip server and the harness import this module without them.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from pathlib import Path
+
+ENV = "CLIP_STEP_DUMP_DIR"
+FORMAT = "clip-steps.v1"
+SUFFIX = ".safetensors"
+FIELDS = ("x", "out", "x_next")
+REQUEST_FIELDS = ("task", "prompt", "width", "height", "num_frames", "fps", "steps", "guidance_scale", "seed",
+                  "noise_seed", "audio")
+# What the checker compares between a dump's request and the request it sent (task and audio are not replayed).
+COMPARED_FIELDS = ("prompt", "width", "height", "num_frames", "fps", "steps", "guidance_scale", "seed", "noise_seed")
+STEP_KEY = re.compile(r"^step\.(\d+)\.([a-z0-9_]+)\.(x|out|x_next)$")
+FINAL_KEY = re.compile(r"^final\.([a-z0-9_]+)$")
+
+
+def request_of(job) -> dict:
+    """The request fields a dump records (a ClipJob or a request body; noise_seed defaults to seed)."""
+    get = job.get if isinstance(job, dict) else lambda k, d=None: getattr(job, k, d)
+    out = {k: get(k) for k in REQUEST_FIELDS}
+    if out.get("noise_seed") is None:
+        out["noise_seed"] = out.get("seed")
+    if out.get("fps") is not None:
+        out["fps"] = float(out["fps"])
+    if out.get("guidance_scale") is not None:
+        out["guidance_scale"] = float(out["guidance_scale"])
+    return out
+
+
+def dump_path(directory, video_sha256: str) -> Path:
+    return Path(directory) / f"{video_sha256.lower()}{SUFFIX}"
+
+
+class StepRecorder:
+    """Collects one request's steps (CPU copies) and writes them in the contract's format."""
+
+    def __init__(self, request: dict):
+        self.request = dict(request)
+        self.tensors: dict = {}
+        self.sigma: list[float] = []
+        self.sigma_next: list[float] = []
+        self.count: dict[str, int] = {}
+        self.finals: dict = {}
+
+    def step(self, stream: str, sigma, sigma_next, x, out, x_next) -> None:
+        i = self.count.get(stream, 0)
+        self.count[stream] = i + 1
+        if i == len(self.sigma):  # the first stream to reach step i records its noise levels
+            self.sigma.append(float(sigma))
+            self.sigma_next.append(float(sigma_next))
+        for name, t in zip(FIELDS, (x, out, x_next)):
+            self.tensors[f"step.{i}.{stream}.{name}"] = t.detach().to("cpu", copy=True).contiguous()
+
+    def final(self, stream: str, latent) -> None:
+        self.finals[stream] = latent.detach().to("cpu", copy=True).contiguous()
+
+    def save(self, path) -> int:
+        """Write the file (temporary name, then rename); its size in bytes."""
+        import torch
+        from safetensors.torch import save_file
+
+        tensors = dict(self.tensors)
+        for s, n in self.count.items():
+            final = self.finals.get(s)
+            tensors[f"final.{s}"] = final if final is not None else tensors[f"step.{n - 1}.{s}.x_next"].clone()
+        tensors["sigma"] = torch.tensor(self.sigma, dtype=torch.float64)
+        tensors["sigma_next"] = torch.tensor(self.sigma_next, dtype=torch.float64)
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_name(path.name + ".partial")
+        save_file(tensors, str(tmp), metadata={"format": FORMAT, "request": json.dumps(self.request, sort_keys=True)})
+        os.replace(tmp, path)
+        return path.stat().st_size
+
+
+class DumpReader:
+    """One dump file, read lazily (tensors load on ``get``)."""
+
+    def __init__(self, path):
+        from safetensors import safe_open
+
+        self.path = Path(path)
+        self._f = safe_open(str(path), framework="pt", device="cpu")
+        self.meta = self._f.metadata() or {}
+        self.keys = set(self._f.keys())
+        try:
+            self.request = json.loads(self.meta.get("request") or "null")
+        except ValueError:
+            self.request = None
+        steps: dict[str, set[int]] = {}
+        for k in self.keys:
+            m = STEP_KEY.match(k)
+            if m:
+                steps.setdefault(m.group(2), set()).add(int(m.group(1)))
+        self.streams = sorted(steps)
+        self.n_steps = max((max(v) + 1 for v in steps.values()), default=0)
+        self._steps = steps
+
+    def get(self, key: str):
+        return self._f.get_tensor(key)
+
+    def has(self, key: str) -> bool:
+        return key in self.keys
+
+    def shape(self, key: str) -> tuple:
+        return tuple(self._f.get_slice(key).get_shape())
+
+    def dtype(self, key: str) -> str:
+        """safetensors' dtype name: F32, BF16, F16, F64..."""
+        return str(self._f.get_slice(key).get_dtype())
+
+
+def layout_problems(d: DumpReader, shapes: dict, steps: int) -> list[str]:
+    """What is wrong with a dump's layout against the request (streams, step count, keys, shapes, dtypes)."""
+    out = []
+    if d.meta.get("format") != FORMAT:
+        out.append(f"metadata format is {d.meta.get('format')!r}, not {FORMAT!r}")
+    if set(d.streams) != set(shapes):
+        out.append(f"streams {d.streams} differ from the model's latent streams {sorted(shapes)}")
+    if d.n_steps != steps:
+        out.append(f"{d.n_steps} steps dumped, {steps} asked")
+    allowed = {"sigma", "sigma_next"}
+    for s in shapes:
+        allowed.add(f"final.{s}")
+        allowed.update(f"step.{i}.{s}.{f}" for i in range(steps) for f in FIELDS)
+    missing, extra = sorted(allowed - d.keys), sorted(d.keys - allowed)
+    if missing:
+        out.append(f"{len(missing)} tensor(s) missing (e.g. {missing[0]})")
+    if extra:
+        out.append(f"{len(extra)} unexpected tensor(s) (e.g. {extra[0]})")
+    for name in ("sigma", "sigma_next"):
+        if d.has(name) and d.shape(name) != (d.n_steps,):
+            out.append(f"{name} has shape {d.shape(name)}, not ({d.n_steps},)")
+    for s, shape in shapes.items():
+        bad = [k for k in sorted(d.keys) if (k == f"final.{s}" or (k.startswith("step.") and f".{s}." in k))
+               and d.shape(k) != tuple(shape)]
+        if bad:
+            out.append(f"{len(bad)} {s} tensor(s) have another shape than the request's latent {list(shape)} "
+                       f"(e.g. {bad[0]}: {list(d.shape(bad[0]))})")
+    return out

inferencekit/engines/gates.py

@@ -0,0 +1,94 @@
+"""Hard gates: configurations whose decode numbers must not be presented as an engine's speed.
+
+A benchmark that silently ran without CUDA graphs, or on a fallback attention
+kernel, measures a misconfiguration rather than the engine. Batch-1 decode is
+dominated by per-kernel launch overhead without CUDA graphs (each token launches
+hundreds of small kernels; a graph replays them with one launch), and fallback
+attention kernels can be several times slower. Such runs stay recorded, but their
+violations go into ``limitations`` and the claim is marked invalid.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Attention implementations that engines use only when the fast path is unavailable.
+FALLBACK_ATTENTION = {"TORCH_SDPA", "XFORMERS", "FLEX_ATTENTION", "TORCH_NATIVE"}
+# Legitimate on some hardware, a fallback on others: recorded as a warning.
+SUSPECT_ATTENTION = {"TRITON_ATTN", "TRITON"}
+
+# Keyed by the name with separators removed, so "Flash Attention", "FLASH_ATTN" and
+# "flash-attn" all map to one canonical spelling.
+_ALIASES = {"FLASHATTENTION": "FLASH_ATTN", "FLASHATTN": "FLASH_ATTN", "FLASHINFER": "FLASHINFER",
+            "TRITONATTENTION": "TRITON_ATTN", "TRITONATTN": "TRITON_ATTN", "TORCHSDPA": "TORCH_SDPA",
+            "FLEXATTENTION": "FLEX_ATTENTION", "XFORMERS": "XFORMERS", "TORCHNATIVE": "TORCH_NATIVE",
+            "FA3": "FA3", "FLASHMLA": "FLASHMLA", "CUTLASSMLA": "CUTLASS_MLA"}
+
+
+def normalize_backend(name: str | None) -> str | None:
+    if not name:
+        return None
+    n = name.strip().upper().replace("ATTENTIONBACKENDENUM.", "").replace(" ", "_").replace("-", "_")
+    return _ALIASES.get(n.replace("_", ""), n)
+
+
+def _is_attention(name: str) -> bool:
+    # "Using X backend" also appears for MoE, sampling and communication; keep attention only.
+    return "ATTN" in name or "ATTENTION" in name or "MLA" in name or name in _ALIASES.values()
+
+
+def attention_backend_from_log(log_text: str) -> str | None:
+    """vLLM logs e.g. 'Using Flash Attention backend' or 'Using FLASHINFER backend'. Last mention wins."""
+    hits = re.findall(r"Using (?:AttentionBackendEnum\.)?([A-Za-z0-9_][A-Za-z0-9_ ]*?) (?:attention )?backend",
+                      log_text)
+    names = [normalize_backend(h) for h in hits]
+    names = [n for n in names if n and _is_attention(n)]
+    return names[-1] if names else None
+
+
+def check(engine: str, flags: dict, attention_backend: str | None) -> dict:
+    """Return {'valid': bool | None, 'violations': [...], 'warnings': [...], 'checked': [...]}.
+
+    valid is None when the adapter could not see enough to check (e.g. a ``cmd``
+    benchmark that reported no flags): unknown is not the same as fine.
+    """
+    violations, warnings, checked = [], [], []
+    if engine == "vllm":
+        checked.append("cuda_graphs")
+        if flags.get("enforce_eager") is True:
+            violations.append("CUDA graphs off (enforce_eager=True)")
+        mode = str(flags.get("cudagraph_mode") or "")
+        if mode.upper().endswith("NONE"):
+            violations.append(f"CUDA graphs off (cudagraph_mode={mode})")
+        sizes = flags.get("cudagraph_capture_sizes")
+        if isinstance(sizes, dict) and sizes.get("count") == 0 and flags.get("enforce_eager") is not True:
+            violations.append("CUDA graphs off (no capture sizes)")
+    elif engine == "sglang":
+        checked.append("cuda_graphs")
+        if flags.get("disable_cuda_graph") is True:
+            violations.append("CUDA graphs off (disable_cuda_graph=True)")
+    elif engine == "llamacpp":
+        checked.append("gpu_offload")
+        ngl = flags.get("n_gpu_layers")
+        if ngl == 0:
+            violations.append("no layers on the GPU (n_gpu_layers=0)")
+        elif isinstance(ngl, int) and ngl < 99:
+            warnings.append(f"n_gpu_layers={ngl}: check that every layer is offloaded")
+        backends = str(flags.get("backends") or "")
+        if backends and not re.search(r"CUDA|Vulkan|Metal|HIP|SYCL|MUSA", backends, re.IGNORECASE):
+            violations.append(f"no GPU backend in this llama.cpp build (backends={backends})")
+    elif engine == "cmd":
+        if "cuda_graphs" in flags:
+            checked.append("cuda_graphs")
+            if flags["cuda_graphs"] is False:
+                violations.append("CUDA graphs off (reported by the command)")
+    backend = normalize_backend(attention_backend or flags.get("attention_backend"))
+    if backend:
+        checked.append("attention_backend")
+        if backend in FALLBACK_ATTENTION:
+            violations.append(f"fallback attention backend ({backend})")
+        elif backend in SUSPECT_ATTENTION:
+            warnings.append(f"attention backend {backend} is a fallback on many NVIDIA GPUs: confirm it is intended")
+    valid = (not violations) if checked else None
+    return {"valid": valid, "violations": violations, "warnings": warnings, "checked": checked,
+            "attention_backend": backend}

inferencekit/engines/sglang_worker.py

@@ -0,0 +1,86 @@
+"""Batch-1 decode timing inside SGLang's own Python (offline Engine). Standard library + sglang only.
+
+Usage: python sglang_worker.py CONFIG.json
+Same protocol and timing definitions as vllm_worker.py. SGLang's radix cache reuses
+prompt prefixes, so each call again gets a fresh synthetic prompt.
+"""
+
+import json
+import os
+import random
+import sys
+import time
+
+MARK = "INFERKIT_RESULT "
+# Only these server arguments are recorded. Never dump the whole object: it can hold an API key.
+FLAG_FIELDS = ("disable_cuda_graph", "cuda_graph_max_bs", "cuda_graph_bs", "attention_backend",
+               "prefill_attention_backend", "decode_attention_backend", "sampling_backend", "quantization", "dtype",
+               "kv_cache_dtype", "enable_torch_compile", "tp_size", "disable_radix_cache", "mem_fraction_static",
+               "context_length", "speculative_algorithm")
+
+
+def prompt_ids(seed, label, n, vocab):
+    lo = 1000 if vocab > 2000 else 10
+    hi = min(vocab, 30000)
+    rng = random.Random(f"{seed}:{label}")
+    return [lo + int(rng.random() * (hi - lo)) for _ in range(n)]
+
+
+def plain(x):
+    if x is None or isinstance(x, (bool, int, float, str)):
+        return x
+    if isinstance(x, (list, tuple)):
+        return [plain(v) for v in list(x)[:256]]
+    return str(x)
+
+
+def main():
+    cfg = json.load(open(sys.argv[1]))
+    import sglang
+
+    engine_args = dict(cfg.get("engine_args", {}))
+    engine_args.setdefault("random_seed", cfg["seed"])
+    engine = sglang.Engine(model_path=cfg["model"], **engine_args)
+    try:
+        vocab = len(engine.tokenizer_manager.tokenizer)
+    except Exception:  # noqa: BLE001
+        vocab = 32000
+
+    def generate(ids, max_new):
+        params = {"temperature": 0.0, "max_new_tokens": max_new, "ignore_eos": True}
+        t0 = time.perf_counter()
+        out = engine.generate(input_ids=ids, sampling_params=params)
+        dt = time.perf_counter() - t0
+        toks = out.get("output_ids") or []
+        n_out = (out.get("meta_info") or {}).get("completion_tokens", len(toks))
+        return dt, [int(t) for t in toks], int(n_out)
+
+    n, g = cfg["prompt_len"], cfg["gen_len"]
+    _, ref_tokens, _ = generate(prompt_ids(cfg["seed"], "reference", n, vocab), g)
+    for w in range(cfg.get("warmup", 1) - 1):
+        generate(prompt_ids(cfg["seed"], f"warmup{w}", n, vocab), g)
+    rounds = []
+    for k in range(cfg["rounds"]):
+        i = cfg.get("round_offset", 0) + k
+        t_start = time.time()
+        ttft, _, _ = generate(prompt_ids(cfg["seed"], f"round{i}:first", n, vocab), 1)
+        total, _, n_out = generate(prompt_ids(cfg["seed"], f"round{i}:full", n, vocab), g)
+        t_end = time.time()
+        decode_s = total - ttft
+        ok = n_out > 1 and decode_s > 0
+        rounds.append({"index": i, "t_start": t_start, "t_end": t_end, "ttft_ms": ttft * 1e3,
+                       "total_ms": total * 1e3, "gen_tokens": n_out,
+                       "decode_tok_s": (n_out - 1) / decode_s if ok else None,
+                       "tpot_ms": decode_s * 1e3 / (n_out - 1) if ok else None})
+    args = getattr(engine, "server_args", None)
+    flags = {f: plain(getattr(args, f, None)) for f in FLAG_FIELDS if hasattr(args, f)}
+    result = {"engine": {"name": "sglang", "version": getattr(sglang, "__version__", None)}, "flags": flags,
+              "rounds": rounds, "tokens": ref_tokens,
+              "prompt": {"vocab_bound": min(vocab, 30000), "rule": "ids from random.Random('<seed>:<label>')"}}
+    sys.stdout.write(MARK + json.dumps(result) + "\n")
+    sys.stdout.flush()
+    os._exit(0)
+
+
+if __name__ == "__main__":
+    main()

inferencekit/engines/vllm_worker.py

@@ -0,0 +1,129 @@
+"""Batch-1 decode timing inside vLLM's own Python. Standard library + vllm only.
+
+Usage: python vllm_worker.py CONFIG.json
+Prints exactly one line ``INFERKIT_RESULT {...}`` on stdout; logs go to stderr.
+
+Timing per round (offline engine, no HTTP, no network):
+  ttft   = wall time of a generation with max_tokens=1 (prefill + first sample)
+  total  = wall time of a generation with max_tokens=gen_len
+  decode tok/s = (tokens - 1) / (total - ttft)
+Every call gets a fresh synthetic prompt: vLLM caches prompt prefixes, and a cached
+prompt would make TTFT look far better than a real request's.
+"""
+
+import json
+import os
+import random
+import sys
+import time
+
+MARK = "INFERKIT_RESULT "
+
+
+def prompt_ids(seed, label, n, vocab):
+    """Deterministic synthetic token ids, avoiding the low ids where special tokens live."""
+    lo = 1000 if vocab > 2000 else 10
+    hi = min(vocab, 30000)
+    rng = random.Random(f"{seed}:{label}")  # str seeds hash with SHA-512: stable everywhere
+    return [lo + int(rng.random() * (hi - lo)) for _ in range(n)]
+
+
+def get(obj, path):
+    for part in path.split("."):
+        if obj is None:
+            return None
+        obj = getattr(obj, part, None)
+    return obj
+
+
+def plain(x):
+    """Enums and config objects become strings; containers are kept small."""
+    if x is None or isinstance(x, (bool, int, float, str)):
+        return x
+    if isinstance(x, (list, tuple, set)):
+        return [plain(v) for v in list(x)[:256]]
+    return str(x)
+
+
+def effective_flags(llm):
+    """What the engine actually runs with (after its own defaults), from its config objects."""
+    eng = llm.llm_engine
+    cfg = get(eng, "vllm_config")
+    if cfg is None and hasattr(eng, "get_vllm_config"):
+        try:
+            cfg = eng.get_vllm_config()
+        except Exception:  # noqa: BLE001
+            cfg = None
+    sizes = plain(get(cfg, "compilation_config.cudagraph_capture_sizes"))
+    flags = {
+        "enforce_eager": plain(get(cfg, "model_config.enforce_eager")),
+        "dtype": plain(get(cfg, "model_config.dtype")),
+        "quantization": plain(get(cfg, "model_config.quantization")),
+        "max_model_len": plain(get(cfg, "model_config.max_model_len")),
+        "kv_cache_dtype": plain(get(cfg, "cache_config.cache_dtype")),
+        "block_size": plain(get(cfg, "cache_config.block_size")),
+        "enable_prefix_caching": plain(get(cfg, "cache_config.enable_prefix_caching")),
+        "gpu_memory_utilization": plain(get(cfg, "cache_config.gpu_memory_utilization")),
+        "tensor_parallel_size": plain(get(cfg, "parallel_config.tensor_parallel_size")),
+        "compilation_level": plain(get(cfg, "compilation_config.level")),
+        "compilation_mode": plain(get(cfg, "compilation_config.mode")),
+        "cudagraph_mode": plain(get(cfg, "compilation_config.cudagraph_mode")),
+        "speculative_decoding": get(cfg, "speculative_config") is not None,
+    }
+    if isinstance(sizes, list):
+        flags["cudagraph_capture_sizes"] = {"count": len(sizes), "min": min(sizes) if sizes else None,
+                                            "max": max(sizes) if sizes else None, "includes_1": 1 in sizes}
+    return flags
+
+
+def main():
+    cfg = json.load(open(sys.argv[1]))
+    import vllm
+    from vllm import LLM, SamplingParams
+
+    engine_args = dict(cfg.get("engine_args", {}))
+    engine_args.setdefault("seed", cfg["seed"])
+    llm = LLM(model=cfg["model"], **engine_args)
+    try:
+        vocab = len(llm.get_tokenizer())
+    except Exception:  # noqa: BLE001 - e.g. skip_tokenizer_init
+        vocab = 32000
+
+    def generate(ids, max_tokens):
+        sp = SamplingParams(temperature=0.0, max_tokens=max_tokens, ignore_eos=True)
+        t0 = time.perf_counter()
+        try:
+            outs = llm.generate([{"prompt_token_ids": ids}], sp, use_tqdm=False)
+        except TypeError:  # older vLLM: token ids as a keyword argument
+            outs = llm.generate(prompt_token_ids=[ids], sampling_params=sp, use_tqdm=False)
+        dt = time.perf_counter() - t0
+        return dt, [int(t) for t in outs[0].outputs[0].token_ids]
+
+    n, g = cfg["prompt_len"], cfg["gen_len"]
+    # Warm-up on the fixed reference prompt; its tokens feed the correctness check.
+    _, ref_tokens = generate(prompt_ids(cfg["seed"], "reference", n, vocab), g)
+    for w in range(cfg.get("warmup", 1) - 1):
+        generate(prompt_ids(cfg["seed"], f"warmup{w}", n, vocab), g)
+    rounds = []
+    for k in range(cfg["rounds"]):
+        i = cfg.get("round_offset", 0) + k
+        t_start = time.time()
+        ttft, _ = generate(prompt_ids(cfg["seed"], f"round{i}:first", n, vocab), 1)
+        total, toks = generate(prompt_ids(cfg["seed"], f"round{i}:full", n, vocab), g)
+        t_end = time.time()
+        decode_s = total - ttft
+        ok = len(toks) > 1 and decode_s > 0
+        rounds.append({"index": i, "t_start": t_start, "t_end": t_end, "ttft_ms": ttft * 1e3,
+                       "total_ms": total * 1e3, "gen_tokens": len(toks),
+                       "decode_tok_s": (len(toks) - 1) / decode_s if ok else None,
+                       "tpot_ms": decode_s * 1e3 / (len(toks) - 1) if ok else None})
+    result = {"engine": {"name": "vllm", "version": getattr(vllm, "__version__", None)},
+              "flags": effective_flags(llm), "rounds": rounds, "tokens": ref_tokens,
+              "prompt": {"vocab_bound": min(vocab, 30000), "rule": "ids from random.Random('<seed>:<label>')"}}
+    sys.stdout.write(MARK + json.dumps(result) + "\n")
+    sys.stdout.flush()
+    os._exit(0)  # skip engine teardown, which can hang on some versions; the OS frees the GPU
+
+
+if __name__ == "__main__":
+    main()