servcalc 0.1.0__py3-none-any.whl

servcalc/__init__.py

@@ -0,0 +1,15 @@
+"""servcalc: predict single-GPU LLM serving capacity from config.json."""
+
+from . import profiles
+from .calibration import Calibration, Curve
+from .estimate import EstimateParams, ServingEstimate, estimate
+from .gpus import GpuSpec
+from .modes import BatchEstimate, DoesNotFitError
+from .profiles import EngineProfile
+from .timing import Range
+from .workload import Workload
+
+__version__ = "0.1.0"
+__all__ = ["BatchEstimate", "Calibration", "Curve", "DoesNotFitError", "EngineProfile",
+           "EstimateParams", "GpuSpec", "Range", "ServingEstimate", "Workload", "estimate",
+           "profiles"]

servcalc/calibration.py

@@ -0,0 +1,94 @@
+"""Efficiency curves that turn work into time. Priors until measured."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from itertools import pairwise
+
+
+@dataclass(frozen=True)
+class Curve:
+    points: tuple[tuple[float, float, float, float], ...]   # (x, low, mid, high)
+
+    def __post_init__(self):
+        if not self.points:
+            raise ValueError("Curve needs at least one point")
+        xs = [p[0] for p in self.points]
+        if xs != sorted(xs) or len(set(xs)) != len(xs):
+            raise ValueError("Curve x values must be strictly ascending")
+        for x, lo, mid, hi in self.points:
+            if not (lo <= mid <= hi):
+                raise ValueError(f"Curve point at x={x} must satisfy low <= mid <= high")
+            if lo <= 0:
+                raise ValueError(f"Curve values must be positive, got {lo} at x={x}")
+
+    def at(self, x: float) -> tuple[float, float, float]:
+        pts = self.points
+        if x <= pts[0][0]:
+            return pts[0][1:]
+        if x >= pts[-1][0]:
+            return pts[-1][1:]
+        for (x0, *a), (x1, *b) in pairwise(pts):
+            if x0 <= x <= x1:
+                t = (x - x0) / (x1 - x0)
+                return tuple(ya + t * (yb - ya) for ya, yb in zip(a, b))
+        raise AssertionError("unreachable")
+
+
+@dataclass(frozen=True)
+class Calibration:
+    bw_util: Curve
+    mfu: Curve
+    overhead_ms: tuple[float, float, float]
+    source: str
+
+    def __post_init__(self):
+        lo, mid, hi = self.overhead_ms
+        if not (0 <= lo <= mid <= hi):
+            raise ValueError("overhead_ms must satisfy 0 <= low <= mid <= high")
+
+
+def _curve(*pts: tuple[float, float, float]) -> Curve:
+    """Build a Curve from (x, low, high) with mid at the midpoint."""
+    return Curve(tuple((x, lo, (lo + hi) / 2, hi) for x, lo, hi in pts))
+
+
+_BW_X = (1, 8, 64, 512)
+_MFU_X = (1, 64, 512, 4096)
+
+
+def _prior(bw: tuple, mfu: tuple, overhead: tuple[float, float]) -> Calibration:
+    lo, hi = overhead
+    return Calibration(
+        bw_util=_curve(*((x, a, b) for x, (a, b) in zip(_BW_X, bw))),
+        mfu=_curve(*((x, a, b) for x, (a, b) in zip(_MFU_X, mfu))),
+        overhead_ms=(lo, (lo + hi) / 2, hi),
+        source="prior",
+    )
+
+
+_VLLM_MFU = ((0.02, 0.05), (0.1, 0.2), (0.3, 0.45), (0.35, 0.5))
+
+PRIORS: dict[str, Calibration] = {
+    "vllm_bf16": _prior(((0.5, 0.7), (0.6, 0.8), (0.65, 0.85), (0.65, 0.85)), _VLLM_MFU, (1, 3)),
+    "vllm_int4": _prior(((0.35, 0.55), (0.45, 0.65), (0.5, 0.7), (0.5, 0.7)), _VLLM_MFU, (1, 3)),
+    "llamacpp_q4": _prior(((0.3, 0.5), (0.35, 0.6), (0.4, 0.65), (0.4, 0.65)),
+                          ((0.01, 0.03), (0.08, 0.15), (0.2, 0.35), (0.25, 0.4)), (2, 5)),
+    "hf_bf16": _prior(((0.4, 0.6), (0.5, 0.7), (0.5, 0.7), (0.5, 0.7)),
+                      ((0.02, 0.05), (0.1, 0.2), (0.25, 0.4), (0.3, 0.45)), (3, 8)),
+}
+# Every (engine, quant family) a factory can name resolves here. Families: bf16, int4, q4, q8.
+PRIORS["llamacpp_q8"] = PRIORS["llamacpp_q4"]
+PRIORS["llamacpp_bf16"] = PRIORS["llamacpp_q4"]
+PRIORS["llamacpp_int4"] = PRIORS["llamacpp_q4"]
+PRIORS["vllm_q4"] = PRIORS["vllm_int4"]
+PRIORS["vllm_q8"] = PRIORS["vllm_bf16"]
+PRIORS["hf_int4"] = PRIORS["vllm_int4"]
+PRIORS["hf_q4"] = PRIORS["vllm_int4"]
+PRIORS["hf_q8"] = PRIORS["hf_bf16"]
+
+
+def get(name: str) -> Calibration:
+    if name not in PRIORS:
+        raise KeyError(f"unknown calibration {name!r}; built-in: {', '.join(sorted(PRIORS))}")
+    return PRIORS[name]

servcalc/cli.py

@@ -0,0 +1,113 @@
+"""Command-line interface: `servcalc MODEL --gpu NAME ...` and `servcalc gpus`."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import json
+import sys
+
+from vramcalc import UnsupportedModelError
+from vramcalc import gpus as _vg
+
+from . import profiles
+from .estimate import estimate
+from .gpus import SPECS
+from .modes import DoesNotFitError
+from .workload import Workload
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="servcalc", description=__doc__)
+    sub = parser.add_subparsers(dest="command")
+    est = sub.add_parser("estimate", help="predict serving capacity (default when MODEL is given)")
+    est.add_argument("model", help="config.json path, model directory, or Hub id")
+    est.add_argument("--gpu", required=True, help="GPU name from `servcalc gpus`")
+    est.add_argument("--engine", default="vllm", choices=sorted(profiles.BUILTIN))
+    est.add_argument("--prompt", type=int, default=1024, help="prompt tokens per request")
+    est.add_argument("--gen", type=int, default=512, help="generated tokens per request")
+    est.add_argument("--concurrency", default="1,2,4,8,16,32,64,128,256",
+                     help="comma-separated batch sizes to evaluate")
+    est.add_argument("--quant", help="weight format (awq, gptq, fp8, gguf_q4_k_m, ...)")
+    est.add_argument("--kv-bits", type=int, help="KV cache bits (8 or 16)")
+    est.add_argument("--max-seqs", type=int, help="engine sequence cap (ollama: num_parallel)")
+    est.add_argument("--num-ctx", type=int, help="ollama num_ctx")
+    est.add_argument("--token-budget", type=int, help="scheduler tokens per step")
+    est.add_argument("--no-chunked", action="store_true", help="vllm: disable chunked prefill")
+    est.add_argument("--slo-itl", type=float, metavar="MS",
+                     help="report max users whose mean ITL is within MS")
+    est.add_argument("--json", action="store_true")
+    sub.add_parser("gpus", help="list GPUs with bandwidth and bf16 TFLOPs")
+    return parser
+
+
+def _profile(a: argparse.Namespace) -> profiles.EngineProfile:
+    """Factory defaults plus CLI overrides; quant left None is inferred from the config later."""
+    if a.num_ctx is not None and a.engine != "ollama":
+        raise ValueError("--num-ctx applies to --engine ollama only")
+    if a.no_chunked and a.engine != "vllm":
+        raise ValueError("--no-chunked applies to --engine vllm only")
+    p = profiles.BUILTIN[a.engine]()
+    if a.quant is not None:
+        p = profiles.with_quant(p, a.quant)
+    overrides = {}
+    if a.kv_bits is not None:
+        overrides["kv_bits"] = a.kv_bits
+    if a.max_seqs is not None:
+        overrides["max_seqs"] = a.max_seqs
+    if a.num_ctx is not None:
+        overrides["kv_prealloc_ctx"] = a.num_ctx
+    if a.token_budget is not None:
+        overrides["token_budget"] = a.token_budget
+    if a.no_chunked:
+        overrides.update(token_budget=None, mixes_prefill=False)
+    return dataclasses.replace(p, **overrides) if overrides else p
+
+
+def cmd_estimate(a: argparse.Namespace) -> int:
+    conc = tuple(int(x) for x in a.concurrency.split(","))
+    e = estimate(a.model, a.gpu, workload=Workload(a.prompt, a.gen, conc), profile=_profile(a))
+    if a.json:
+        d = e.to_dict()
+        if a.slo_itl is not None:
+            d["max_users"] = e.max_users(a.slo_itl)
+        print(json.dumps(d, indent=2))
+        return 0
+    print(e.table())
+    if a.slo_itl is not None:
+        print(f"\nmax users with mean ITL <= {a.slo_itl:g} ms: {e.max_users(a.slo_itl)}")
+    for w in e.warnings:
+        print(f"warning: {w}")
+    return 0
+
+
+def cmd_gpus(_: argparse.Namespace) -> int:
+    print(f"{'GPU':<20} {'VRAM GiB':>9} {'GB/s':>7} {'bf16 TFLOPs':>12}")
+    for name, vram in _vg.GPUS.items():
+        bw, tf = SPECS[name]
+        print(f"{name:<20} {vram / 2**30:>9.0f} {bw:>7.0f} {tf:>12.0f}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if argv and argv[0] not in ("gpus", "estimate") and not argv[0].startswith("-"):
+        argv.insert(0, "estimate")
+    parser = build_parser()
+    a = parser.parse_args(argv)
+    if a.command is None:
+        parser.print_help()
+        return 2
+    try:
+        return {"estimate": cmd_estimate, "gpus": cmd_gpus}[a.command](a)
+    except DoesNotFitError as exc:
+        print(f"servcalc: {exc}", file=sys.stderr)
+        return 1
+    except (ValueError, KeyError, FileNotFoundError, ImportError, UnsupportedModelError) as exc:
+        msg = exc.args[0] if exc.args else str(exc)
+        print(f"servcalc: {msg}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

servcalc/estimate.py

@@ -0,0 +1,160 @@
+"""Compose work, timing, modes and profiles into a ServingEstimate."""
+
+from __future__ import annotations
+
+import dataclasses
+from dataclasses import dataclass
+
+from vramcalc import Arch
+from vramcalc.loaders import load_arch, load_config
+
+from . import work
+from .calibration import Calibration
+from .calibration import get as get_calibration
+from .gpus import GpuSpec
+from .gpus import resolve as resolve_gpu
+from .modes import BatchEstimate, capacity, point
+from .profiles import EngineProfile, resolve_profile
+from .timing import Range
+from .workload import Workload
+
+GB = 10**9
+
+ASSUMPTIONS = (
+    "steady state; admission queueing excluded",
+    "equal request lengths (prompt_len, gen_len) for every request",
+    "prefix caching off; no speculative decoding",
+    "FCFS prefill scheduling with leftover token budget passed to the next request",
+    "total_tok_s counts decode tokens only (the first token comes from prefill)",
+)
+
+
+@dataclass(frozen=True)
+class EstimateParams:
+    arch: Arch
+    gpu: GpuSpec
+    workload: Workload
+    profile: EngineProfile
+    calibration_source: str
+
+
+def _rng(r: Range | None) -> dict | None:
+    return None if r is None else dataclasses.asdict(r)
+
+
+def _fmt(r: Range | None, digits: int = 0) -> str:
+    if r is None:
+        return "-"
+    return f"{r.mid:.{digits}f} [{r.low:.{digits}f}~{r.high:.{digits}f}]"
+
+
+@dataclass(frozen=True)
+class ServingEstimate:
+    memory_capacity_seqs: int
+    configured_capacity_seqs: int
+    saturating_batch: int | None
+    points: tuple[BatchEstimate, ...]
+    weight_bytes: int
+    weight_stream_bytes: int
+    kv_bytes_per_token: int
+    params: EstimateParams
+    assumptions: tuple[str, ...]
+    warnings: tuple[str, ...]
+
+    def max_users(self, itl_ms: float) -> int:
+        ok = [p.batch for p in self.points if p.feasible and p.itl_ms.mid <= itl_ms]
+        return max(ok) if ok else 0
+
+    def to_dict(self) -> dict:
+        points = []
+        for p in self.points:
+            d = dataclasses.asdict(p)
+            for k in ("ttft_ms", "itl_ms", "seq_tok_s", "total_tok_s"):
+                d[k] = _rng(getattr(p, k))
+            points.append(d)
+        return {
+            "memory_capacity_seqs": self.memory_capacity_seqs,
+            "configured_capacity_seqs": self.configured_capacity_seqs,
+            "saturating_batch": self.saturating_batch,
+            "weight_bytes": self.weight_bytes,
+            "weight_stream_bytes": self.weight_stream_bytes,
+            "kv_bytes_per_token": self.kv_bytes_per_token,
+            "points": points,
+            "params": {
+                "arch": dataclasses.asdict(self.params.arch),
+                "gpu": dataclasses.asdict(self.params.gpu),
+                "workload": dataclasses.asdict(self.params.workload),
+                "profile": dataclasses.asdict(self.params.profile),
+                "calibration_source": self.params.calibration_source,
+            },
+            "assumptions": list(self.assumptions),
+            "warnings": list(self.warnings),
+        }
+
+    def table(self) -> str:
+        p = self.params
+        sat = f"; saturating batch {self.saturating_batch}" if self.saturating_batch else ""
+        head = [
+            f"{p.arch.name or 'model'} on {p.gpu.name} via {p.profile.name} ({p.profile.version})",
+            (f"weights {self.weight_bytes / GB:.2f} GB resident, "
+             f"{self.weight_stream_bytes / GB:.2f} GB streamed per step; "
+             f"KV {self.kv_bytes_per_token / 1e3:.1f} KB/token"),
+            (f"capacity: memory {self.memory_capacity_seqs} seqs, "
+             f"configured {self.configured_capacity_seqs} seqs{sat}"),
+            f"calibration: {p.calibration_source} (ranges are low~high)",
+            "",
+            (f"{'N':>5} {'R':>5} {'ok':<16} {'TTFT ms':>22} {'ITL ms':>22} "
+             f"{'seq tok/s':>20} {'total tok/s':>22} bound"),
+        ]
+        rows = [
+            f"{b.batch:>5} {b.resident:>5} "
+            f"{('yes' if b.feasible else 'no:' + str(b.infeasible_reason)):<16} "
+            f"{_fmt(b.ttft_ms):>22} {_fmt(b.itl_ms, 1):>22} "
+            f"{_fmt(b.seq_tok_s, 1):>20} {_fmt(b.total_tok_s):>22} {b.bound}"
+            for b in self.points
+        ]
+        return "\n".join(head + rows)
+
+
+def _saturating(points: tuple[BatchEstimate, ...]) -> int | None:
+    prev = None
+    for p in points:
+        if prev == "memory" and p.bound == "compute":
+            return p.batch
+        if p.bound != "overhead":
+            prev = p.bound
+    return None
+
+
+def estimate(model, gpu, *, workload: Workload, profile: EngineProfile | str = "vllm",
+             calibration: Calibration | None = None,
+             weight_bytes: int | None = None) -> ServingEstimate:
+    config = None if isinstance(model, Arch) else load_config(model)
+    arch = load_arch(model)
+    gpu_spec = resolve_gpu(gpu)
+    prof, warnings = resolve_profile(profile, config)
+    cal = calibration if calibration is not None else get_calibration(prof.calibration)
+    if weight_bytes is not None and (
+            not isinstance(weight_bytes, int) or isinstance(weight_bytes, bool)
+            or weight_bytes <= 0):
+        raise ValueError(f"weight_bytes must be a positive integer, got {weight_bytes!r}")
+    wts = work.weight_bytes(arch, prof.weight_bits, weight_bytes)
+    if weight_bytes is None and prof.quant is not None:
+        warnings.append("quantized weight bytes approximate non-linear tensors as 16-bit; "
+                        "pass weight_bytes to override")
+    kv_tok = work.kv_bytes_per_token(arch, prof.kv_bits)
+    cap = capacity(gpu_spec, wts.resident, kv_tok, workload, prof)
+    points = tuple(point(n, arch, gpu_spec, workload, prof, cal, wts, kv_tok, cap)
+                   for n in workload.concurrency)
+    return ServingEstimate(
+        memory_capacity_seqs=cap.memory_capacity,
+        configured_capacity_seqs=cap.configured,
+        saturating_batch=_saturating(points),
+        points=points,
+        weight_bytes=wts.resident,
+        weight_stream_bytes=wts.stream,
+        kv_bytes_per_token=kv_tok,
+        params=EstimateParams(arch, gpu_spec, workload, prof, cal.source),
+        assumptions=ASSUMPTIONS,
+        warnings=tuple(warnings),
+    )

servcalc/gpus.py

@@ -0,0 +1,57 @@
+"""GPU specs: vramcalc's capacity table extended with bandwidth and bf16 compute."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+from vramcalc import gpus as _vg
+
+# (memory bandwidth GB/s, dense bf16 TFLOPs with fp32 accumulate). Vendor spec sheets, no sparsity.
+SPECS: dict[str, tuple[float, float]] = {
+    "RTX 3060": (360, 25), "RTX 3070": (448, 40), "RTX 3080": (760, 60), "RTX 3080 Ti": (912, 68),
+    "RTX 3090": (936, 71), "RTX 3090 Ti": (1008, 80),
+    "RTX 4060": (272, 30), "RTX 4060 Ti": (288, 44), "RTX 4070": (504, 58), "RTX 4070 Ti": (504, 80),
+    "RTX 4070 Ti Super": (672, 88), "RTX 4080": (717, 97), "RTX 4080 Super": (736, 104),
+    "RTX 4090": (1008, 165),
+    "RTX 5070": (672, 62), "RTX 5070 Ti": (896, 88), "RTX 5080": (960, 112), "RTX 5090": (1792, 210),
+    "T4": (320, 65), "V100 16GB": (900, 125), "V100 32GB": (900, 125),   # no bf16: fp16 tensor peak
+    "A10": (600, 125), "A40": (696, 150), "RTX A6000": (768, 155), "RTX 6000 Ada": (960, 364),
+    "A100 40GB": (1555, 312), "A100 80GB": (2039, 312), "A800 80GB": (2039, 312),
+    "H100 80GB": (3350, 989), "H800 80GB": (3350, 989), "H200": (4800, 989),
+    "L4": (300, 121), "L40S": (864, 362), "B200": (8000, 2250),
+}
+
+
+_NUMBER_HINT = ("gpu must be a name from the built-in table (see `servcalc gpus`) or a GpuSpec "
+                "with bandwidth and TFLOPs; a bare number ({gpu!r}) cannot be timed")
+
+
+@dataclass(frozen=True)
+class GpuSpec:
+    name: str
+    vram_bytes: int
+    bandwidth_gbs: float
+    bf16_tflops: float
+
+    def __post_init__(self):
+        for f in ("vram_bytes", "bandwidth_gbs", "bf16_tflops"):
+            v = getattr(self, f)
+            if (not isinstance(v, (int, float)) or isinstance(v, bool)
+                    or not math.isfinite(v) or v <= 0):
+                raise ValueError(f"{f} must be a positive finite number, got {v!r}")
+
+
+def resolve(gpu) -> GpuSpec:
+    if isinstance(gpu, GpuSpec):
+        return gpu
+    if not isinstance(gpu, str):
+        raise ValueError(_NUMBER_HINT.format(gpu=gpu))  # noqa: TRY004
+    try:
+        name, vram = _vg.lookup(gpu)
+    except KeyError:
+        if gpu.strip().replace(".", "", 1).isdigit():
+            raise ValueError(_NUMBER_HINT.format(gpu=gpu)) from None
+        raise
+    bw, tf = SPECS[name]
+    return GpuSpec(name, vram, float(bw), float(tf))

servcalc/modes.py

@@ -0,0 +1,170 @@
+"""Batching modes (static, continuous steady state) and capacity. Spec sections 5.4 and 7."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+from vramcalc import Arch
+
+from . import work
+from .calibration import Calibration
+from .gpus import GpuSpec
+from .profiles import EngineProfile
+from .timing import Range, StepTime, step_time
+from .work import Weights, Work
+from .workload import Workload
+
+
+class DoesNotFitError(ValueError):
+    """Weights alone exceed the engine's memory budget; offload is not modelled."""
+
+
+@dataclass(frozen=True)
+class Capacity:
+    usable: float
+    seq_kv_peak: int
+    memory_capacity: int
+    configured: int
+
+
+@dataclass(frozen=True)
+class BatchEstimate:
+    batch: int
+    resident: int
+    feasible: bool
+    infeasible_reason: str | None
+    ttft_ms: Range | None
+    itl_ms: Range
+    seq_tok_s: Range
+    total_tok_s: Range
+    bound: str
+
+
+def _roundup(x: int, block: int) -> int:
+    return x if block == 0 else math.ceil(x / block) * block
+
+
+def capacity(gpu: GpuSpec, w_res: int, kv_tok: int, workload: Workload,
+             profile: EngineProfile) -> Capacity:
+    budget = gpu.vram_bytes * (profile.memory_budget_ratio - profile.workspace_ratio)
+    usable = budget - w_res
+    if usable <= 0:
+        raise DoesNotFitError(
+            f"weights ({w_res / 1e9:.2f} GB) exceed the memory budget of {gpu.name} "
+            f"({budget / 1e9:.2f} GB usable); CPU offload is not modelled")
+    total_len = workload.prompt_len + workload.gen_len
+    if profile.kv_prealloc_ctx is not None:
+        if profile.kv_prealloc_ctx < total_len:
+            raise ValueError(
+                f"kv_prealloc_ctx={profile.kv_prealloc_ctx} is shorter than "
+                f"prompt_len + gen_len={total_len}; truncation and context shifting "
+                "are not modelled")
+        seq_kv_peak = kv_tok * profile.kv_prealloc_ctx
+    else:
+        seq_kv_peak = kv_tok * _roundup(total_len, profile.kv_block)
+    mem = int(usable // seq_kv_peak)
+    conf = mem if profile.max_seqs is None else min(mem, profile.max_seqs)
+    return Capacity(usable, seq_kv_peak, mem, conf)
+
+
+def _prefill_ctx_bytes(kv_tok: int, prompt_len: int, profile: EngineProfile) -> int:
+    if profile.kv_prealloc_ctx is not None:
+        return kv_tok * profile.kv_prealloc_ctx
+    return kv_tok * _roundup(prompt_len, profile.kv_block)
+
+
+def _decode_only(arch, n, workload, kv_tok, wts, profile) -> Work:
+    return work.decode_step(arch, n, n * workload.mean_decode_ctx, kv_tok, wts.stream,
+                            profile.kv_realloc)
+
+
+def _mean(ranges: list[Range]) -> Range:
+    k = len(ranges)
+    return Range(sum(r.low for r in ranges) / k, sum(r.mid for r in ranges) / k,
+                 sum(r.high for r in ranges) / k)
+
+
+def _sum(ranges: list[Range]) -> Range:
+    total = ranges[0]
+    for r in ranges[1:]:
+        total = total + r
+    return total
+
+
+def _finish(n, resident, reasons: list[str], ttft, itl: StepTime, cap: Capacity,
+            kv_needed: float, profile) -> BatchEstimate:
+    # reason priority: memory, max_seqs, token_budget
+    if profile.max_seqs is not None and resident > profile.max_seqs:
+        reasons.insert(0, "max_seqs")
+    if kv_needed > cap.usable:
+        reasons.insert(0, "memory")
+    reason = reasons[0] if reasons else None
+    seq = itl.ms.invert(1000.0)
+    return BatchEstimate(n, resident, reason is None, reason, ttft, itl.ms, seq, seq.scale(n),
+                         itl.bound)
+
+
+def point(n: int, arch: Arch, gpu: GpuSpec, workload: Workload, profile: EngineProfile,
+          cal: Calibration, wts: Weights, kv_tok: int, cap: Capacity) -> BatchEstimate:
+    if profile.batching == "static":
+        return _static(n, arch, gpu, workload, profile, cal, wts, kv_tok, cap)
+    return _continuous(n, arch, gpu, workload, profile, cal, wts, kv_tok, cap)
+
+
+def _static(n, arch, gpu, workload, profile, cal, wts, kv_tok, cap) -> BatchEstimate:
+    plen, d = workload.prompt_len, workload.decode_steps
+    # n independent causal prompts in one forward pass: lm_head once per sequence
+    pre = Work(float(wts.stream) + kv_tok * n * plen,
+               2 * work.p_body(arch) * n * plen + 2 * arch.embed_params() * n
+               + 2 * arch.num_layers * arch.q * n * plen * (plen + 1))
+    ttft = step_time(pre, gpu, cal, n * plen).ms
+    steps = [step_time(work.decode_step(arch, n, n * (plen + 1 + j), kv_tok, wts.stream,
+                                        profile.kv_realloc), gpu, cal, n)
+             for j in range(d)]
+    itl = StepTime(_mean([s.ms for s in steps]), steps[d // 2].bound)
+    return _finish(n, n, [], ttft, itl, cap, n * cap.seq_kv_peak, profile)
+
+
+def _continuous(n, arch, gpu, workload, profile, cal, wts, kv_tok, cap) -> BatchEstimate:
+    plen = workload.prompt_len
+    lam = n / workload.decode_steps
+    base = _decode_only(arch, n, workload, kv_tok, wts, profile)
+    s_d = step_time(base, gpu, cal, n)
+    reasons: list[str] = []
+
+    if profile.mixes_prefill:
+        budget = profile.token_budget
+        c = plen if budget is None else budget - n
+        if budget is not None and (c < 1 or n + lam * plen > budget):
+            reasons.append("token_budget")
+        if c < 1:
+            # prefill cannot be represented; report decode-only numbers
+            return _finish(n, n, reasons, None, s_d, cap, n * cap.seq_kv_peak, profile)
+        chunks = work.prefill_chunks(plen, c)
+        k = len(chunks)
+        f = min(1.0, lam * plen / c)
+        steps = [step_time(base + work.prefill_chunk(arch, clen, c0, last, kv_tok), gpu, cal,
+                           n + clen)
+                 for clen, c0, last in chunks]
+        ttft = _sum([s.ms for s in steps])
+        mixed_mean = ttft.scale(1 / k)   # representative mixed step: mean of the real chunk steps
+        itl = StepTime(Range.weighted(mixed_mean, f, s_d.ms, 1 - f),
+                       steps[0].bound if f >= 0.5 else s_d.bound)
+        m = math.ceil(lam * k)
+    else:
+        chunks = work.prefill_chunks(plen, profile.token_budget)
+        lm_head = 2 * arch.embed_params()   # read only on the chunk that emits the token
+        parts = []
+        for clen, c0, last in chunks:
+            wread = max(0, wts.stream - (0 if last else lm_head))
+            parts.append(step_time(Work(float(wread), 0.0)
+                                   + work.prefill_chunk(arch, clen, c0, last, kv_tok),
+                                   gpu, cal, clen).ms)
+        t_p = _sum(parts)
+        itl = StepTime(s_d.ms + t_p.scale(lam), s_d.bound)
+        ttft = t_p + s_d.ms.scale(0.5)
+        m = math.ceil(lam * t_p.mid / itl.ms.mid)
+
+    kv_needed = n * cap.seq_kv_peak + m * _prefill_ctx_bytes(kv_tok, plen, profile)
+    return _finish(n, n + m, reasons, ttft, itl, cap, kv_needed, profile)

servcalc/profiles.py

@@ -0,0 +1,163 @@
+"""Engine profiles: everything that differs between serving engines, as values."""
+
+from __future__ import annotations
+
+import dataclasses
+from dataclasses import dataclass
+
+from . import quant as _quant
+
+BATCHING = ("continuous", "static")
+
+
+@dataclass(frozen=True)
+class EngineProfile:
+    name: str
+    version: str
+    batching: str
+    max_seqs: int | None
+    quant: str | None
+    kv_bits: int
+    kv_block: int
+    kv_prealloc_ctx: int | None
+    memory_budget_ratio: float
+    workspace_ratio: float
+    token_budget: int | None
+    mixes_prefill: bool
+    kv_realloc: bool
+    calibration: str
+
+    def __post_init__(self):
+        if self.batching not in BATCHING:
+            raise ValueError(f"batching must be one of {BATCHING}, got {self.batching!r}")
+        _quant.bits(self.quant)   # validates the name
+        if self.kv_bits not in (8, 16):
+            raise ValueError(f"kv_bits must be 8 or 16, got {self.kv_bits!r}")
+        for f in ("max_seqs", "kv_prealloc_ctx", "token_budget"):
+            v = getattr(self, f)
+            if v is not None and (not isinstance(v, int) or isinstance(v, bool) or v <= 0):
+                raise ValueError(f"{f} must be a positive integer or None, got {v!r}")
+        if self.token_budget is not None and self.token_budget < 2:
+            raise ValueError(
+                "token_budget must be >= 2 (one decode token plus one prefill token)")
+        if not isinstance(self.kv_block, int) or self.kv_block < 0:
+            raise ValueError(f"kv_block must be a non-negative integer, got {self.kv_block!r}")
+        for f in ("memory_budget_ratio", "workspace_ratio"):
+            v = getattr(self, f)
+            if not (0 < v <= 1):
+                raise ValueError(f"{f} must be in (0, 1], got {v!r}")
+        if self.memory_budget_ratio - self.workspace_ratio <= 0:
+            raise ValueError("workspace_ratio must be smaller than memory_budget_ratio")
+        if self.batching == "static" and self.mixes_prefill:
+            raise ValueError("static batching cannot mix prefill and decode")
+
+    @property
+    def weight_bits(self) -> float:
+        return _quant.bits(self.quant)
+
+
+def _cal(engine: str, quant_name: str | None) -> str:
+    return f"{engine}_{_quant.curve_suffix(quant_name)}"
+
+
+def vllm(chunked: bool = True, **overrides) -> EngineProfile:
+    """vLLM 0.6.3. Chunked prefill with a 512-token budget that includes decode tokens."""
+    quant_name = overrides.pop("quant", None)
+    fields = {
+        "name": "vllm", "version": "0.6.3", "batching": "continuous", "max_seqs": 256,
+        "quant": quant_name, "kv_bits": 16, "kv_block": 16, "kv_prealloc_ctx": None,
+        "memory_budget_ratio": 0.90, "workspace_ratio": 0.05,
+        "token_budget": 512 if chunked else None, "mixes_prefill": chunked, "kv_realloc": False,
+        "calibration": _cal("vllm", quant_name),
+    }
+    return EngineProfile(**{**fields, **overrides})
+
+
+def ollama(num_parallel: int = 4, num_ctx: int = 2048, quant: str | None = "gguf_q4_k_m",
+           **overrides) -> EngineProfile:
+    """ollama 0.5.x (llama.cpp server). Prefill is modelled as a separate pass; confirm in 0.2."""
+    fields = {
+        "name": "ollama", "version": "0.5.x (llama.cpp server)", "batching": "continuous",
+        "max_seqs": num_parallel, "quant": quant, "kv_bits": 16, "kv_block": 0,
+        "kv_prealloc_ctx": num_ctx, "memory_budget_ratio": 1.0, "workspace_ratio": 0.15,
+        "token_budget": 512, "mixes_prefill": False, "kv_realloc": False,
+        "calibration": _cal("llamacpp", quant),
+    }
+    return EngineProfile(**{**fields, **overrides})
+
+
+def hf(**overrides) -> EngineProfile:
+    """transformers 4.46.x generate() with the model loaded explicitly in bf16; static batching."""
+    quant_name = overrides.pop("quant", None)
+    fields = {
+        "name": "hf", "version": "transformers 4.46.x, explicit bf16 load", "batching": "static",
+        "max_seqs": None, "quant": quant_name, "kv_bits": 16, "kv_block": 0,
+        "kv_prealloc_ctx": None, "memory_budget_ratio": 1.0, "workspace_ratio": 0.20,
+        "token_budget": None, "mixes_prefill": False, "kv_realloc": True,
+        "calibration": _cal("hf", quant_name),
+    }
+    return EngineProfile(**{**fields, **overrides})
+
+
+BUILTIN = {"vllm": vllm, "ollama": ollama, "hf": hf}
+
+
+def with_quant(p: EngineProfile, quant_name: str | None) -> EngineProfile:
+    """Copy of p with another weight format and the matching calibration family."""
+    engine = p.calibration.rsplit("_", 1)[0]
+    return dataclasses.replace(p, quant=quant_name, calibration=_cal(engine, quant_name))
+
+
+# HF quant_method -> {bits: quant name}
+_HF_QUANT_METHODS = {
+    "awq": {4: "awq"},
+    "gptq": {4: "gptq", 8: "gptq_8"},
+    "fp8": {8: "fp8"},
+}
+_GGUF_ENGINES = ("ollama",)
+
+
+def _config_quant(config: dict | None) -> str | None:
+    """Quant name implied by config.json's quantization_config, or None when absent."""
+    qc = (config or {}).get("quantization_config")
+    if qc is None:
+        return None
+    method = qc.get("quant_method")
+    if method is None:
+        raise ValueError("quantization_config has no quant_method; pass quant explicitly")
+    if method not in _HF_QUANT_METHODS:
+        raise ValueError(f"unsupported quant_method {method!r}; "
+                         f"supported: {', '.join(_HF_QUANT_METHODS)}")
+    by_bits = _HF_QUANT_METHODS[method]
+    bits = qc.get("bits", next(iter(by_bits)))
+    if bits not in by_bits:
+        raise ValueError(f"{method} with bits={bits!r} is not supported; "
+                         f"supported bits: {', '.join(map(str, by_bits))}")
+    return by_bits[bits]
+
+
+def resolve_profile(profile, config: dict | None) -> tuple[EngineProfile, list[str]]:
+    """Name or EngineProfile into a concrete profile.
+
+    Precedence for quant: explicit factory argument > config quantization_config > default.
+    GGUF engines keep their own format and only warn when the config names an HF quant.
+    """
+    warnings: list[str] = []
+    if isinstance(profile, EngineProfile):
+        p = profile
+    elif profile in BUILTIN:
+        p = BUILTIN[profile]()
+    else:
+        raise KeyError(f"unknown profile {profile!r}; built-in: {', '.join(BUILTIN)}")
+    inferred = _config_quant(config)
+    if inferred is not None:
+        if p.name in _GGUF_ENGINES:
+            warnings.append(
+                f"config.json declares {inferred!r} weights but the {p.name} profile models "
+                f"quant={p.quant!r}; pass weight_bytes for the GGUF file you serve")
+        elif p.quant is None:
+            p = with_quant(p, inferred)
+    if p.quant is not None:
+        warnings.append(f"quant={p.quant!r} sets storage bytes only; "
+                        f"kernel speed comes from the {p.calibration} curve")
+    return p, warnings

servcalc/quant.py

@@ -0,0 +1,32 @@
+"""Weight quantization formats: storage bits per linear weight and the calibration family."""
+
+from __future__ import annotations
+
+# name -> (bits per linear weight, calibration curve suffix)
+QUANTS: dict[str | None, tuple[float, str]] = {
+    None: (16, "bf16"),
+    "fp8": (8, "bf16"),
+    "awq": (4, "int4"),
+    "gptq": (4, "int4"),
+    "gptq_8": (8, "q8"),
+    "gguf_q8_0": (8.5, "q8"),
+    "gguf_q6_k": (6.6, "q4"),
+    "gguf_q5_k_m": (5.7, "q4"),
+    "gguf_q4_k_m": (4.9, "q4"),
+    "gguf_q4_0": (4.5, "q4"),
+}
+
+
+def _entry(name: str | None) -> tuple[float, str]:
+    if name not in QUANTS:
+        known = ", ".join(str(k) for k in QUANTS if k)
+        raise ValueError(f"unknown quant {name!r}; supported: none, {known}")
+    return QUANTS[name]
+
+
+def bits(name: str | None) -> float:
+    return _entry(name)[0]
+
+
+def curve_suffix(name: str | None) -> str:
+    return _entry(name)[1]

servcalc/timing.py

@@ -0,0 +1,61 @@
+"""Turn Work into time with efficiency curves. Spec section 6."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .calibration import Calibration
+from .gpus import GpuSpec
+from .work import Work
+
+
+@dataclass(frozen=True)
+class Range:
+    low: float
+    mid: float
+    high: float
+
+    def __post_init__(self):
+        if not (self.low <= self.mid <= self.high):
+            raise ValueError(f"Range must satisfy low <= mid <= high, got {self}")
+
+    def __add__(self, o: Range) -> Range:
+        return Range(self.low + o.low, self.mid + o.mid, self.high + o.high)
+
+    def scale(self, k: float) -> Range:
+        return Range(self.low * k, self.mid * k, self.high * k)
+
+    def invert(self, numerator: float) -> Range:
+        """numerator / self, swapping ends so that low <= high still holds."""
+        return Range(numerator / self.high, numerator / self.mid, numerator / self.low)
+
+    @staticmethod
+    def weighted(a: Range, wa: float, b: Range, wb: float) -> Range:
+        return a.scale(wa) + b.scale(wb)
+
+
+@dataclass(frozen=True)
+class StepTime:
+    ms: Range
+    bound: str   # "memory" | "compute" | "overhead", judged at mid
+
+
+def step_time(work: Work, gpu: GpuSpec, cal: Calibration, tokens: int) -> StepTime:
+    bw = cal.bw_util.at(tokens)
+    mfu = cal.mfu.at(tokens)
+    oh = cal.overhead_ms
+
+    def one(util_bw: float, util_mfu: float, overhead: float) -> tuple[float, float, float]:
+        t_mem = work.bytes / (gpu.bandwidth_gbs * 1e9 * util_bw) * 1000
+        t_comp = work.flops / (gpu.bf16_tflops * 1e12 * util_mfu) * 1000
+        return t_mem, t_comp, max(t_mem, t_comp) + overhead
+
+    # low time uses high efficiency and low overhead; high time the reverse
+    _, _, lo = one(bw[2], mfu[2], oh[0])
+    t_mem, t_comp, mid = one(bw[1], mfu[1], oh[1])
+    _, _, hi = one(bw[0], mfu[0], oh[2])
+    if oh[1] > max(t_mem, t_comp):
+        bound = "overhead"
+    else:
+        bound = "memory" if t_mem > t_comp else "compute"
+    return StepTime(Range(lo, mid, hi), bound)

servcalc/work.py

@@ -0,0 +1,84 @@
+"""Work formulas: bytes moved and FLOPs per step. Spec section 5. Engine-agnostic."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+from vramcalc import Arch
+from vramcalc.memory import kv_cache
+
+
+@dataclass(frozen=True)
+class Work:
+    bytes: float
+    flops: float
+
+    def __add__(self, other: Work) -> Work:
+        return Work(self.bytes + other.bytes, self.flops + other.flops)
+
+
+@dataclass(frozen=True)
+class Weights:
+    resident: int   # bytes held in VRAM
+    stream: int     # bytes read per step (untied input embedding is a lookup, not a scan)
+
+
+def p_lin(arch: Arch) -> int:
+    return arch.num_layers * (arch.attn_params() + arch.mlp_params())
+
+
+def p_mm(arch: Arch) -> int:
+    embed = arch.embed_params()
+    return arch.param_count() - (0 if arch.tie_word_embeddings else embed)
+
+
+def p_body(arch: Arch) -> int:
+    return p_mm(arch) - arch.embed_params()
+
+
+def weight_bytes(arch: Arch, bits: float, override: int | None = None) -> Weights:
+    lin = p_lin(arch)
+    if override is not None:
+        resident = override
+    else:
+        resident = math.ceil(lin * bits / 8) + (arch.param_count() - lin) * 2
+    embed_bytes = 0 if arch.tie_word_embeddings else arch.embed_params() * 2
+    return Weights(resident, max(0, resident - embed_bytes))
+
+
+def kv_bytes_per_token(arch: Arch, kv_bits: int) -> int:
+    return kv_cache(arch, 1, 1, 1) * kv_bits // 8
+
+
+def decode_step(arch: Arch, n: int, sum_ctx: float, kv_tok: int, w_stream: int,
+                kv_realloc: bool) -> Work:
+    kv_read = kv_tok * sum_ctx
+    bytes_ = w_stream + kv_read + kv_tok * n + (2 * kv_read if kv_realloc else 0)
+    flops = 2 * p_mm(arch) * n + 4 * arch.num_layers * arch.q * sum_ctx
+    return Work(float(bytes_), float(flops))
+
+
+def prefill_chunk(arch: Arch, c_len: int, prior_ctx: int, last: bool, kv_tok: int) -> Work:
+    """One prefill chunk without the weight read (callers add it once per step)."""
+    layers, q = arch.num_layers, arch.q
+    attn = 4 * layers * q * (c_len * prior_ctx + c_len * (c_len + 1) / 2)
+    flops = 2 * p_body(arch) * c_len + attn + (2 * arch.embed_params() if last else 0)
+    return Work(float(kv_tok * (prior_ctx + c_len)), float(flops))
+
+
+def prefill_chunks(prompt_len: int, chunk: int | None) -> list[tuple[int, int, bool]]:
+    """(chunk_len, prior_ctx, is_last) for each chunk of a prompt."""
+    c = prompt_len if chunk is None else chunk
+    k = math.ceil(prompt_len / c)
+    out = []
+    for j in range(k):
+        start = j * c
+        out.append((min(c, prompt_len - start), start, j == k - 1))
+    return out
+
+
+def prefill_total_flops(arch: Arch, prompt_len: int) -> float:
+    i = prompt_len
+    return float(2 * p_body(arch) * i + 2 * arch.embed_params()
+                 + 2 * arch.num_layers * arch.q * i * (i + 1))

servcalc/workload.py

@@ -0,0 +1,39 @@
+"""Request shape: prompt and generation lengths plus the concurrency sweep."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+def _positive_int(name: str, v) -> None:
+    if not isinstance(v, int) or isinstance(v, bool) or v <= 0:
+        raise ValueError(f"{name} must be a positive integer, got {v!r}")
+
+
+@dataclass(frozen=True)
+class Workload:
+    prompt_len: int
+    gen_len: int
+    concurrency: tuple[int, ...] = (1, 2, 4, 8, 16, 32, 64, 128, 256)
+
+    def __post_init__(self):
+        _positive_int("prompt_len", self.prompt_len)
+        if not isinstance(self.gen_len, int) or isinstance(self.gen_len, bool) or self.gen_len < 2:
+            raise ValueError(
+                "gen_len must be an integer >= 2 (prefill emits the first token), "
+                f"got {self.gen_len!r}")
+        if not self.concurrency:
+            raise ValueError("concurrency must contain at least one value")
+        for n in self.concurrency:
+            _positive_int("concurrency", n)
+        if list(self.concurrency) != sorted(set(self.concurrency)):
+            raise ValueError("concurrency must be strictly ascending")
+
+    @property
+    def decode_steps(self) -> int:
+        return self.gen_len - 1
+
+    @property
+    def mean_decode_ctx(self) -> float:
+        """Average context during decode: step j has I + 1 + j for 0 <= j < D."""
+        return self.prompt_len + 1 + (self.decode_steps - 1) / 2

servcalc-0.1.0.dist-info/METADATA

@@ -0,0 +1,185 @@
+Metadata-Version: 2.5
+Name: servcalc
+Version: 0.1.0
+Summary: Predict single-GPU LLM serving capacity (concurrency, tok/s, TTFT, ITL) from config.json with a roofline model
+License-Expression: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: vramcalc>=0.1.2
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Provides-Extra: hf
+Requires-Dist: vramcalc[hf]; extra == 'hf'
+Description-Content-Type: text/markdown
+
+# servcalc
+
+[![PyPI](https://img.shields.io/pypi/v/servcalc)](https://pypi.org/project/servcalc/) [![CI](https://github.com/sacom123/servcalc/actions/workflows/ci.yml/badge.svg)](https://github.com/sacom123/servcalc/actions/workflows/ci.yml) [![Python](https://img.shields.io/pypi/pyversions/servcalc)](https://pypi.org/project/servcalc/) [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+[English](#english) | [한국어](#한국어)
+
+---
+
+## English
+
+Predict how many concurrent users one GPU can serve for an LLM, and at what speed, before you rent the GPU. servcalc reads the Hugging Face `config.json`, a GPU name and a serving engine (vLLM, ollama, or plain `transformers`), and returns per batch size: resident sequences, time to first token (TTFT), inter-token latency (ITL), tokens per second, and whether that batch size is feasible on the chosen engine. It is the next question after [vramcalc](https://github.com/sacom123/vramcalc): vramcalc answers "does it fit", servcalc answers "how fast, for how many".
+
+```python
+from servcalc import Workload, estimate
+
+e = estimate("meta-llama/Llama-3.1-8B", "RTX 3090", workload=Workload(prompt_len=1024, gen_len=512))
+print(e.table())
+print(e.max_users(itl_ms=50))          # largest feasible batch whose mean ITL is within 50 ms
+for p in e.points:
+    print(p.batch, p.feasible, p.itl_ms.mid, p.total_tok_s.mid)
+```
+
+```
+$ servcalc meta-llama/Llama-3.1-8B --gpu "RTX 3090" --concurrency 1,4,16,64 --slo-itl 50
+Llama-3.1-8B on RTX 3090 via vllm (0.6.3)
+weights 16.06 GB resident, 15.01 GB streamed per step; KV 131.1 KB/token
+capacity: memory 29 seqs, configured 29 seqs; saturating batch 16
+calibration: prior (ranges are low~high)
+
+    N     R ok                              TTFT ms                 ITL ms            seq tok/s            total tok/s bound
+    1     2 yes                       579 [482~724]       29.7 [24.7~36.2]     33.7 [27.6~40.5]             34 [28~40] memory
+    4     5 yes                       600 [493~768]       30.8 [25.8~43.5]     32.5 [23.0~38.8]           130 [92~155] memory
+   16    17 yes                       638 [522~821]      68.6 [50.8~105.2]      14.6 [9.5~19.7]          233 [152~315] compute
+   64    65 no:memory                 747 [612~956]    139.9 [109.6~194.3]        7.1 [5.1~9.1]          457 [329~584] compute
+
+max users with mean ITL <= 50 ms: 4
+```
+
+`N` is the number of sequences in decode, `R` the sequences resident in memory including the ones being prefilled. `ok` names the first limit hit: `memory`, `max_seqs`, or `token_budget`. Every number is a `[low~high]` range around a mid value.
+
+### Install
+
+| Command | What you get |
+|---|---|
+| `pip install servcalc` | Estimator and CLI. Depends only on `vramcalc`. Takes a local `config.json` or a model directory |
+| `pip install "servcalc[hf]"` | Hugging Face Hub repo ids as input (`huggingface_hub`) |
+
+Offline or on-premises: pass a model directory. With `HF_HUB_OFFLINE=1`, Hub ids resolve from the local cache only.
+
+### Why
+
+Serving questions arrive in the same breath as memory questions: "Will an 8B model on a 3090 handle twenty users at a readable speed?" The usual answer is to rent the GPU, start vLLM and run a load test. That works, but it costs an afternoon per configuration, and the result says nothing about why the number is what it is. servcalc writes the roofline model down explicitly. Decode steps are bounded by memory bandwidth (weights and KV cache read once per step), prefill is bounded by compute, and an engine's scheduler decides how the two share a step. Each term is a formula you can read in `docs/design.md`, so when a measurement disagrees you know which assumption to fix.
+
+### Scope
+
+- Models: dense decoder-only models that vramcalc parses (`llama`, `mistral`, `qwen2`, `gemma`, `gemma2`). GQA, tied embeddings and QKV bias are read from the config.
+- GPUs: 34 cards from RTX 3060 to B200, with memory bandwidth and dense bf16 TFLOPs from vendor spec sheets. Pass a `GpuSpec` for anything else.
+- Engines, as value-only profiles: `vllm` 0.6.3 (continuous batching, chunked prefill with a 512-token budget, paged KV in blocks of 16, `max_num_seqs` 256), `ollama` 0.5.x (llama.cpp server, GGUF Q4_K_M by default, `num_parallel` 4, `num_ctx` 2048 preallocated per slot), `hf` (`transformers` `generate()` in bf16, static batching, KV cache reallocated every step).
+- Workload: equal prompt and generation lengths for every request, a list of batch sizes to sweep.
+
+Out of scope for 0.1: MoE, MLA and sliding-window attention, speculative decoding, prefix caching, multi-GPU (tensor or pipeline parallel), CPU offload, admission queueing, and request length distributions. These are all deliberate: each one is a separate term or a separate model, and 0.1 keeps the closed-form core small enough to validate.
+
+### How it is computed
+
+**Work.** For a decode step with `N` sequences and total context `sum_ctx`, bytes moved are `W_stream + KV_tok x sum_ctx + KV_tok x N`, where `W_stream` is the weight bytes read per step (input embedding rows are a lookup and excluded) and `KV_tok` the KV bytes per token. FLOPs are `2 x P_mm x N + 4 x L x q x sum_ctx`. A prefill chunk of `C` tokens with `c0` tokens already cached costs `2 x P_body x C + 4 x L x q x (C x c0 + C(C+1)/2)` FLOPs and writes `KV_tok x C` bytes. Summing chunks reproduces the whole-prompt cost exactly, whatever the chunk size.
+
+**Time.** `step_ms = max(bytes / (bandwidth x bw_util), flops / (tflops x mfu)) + overhead`. `bw_util` and `mfu` are efficiency curves over the number of tokens in the step; `overhead` is the per-step scheduler and launch cost. Each curve has a low, mid and high value, which is where the ranges come from. The step is labelled `memory`, `compute` or `overhead` by its largest term at mid, and `saturating batch` is the first batch where decode flips from memory- to compute-bound.
+
+**Modes.** Static batching (hf) runs one prefill for the whole batch, then `gen_len - 1` decode steps whose context grows each step; ITL is their mean. Continuous batching (vllm, ollama) is solved in steady state: with `N` sequences each producing `gen_len - 1` tokens, requests arrive at `N / (gen_len - 1)` per step. When the engine mixes prefill into decode steps (vLLM chunked prefill), the prompt is split into `k = ceil(prompt_len / (budget - N))` chunks and TTFT is the sum of those mixed steps; ITL is the frequency-weighted mean of mixed and decode-only steps. When prefill runs as its own pass (llama.cpp), ITL is `s_decode + arrival_rate x t_prefill` and TTFT is `t_prefill + s_decode / 2`.
+
+**Profiles and capacity.** A profile is a dataclass of values: batching mode, sequence cap, quantization, KV bits and block size, memory budget and workspace ratio, token budget, whether prefill is mixed, whether KV is reallocated. Capacity is `(VRAM x budget - weights - VRAM x workspace) / KV per sequence at peak length`, rounded to KV blocks or to `num_ctx` for ollama. A batch is infeasible when its resident sequences exceed memory, the sequence cap, or the token budget, in that order.
+
+### Accuracy
+
+**All numbers in 0.1 are prior ranges.** The efficiency curves are informed guesses from published vLLM and llama.cpp benchmarks, not measurements, and `calibration: prior` in the output says so. servcalc does not claim an error figure yet. 0.2 will run vLLM's `benchmark_serving.py` and llama.cpp's server benchmark on rented GPUs, fit the curves per engine and quantization, and report the residual the same way vramcalc reports its 19-run table. Until then, read the ranges as "the model is somewhere in here if the formulas hold", and use the `bound` column and `saturating batch` as the qualitative result.
+
+### Limits
+
+- The steady-state solution is a fluid approximation: it assumes arrivals spread evenly across steps. Real schedulers admit requests in bursts, which widens TTFT.
+- Quantized weight bytes count non-linear tensors (embeddings, norms) as 16-bit. Pass `weight_bytes=` with the real checkpoint size if you have it.
+- `bf16_tflops` for T4 and V100 is the fp16 tensor-core peak since those cards have no bf16.
+- Whether ollama's llama.cpp build mixes prefill into decode steps depends on its version and flags. 0.1 models it as a separate pass; 0.2 will confirm.
+
+### License
+
+MIT
+
+---
+
+## 한국어
+
+GPU를 빌리기 전에, 하나의 GPU가 LLM 사용자를 동시에 몇 명까지 어느 속도로 서빙할 수 있는지 예측하는 라이브러리입니다. Hugging Face `config.json`, GPU 이름, 서빙 엔진(vLLM, ollama, 일반 `transformers`)을 입력받아 배치 크기별로 상주 시퀀스 수, 첫 토큰 시간(TTFT), 토큰 간 지연(ITL), 초당 토큰 수, 그리고 해당 배치가 선택한 엔진에서 실행 가능한지를 반환합니다. [vramcalc](https://github.com/sacom123/vramcalc)의 다음 질문에 해당합니다. vramcalc는 "들어가는가"에, servcalc는 "얼마나 빠르게, 몇 명에게"에 답합니다.
+
+```python
+from servcalc import Workload, estimate
+
+e = estimate("meta-llama/Llama-3.1-8B", "RTX 3090", workload=Workload(prompt_len=1024, gen_len=512))
+print(e.table())
+print(e.max_users(itl_ms=50))          # 평균 ITL이 50 ms 이내인 가장 큰 실행 가능 배치
+for p in e.points:
+    print(p.batch, p.feasible, p.itl_ms.mid, p.total_tok_s.mid)
+```
+
+```
+$ servcalc meta-llama/Llama-3.1-8B --gpu "RTX 3090" --concurrency 1,4,16,64 --slo-itl 50
+Llama-3.1-8B on RTX 3090 via vllm (0.6.3)
+weights 16.06 GB resident, 15.01 GB streamed per step; KV 131.1 KB/token
+capacity: memory 29 seqs, configured 29 seqs; saturating batch 16
+calibration: prior (ranges are low~high)
+
+    N     R ok                              TTFT ms                 ITL ms            seq tok/s            total tok/s bound
+    1     2 yes                       579 [482~724]       29.7 [24.7~36.2]     33.7 [27.6~40.5]             34 [28~40] memory
+    4     5 yes                       600 [493~768]       30.8 [25.8~43.5]     32.5 [23.0~38.8]           130 [92~155] memory
+   16    17 yes                       638 [522~821]      68.6 [50.8~105.2]      14.6 [9.5~19.7]          233 [152~315] compute
+   64    65 no:memory                 747 [612~956]    139.9 [109.6~194.3]        7.1 [5.1~9.1]          457 [329~584] compute
+
+max users with mean ITL <= 50 ms: 4
+```
+
+`N`은 decode 중인 시퀀스 수, `R`은 prefill 중인 것까지 포함해 메모리에 상주하는 시퀀스 수입니다. `ok`는 처음 걸리는 한계를 표시합니다(`memory`, `max_seqs`, `token_budget`). 모든 수치는 중앙값과 `[low~high]` 범위로 제공됩니다.
+
+### 설치
+
+| 명령 | 제공 내용 |
+|---|---|
+| `pip install servcalc` | 추정기와 CLI. 의존성은 `vramcalc` 하나입니다. 로컬 `config.json` 또는 모델 디렉터리를 입력받습니다 |
+| `pip install "servcalc[hf]"` | Hugging Face Hub 저장소 id 입력(`huggingface_hub`) |
+
+오프라인 또는 온프레미스 환경에서는 모델 디렉터리를 전달하면 됩니다. `HF_HUB_OFFLINE=1`을 설정하면 Hub id는 로컬 캐시에서만 해석됩니다.
+
+### 만든 이유
+
+서빙에 관한 질문은 메모리 질문과 함께 옵니다. "3090에서 8B 모델로 사용자 20명을 읽을 만한 속도로 받을 수 있는가"가 대표적입니다. 통상적인 답은 GPU를 빌려 vLLM을 올리고 부하 시험을 돌리는 것입니다. 유효한 방법이지만 구성 하나에 반나절이 들고, 결과 수치가 왜 그 값인지는 알려 주지 않습니다. servcalc는 roofline 모델을 명시적으로 기술합니다. decode step은 메모리 대역폭(매 step 가중치와 KV 캐시를 한 번 읽음)에, prefill은 연산량에 묶이며, 엔진의 스케줄러가 둘을 한 step에서 어떻게 섞는지를 결정합니다. 각 항은 `docs/design.md`에 식으로 적혀 있으므로, 실측과 다를 때 어느 가정을 고쳐야 하는지 알 수 있습니다.
+
+### 지원 범위
+
+- 모델: vramcalc가 해석하는 dense decoder-only 모델(`llama`, `mistral`, `qwen2`, `gemma`, `gemma2`). GQA, tied embedding, QKV bias는 config에서 읽습니다.
+- GPU: RTX 3060부터 B200까지 34종. 메모리 대역폭과 dense bf16 TFLOPs는 제조사 사양표 기준입니다. 그 외 장비는 `GpuSpec`으로 전달합니다.
+- 엔진(값으로만 구성된 프로필): `vllm` 0.6.3(continuous batching, 512 토큰 예산의 chunked prefill, 16 토큰 블록 단위 paged KV, `max_num_seqs` 256), `ollama` 0.5.x(llama.cpp server, 기본 GGUF Q4_K_M, `num_parallel` 4, 슬롯당 `num_ctx` 2048 선할당), `hf`(`transformers` `generate()` bf16, static batching, 매 step KV 캐시 재할당).
+- 워크로드: 모든 요청의 prompt 길이와 생성 길이가 같다고 가정하며, 배치 크기 목록을 순회합니다.
+
+0.1에서 제외한 항목: MoE, MLA, sliding-window attention, speculative decoding, prefix caching, 다중 GPU(tensor 또는 pipeline parallel), CPU offload, 대기열, 요청 길이 분포. 각각 별도의 항 또는 별도의 모델이 필요하므로, 0.1은 검증 가능한 크기의 닫힌 식 핵심만 담았습니다.
+
+### 계산 구조
+
+**작업량.** 시퀀스 `N`개, 총 컨텍스트 `sum_ctx`인 decode step의 이동 바이트는 `W_stream + KV_tok x sum_ctx + KV_tok x N`입니다. `W_stream`은 step마다 읽는 가중치 바이트(입력 embedding은 조회이므로 제외), `KV_tok`은 토큰당 KV 바이트입니다. FLOPs는 `2 x P_mm x N + 4 x L x q x sum_ctx`입니다. 이미 `c0` 토큰이 캐시된 상태에서 `C` 토큰을 prefill하는 조각은 `2 x P_body x C + 4 x L x q x (C x c0 + C(C+1)/2)` FLOPs를 소비하고 `KV_tok x C` 바이트를 기록합니다. 조각의 합은 조각 크기와 무관하게 전체 prompt 비용과 정확히 일치합니다.
+
+**시간.** `step_ms = max(bytes / (bandwidth x bw_util), flops / (tflops x mfu)) + overhead`입니다. `bw_util`과 `mfu`는 step 내 토큰 수에 대한 효율 곡선이고, `overhead`는 step당 스케줄러와 커널 실행 비용입니다. 각 곡선이 low, mid, high 값을 가지므로 결과가 범위로 나옵니다. step은 mid 기준으로 가장 큰 항에 따라 `memory`, `compute`, `overhead`로 표시되고, `saturating batch`는 decode가 memory-bound에서 compute-bound로 바뀌는 첫 배치입니다.
+
+**모드.** static batching(hf)은 배치 전체를 한 번 prefill한 뒤 컨텍스트가 매 step 늘어나는 `gen_len - 1`회의 decode step을 수행하며, ITL은 그 평균입니다. continuous batching(vllm, ollama)은 정상 상태로 풉니다. 시퀀스 `N`개가 각각 `gen_len - 1` 토큰을 생성하므로 요청은 step당 `N / (gen_len - 1)`개 도착합니다. 엔진이 prefill을 decode step에 섞는 경우(vLLM chunked prefill) prompt는 `k = ceil(prompt_len / (budget - N))`개 조각으로 나뉘고 TTFT는 그 혼합 step들의 합, ITL은 혼합 step과 decode 전용 step의 빈도 가중 평균입니다. prefill이 별도 pass로 실행되는 경우(llama.cpp) ITL은 `s_decode + arrival_rate x t_prefill`, TTFT는 `t_prefill + s_decode / 2`입니다.
+
+**프로필과 용량.** 프로필은 값만 담은 dataclass입니다. batching 방식, 시퀀스 상한, 양자화, KV 비트와 블록 크기, 메모리 예산과 workspace 비율, 토큰 예산, prefill 혼합 여부, KV 재할당 여부가 들어갑니다. 용량은 `(VRAM x budget - weights - VRAM x workspace) / 최대 길이 시 시퀀스당 KV`이며, KV 블록 또는 ollama의 `num_ctx` 단위로 올림합니다. 상주 시퀀스가 메모리, 시퀀스 상한, 토큰 예산을 이 순서로 초과하면 해당 배치는 실행 불가로 표시됩니다.
+
+### 정확도
+
+**0.1의 모든 수치는 실측 전 사전 범위입니다.** 효율 곡선은 공개된 vLLM과 llama.cpp 벤치마크를 참고한 추정값이며 측정값이 아닙니다. 출력의 `calibration: prior`가 이를 표시합니다. servcalc는 아직 오차 수치를 주장하지 않습니다. 0.2에서는 임대 GPU에서 vLLM의 `benchmark_serving.py`와 llama.cpp server 벤치마크를 실행해 엔진과 양자화별로 곡선을 맞추고, vramcalc의 19회 실측 표와 같은 방식으로 잔차를 보고할 예정입니다. 그때까지 범위는 "식이 맞다면 이 안에 있다"로 읽고, `bound` 열과 `saturating batch`를 정성적 결과로 활용하시기 바랍니다.
+
+### 한계
+
+- 정상 상태 해는 유체 근사입니다. 요청 도착이 step에 균등하게 퍼진다고 가정하므로, 실제 스케줄러의 묶음 단위 admission은 TTFT를 더 넓게 만듭니다.
+- 양자화 가중치 바이트는 비선형 텐서(embedding, norm)를 16비트로 계산합니다. 실제 체크포인트 크기를 알고 있다면 `weight_bytes=`로 전달하시기 바랍니다.
+- T4와 V100은 bf16이 없으므로 `bf16_tflops`에 fp16 tensor core 피크를 사용합니다.
+- ollama의 llama.cpp 빌드가 prefill을 decode step에 섞는지는 버전과 옵션에 따라 다릅니다. 0.1은 별도 pass로 모델링하며 0.2에서 확인합니다.
+
+### 라이선스
+
+MIT

servcalc-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+servcalc/__init__.py,sha256=hrZMCNNv7TK11KqQeIXPu_eNBm-ptMlEYdvBY9Rn-ww,601
+servcalc/calibration.py,sha256=WlGjYudarBE-K0vyaeNUzfjJHLqbzP-6XsLOfmdzuJw,3473
+servcalc/cli.py,sha256=Zn4cBHAHICwwsW-gpQ-9VdNbnAT7yvwF2Jo6i3u72Fk,4707
+servcalc/estimate.py,sha256=XcDmR6pgdnjpaGlHDeIOs0URlGlRaNNg6dvIyx8kh1w,6207
+servcalc/gpus.py,sha256=hjq5LON16LVqD7nPAe1TSIBlGmyOhgpGAvlJqmqFu0E,2358
+servcalc/modes.py,sha256=AmDM44ueVMq0-TTPb9qDUzWI3VH-kDK7RE5vjVVmId8,6799
+servcalc/profiles.py,sha256=if0cHU4XHFzHo-jy09_FDLxkfU4st6AUjWSVdB9ZMyc,6785
+servcalc/quant.py,sha256=JOwO68FIj5doFVBKFL8z0M8NN4Q1OGbcJvGB_FMjKEQ,893
+servcalc/timing.py,sha256=quyGRAIYjmLCXCJVLuWiZfqs9pyjvCUZDkKASLkPn3Q,2027
+servcalc/work.py,sha256=jnrKiIKp3r9pjw1n1KhkpL9eAuC5B3R6Vh7TqJt-Sak,2804
+servcalc/workload.py,sha256=u-MCiL4NgzqZddgP7QdiTWfok4Ko9eD6C-AMy-l4H0Y,1429
+servcalc-0.1.0.dist-info/METADATA,sha256=Z5hF51l0VufucCpAlPIVjzi5I5Qltv4PsEUeLBGSvt0,17765
+servcalc-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+servcalc-0.1.0.dist-info/entry_points.txt,sha256=1UN_Okjyezn5eLLvJ7YCQvZLqBREKYBXe5Pd8Bh6aQ4,47
+servcalc-0.1.0.dist-info/licenses/LICENSE,sha256=sZhDRicBLDYEeQNMKQ5o6sRsujSsfddgm_NG8u0a5iY,1068
+servcalc-0.1.0.dist-info/RECORD,,

servcalc-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

servcalc-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+servcalc = servcalc.cli:main

servcalc-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Seonho Hong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.