lmcache-cli 0.4.5.dev0__py3-none-any.whl

lmcache/cli/commands/trace/stats.py

@@ -0,0 +1,289 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Per-qualname latency statistics for trace replay.
+
+The replay driver times every dispatched call.  Timings feed into this
+collector, which computes count + mean + percentiles (p50, p90, p99)
+per qualname.
+
+The shape is deliberately simpler than
+:class:`lmcache.cli.commands.bench.engine_bench.stats.StatsCollector`
+— that one is tailored to OpenAI-style streaming inference (TTFT,
+decode speed, etc.), which is not applicable to in-process storage
+replay.  Sharing the computation code between the two would push a
+small helper deep into the bench module; keeping this collector local
+keeps the storage-replay code self-contained and easy to evolve.
+"""
+
+# Future
+from __future__ import annotations
+
+# Standard
+from dataclasses import dataclass, field
+import csv
+import json
+import statistics
+import threading
+
+# First Party
+from lmcache.logging import init_logger
+
+logger = init_logger(__name__)
+
+
+@dataclass
+class OpStats:
+    """Aggregate timing stats for one qualname.
+
+    Attributes:
+        qualname: The qualname these stats cover.
+        count: Number of successful replays for this qualname.
+        error_count: Number of replays that raised.
+        total_s: Total wall time spent replaying this qualname.
+        mean_ms: Mean per-call latency in milliseconds.
+        p50_ms: 50th-percentile latency in milliseconds.
+        p90_ms: 90th-percentile latency in milliseconds.
+        p99_ms: 99th-percentile latency in milliseconds.
+        min_ms: Minimum observed latency in milliseconds.
+        max_ms: Maximum observed latency in milliseconds.
+    """
+
+    qualname: str
+    count: int
+    error_count: int
+    total_s: float
+    mean_ms: float
+    p50_ms: float
+    p90_ms: float
+    p99_ms: float
+    min_ms: float
+    max_ms: float
+
+
+@dataclass
+class _Bucket:
+    """Internal per-qualname sample bucket."""
+
+    latencies_ms: list[float] = field(default_factory=list)
+    errors: int = 0
+
+
+def _percentile(sorted_values: list[float], pct: float) -> float:
+    """Return the *pct*-th percentile from an already-sorted list.
+
+    Uses nearest-rank with no interpolation: for N samples, the
+    p-th percentile is the sample at index ``ceil(p/100 * N) - 1``.
+    Returns 0.0 for empty input.
+
+    Args:
+        sorted_values: Ascending-sorted list of values.
+        pct: Percentile in ``[0, 100]``.
+
+    Returns:
+        The percentile value, or 0.0 for empty input.
+    """
+    if not sorted_values:
+        return 0.0
+    if pct <= 0:
+        return sorted_values[0]
+    if pct >= 100:
+        return sorted_values[-1]
+    # bisect_left gives the insertion index; the nearest-rank formula
+    # maps p to ceil(p/100 * N) which equals floor((p/100 * N - eps) + 1).
+    n = len(sorted_values)
+    idx = max(0, min(n - 1, int((pct / 100.0) * n)))
+    return sorted_values[idx]
+
+
+class ReplayStatsCollector:
+    """Thread-safe per-qualname latency collector.
+
+    The replay driver is single-threaded at the dispatcher boundary,
+    but the underlying StorageManager performs async work on helper
+    threads whose timings may eventually feed back here.  A lock keeps
+    concurrent ``record()`` calls safe.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._buckets: dict[str, _Bucket] = {}
+        self._wall_start_s: float | None = None
+        self._wall_end_s: float | None = None
+
+    def mark_start(self, wall_time_s: float) -> None:
+        """Record the wall-clock time when replay began.
+
+        Called once before the first ``record()``; replay-duration
+        metrics are derived from start/end marks.
+
+        Args:
+            wall_time_s: ``time.time()`` at replay start.
+        """
+        with self._lock:
+            self._wall_start_s = wall_time_s
+
+    def mark_end(self, wall_time_s: float) -> None:
+        """Record the wall-clock time when replay finished.
+
+        Args:
+            wall_time_s: ``time.time()`` at replay end.
+        """
+        with self._lock:
+            self._wall_end_s = wall_time_s
+
+    def record(self, qualname: str, latency_s: float, failed: bool = False) -> None:
+        """Record one replayed call.
+
+        Args:
+            qualname: Qualified name of the replayed function.
+            latency_s: Elapsed seconds for the call.
+            failed: ``True`` if the call raised.  Failed calls still
+                contribute to the count but do not add a latency
+                sample — the raising path's timing is not comparable
+                to successful calls.
+        """
+        with self._lock:
+            bucket = self._buckets.get(qualname)
+            if bucket is None:
+                bucket = _Bucket()
+                self._buckets[qualname] = bucket
+            if failed:
+                bucket.errors += 1
+                return
+            # Append is O(1); the one-shot sort in :meth:`summary` is
+            # O(N log N), which beats the O(N) shift from keeping the
+            # list sorted on insert.  For large traces (>1M records
+            # per qualname) the driver should sample or switch to an
+            # approximation; for now, exact percentiles suffice.
+            bucket.latencies_ms.append(latency_s * 1000.0)
+
+    def total_duration_s(self) -> float:
+        """Return replay wall-clock duration in seconds.
+
+        Returns:
+            ``mark_end - mark_start`` if both were set, else 0.0.
+        """
+        with self._lock:
+            if self._wall_start_s is None or self._wall_end_s is None:
+                return 0.0
+            return max(0.0, self._wall_end_s - self._wall_start_s)
+
+    def summary(self) -> dict[str, OpStats]:
+        """Return a per-qualname :class:`OpStats` snapshot.
+
+        Returns:
+            A dict keyed by qualname.  A qualname with only errors
+            still appears, with zero latency stats.
+        """
+        with self._lock:
+            result: dict[str, OpStats] = {}
+            for qualname, bucket in self._buckets.items():
+                # Sort once per summary call — ``record`` keeps the
+                # list unsorted (O(1) append) so the total work is
+                # O(N log N) per summary rather than O(N) per insert.
+                lats = sorted(bucket.latencies_ms)
+                if lats:
+                    mean = statistics.fmean(lats)
+                    total_s = sum(lats) / 1000.0
+                    p50 = _percentile(lats, 50)
+                    p90 = _percentile(lats, 90)
+                    p99 = _percentile(lats, 99)
+                    lo, hi = lats[0], lats[-1]
+                else:
+                    mean = total_s = p50 = p90 = p99 = lo = hi = 0.0
+                result[qualname] = OpStats(
+                    qualname=qualname,
+                    count=len(lats),
+                    error_count=bucket.errors,
+                    total_s=total_s,
+                    mean_ms=mean,
+                    p50_ms=p50,
+                    p90_ms=p90,
+                    p99_ms=p99,
+                    min_ms=lo,
+                    max_ms=hi,
+                )
+            return result
+
+    def export_csv(self, path: str) -> None:
+        """Write per-qualname stats to a CSV file.
+
+        Columns: ``qualname, count, errors, mean_ms, p50_ms, p90_ms,
+        p99_ms, min_ms, max_ms``.  One row per qualname.
+
+        Args:
+            path: File path to write.  Overwritten if it exists.
+        """
+        summary = self.summary()
+        with open(path, "w", newline="") as f:
+            w = csv.writer(f)
+            w.writerow(
+                [
+                    "qualname",
+                    "count",
+                    "errors",
+                    "mean_ms",
+                    "p50_ms",
+                    "p90_ms",
+                    "p99_ms",
+                    "min_ms",
+                    "max_ms",
+                ]
+            )
+            for qn in sorted(summary):
+                s = summary[qn]
+                w.writerow(
+                    [
+                        s.qualname,
+                        s.count,
+                        s.error_count,
+                        f"{s.mean_ms:.6f}",
+                        f"{s.p50_ms:.6f}",
+                        f"{s.p90_ms:.6f}",
+                        f"{s.p99_ms:.6f}",
+                        f"{s.min_ms:.6f}",
+                        f"{s.max_ms:.6f}",
+                    ]
+                )
+
+    def export_json(self, path: str) -> None:
+        """Write per-qualname stats + replay duration to a JSON file.
+
+        Schema::
+
+            {
+              "duration_s": <float>,
+              "ops": {
+                "<qualname>": {
+                  "count": int, "errors": int,
+                  "mean_ms": float, "p50_ms": float,
+                  "p90_ms": float, "p99_ms": float,
+                  "min_ms": float, "max_ms": float
+                },
+                ...
+              }
+            }
+
+        Args:
+            path: File path to write.  Overwritten if it exists.
+        """
+        summary = self.summary()
+        payload = {
+            "duration_s": self.total_duration_s(),
+            "ops": {
+                qn: {
+                    "count": s.count,
+                    "errors": s.error_count,
+                    "mean_ms": s.mean_ms,
+                    "p50_ms": s.p50_ms,
+                    "p90_ms": s.p90_ms,
+                    "p99_ms": s.p99_ms,
+                    "min_ms": s.min_ms,
+                    "max_ms": s.max_ms,
+                }
+                for qn, s in summary.items()
+            },
+        }
+        with open(path, "w") as f:
+            json.dump(payload, f, indent=2, sort_keys=True)
+            f.write("\n")

lmcache/cli/documents/lmcache.txt

@@ -0,0 +1,11 @@
+LMCache is a high-performance key–value (KV) cache management system designed to accelerate large language model (LLM) inference by efficiently storing, transferring, and reusing intermediate attention states. As modern LLM serving increasingly becomes bottlenecked by memory bandwidth, redundant computation, and cross-device communication, LMCache provides a system-level solution that decouples KV cache storage from the model execution pipeline and enables scalable, low-latency reuse across requests, processes, and even distributed nodes.
+
+At its core, LMCache targets one of the most expensive components of autoregressive inference: the KV cache generated during the prefill phase. In conventional serving systems, this cache is tightly coupled to a single process or GPU, making it difficult to reuse across requests or share between instances. As a result, repeated prompts or multi-turn conversations often trigger redundant computation, increasing both latency and resource consumption. LMCache addresses this limitation by introducing a unified KV cache abstraction that can be externally managed, retrieved asynchronously, and seamlessly reintegrated into the decoding pipeline.
+
+A key design principle of LMCache is minimizing the impact of cache operations on the critical path of inference. By supporting asynchronous KV retrieval and background prefetching, LMCache allows decoding to proceed without blocking on cache transfers. This is particularly important in disaggregated or multi-process (MP) deployments, where different inference engines may collaborate through a shared cache backend. In such settings, LMCache enables one instance to reuse KV states computed by another, significantly reducing time-to-first-token (TTFT) and improving tail latency. Empirical results show substantial gains in multi-turn and stateful workloads, where reuse opportunities are abundant.
+
+LMCache also provides flexible support for heterogeneous memory and storage backends, including GPU memory, host memory, persistent storage (e.g., SSD or DAX devices), and high-speed interconnects such as RDMA. Through pluggable connectors and configurable transfer policies, it can adapt to diverse deployment environments, from single-node setups to large-scale distributed clusters. This flexibility enables system designers to balance trade-offs between latency, capacity, and cost, while maintaining high throughput.
+
+Beyond performance optimization, LMCache emphasizes observability and integration. It exposes detailed metrics for cache lookup, retrieval, and storage operations, allowing users to understand system behavior and diagnose bottlenecks. It is designed to integrate seamlessly with popular serving frameworks such as vLLM, requiring minimal changes to existing workflows while unlocking advanced caching capabilities.
+
+In summary, LMCache rethinks KV cache management as a first-class system component for LLM serving. By enabling efficient reuse, asynchronous data movement, and cross-instance sharing, it addresses fundamental inefficiencies in current inference pipelines and paves the way for more scalable, responsive, and resource-efficient AI systems.

lmcache/cli/main.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+"""LMCache CLI entry point.
+
+Subcommands are explicitly registered in
+``lmcache.cli.commands.ALL_COMMANDS``.
+"""
+
+# Standard
+import argparse
+import sys
+
+# First Party
+from lmcache.cli.commands import ALL_COMMANDS
+from lmcache.logging import init_logger
+
+logger = init_logger(__name__)
+
+
+def main() -> None:
+    """CLI entry point registered as ``lmcache`` in *pyproject.toml*."""
+    parser = argparse.ArgumentParser(
+        prog="lmcache",
+        description="LMCache — KV cache management for LLM serving",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    for cmd in ALL_COMMANDS:
+        cmd.register(subparsers)
+
+    args = parser.parse_args()
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        args.func(args)
+    except KeyboardInterrupt:
+        sys.exit(130)
+    except Exception:  # noqa: BLE001
+        logger.exception("Command failed")
+        sys.exit(1)

lmcache/cli/metrics/__init__.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Metrics package — collector, handlers, and formatters."""
+
+# First Party
+from lmcache.cli.metrics.formatter import (
+    JsonFormatter,
+    MetricsFormatter,
+    TerminalFormatter,
+    get_formatter,
+)
+from lmcache.cli.metrics.handler import (
+    FileHandler,
+    MetricsHandler,
+    StreamHandler,
+)
+from lmcache.cli.metrics.metrics import Metrics
+from lmcache.cli.metrics.section import Section
+
+__all__ = [
+    "FileHandler",
+    "JsonFormatter",
+    "get_formatter",
+    "Metrics",
+    "MetricsFormatter",
+    "MetricsHandler",
+    "Section",
+    "StreamHandler",
+    "TerminalFormatter",
+]

lmcache/cli/metrics/formatter.py

@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Metrics formatters — control *how* metrics are rendered.
+
+A formatter converts a title + sections into a string (or dict).
+Formatters are attached to handlers, separating rendering from destination.
+"""
+
+# Standard
+from typing import Any
+import abc
+import inspect
+import json
+
+# First Party
+from lmcache.cli.metrics.section import Section, sections_to_dict
+
+
+class MetricsFormatter(abc.ABC):
+    """Base class for metrics formatters."""
+
+    @abc.abstractmethod
+    def format(self, title: str, sections: list[Section]) -> str:
+        """Render metrics into a string.
+
+        Args:
+            title: The report title.
+            sections: Ordered list of ``Section`` objects.
+
+        Returns:
+            The formatted string.
+        """
+
+
+# ---------------------------------------------------------------------------
+# Formatter registry
+# ---------------------------------------------------------------------------
+
+_FORMATTER_REGISTRY: dict[str, type[MetricsFormatter]] = {}
+
+
+def register_formatter(name: str):
+    """Decorator that registers a ``MetricsFormatter`` subclass under *name*.
+
+    Args:
+        name: The format name used for CLI lookup (e.g. ``"json"``).
+
+    Returns:
+        A class decorator that registers the class and returns it unchanged.
+    """
+
+    def decorator(cls: type[MetricsFormatter]) -> type[MetricsFormatter]:
+        _FORMATTER_REGISTRY[name] = cls
+        return cls
+
+    return decorator
+
+
+def get_formatter(name: str, **kwargs: Any) -> MetricsFormatter:
+    """Instantiate a formatter by its registered name.
+
+    Args:
+        name: Registered format name (e.g. ``"terminal"``, ``"json"``).
+        **kwargs: Forwarded to the formatter constructor (e.g. ``width``).
+
+    Returns:
+        A new formatter instance.
+
+    Raises:
+        ValueError: If *name* is not registered.
+    """
+    cls = _FORMATTER_REGISTRY.get(name)
+    if cls is None:
+        available = ", ".join(sorted(_FORMATTER_REGISTRY))
+        raise ValueError(f"Unknown format {name!r}. Available: {available}")
+    # Only forward kwargs that the constructor accepts.
+    sig = inspect.signature(cls.__init__)
+    valid = {k: v for k, v in kwargs.items() if k in sig.parameters}
+    return cls(**valid)
+
+
+# ---------------------------------------------------------------------------
+# Built-in formatters
+# ---------------------------------------------------------------------------
+
+
+def _format_value(value: Any) -> str:
+    """Format a metric value for terminal display."""
+    if value is None:
+        return "N/A"
+    if isinstance(value, float):
+        return f"{value:.2f}"
+    return str(value)
+
+
+@register_formatter("terminal")
+class TerminalFormatter(MetricsFormatter):
+    """Plain ASCII table formatter for terminal output.
+
+    Title is centered in ``=`` borders, section headers are centered in
+    ``-`` borders, key-value lines have left-aligned labels and
+    right-aligned values.
+
+    Args:
+        width: Target total character width for the output.
+    """
+
+    def __init__(self, width: int = 48) -> None:
+        self._width = width
+
+    def format(self, title: str, sections: list[Section]) -> str:
+        """Render metrics as an ASCII table.
+
+        Args:
+            title: The report title.
+            sections: Ordered list of ``Section`` objects.
+
+        Returns:
+            Rendered multi-line string.
+        """
+        width = self._width
+        lines: list[str] = []
+
+        # Title bar
+        title_text = f" {title} "
+        lines.append(title_text.center(width, "="))
+
+        for section in sections:
+            # Section header (skip for unnamed section)
+            if section.label is not None:
+                header_text = f" {section.label} "
+                lines.append(header_text.center(width, "-"))
+
+            for _key, label, value in section.entries:
+                formatted = _format_value(value)
+                label_part = f"{label}:"
+                padding = width - len(label_part) - len(formatted)
+                if padding < 1:
+                    padding = 1
+                lines.append(f"{label_part}{' ' * padding}{formatted}")
+
+        # Footer
+        lines.append("=" * width)
+
+        return "\n".join(lines)
+
+
+@register_formatter("json")
+class JsonFormatter(MetricsFormatter):
+    """Renders metrics as a JSON string.
+
+    Args:
+        indent: JSON indentation level.
+    """
+
+    def __init__(self, indent: int = 2) -> None:
+        self._indent = indent
+
+    def format(self, title: str, sections: list[Section]) -> str:
+        """Render metrics as indented JSON.
+
+        Args:
+            title: The report title.
+            sections: Ordered list of ``Section`` objects.
+
+        Returns:
+            JSON string.
+        """
+        return json.dumps(
+            sections_to_dict(title, sections),
+            indent=self._indent,
+        )

lmcache/cli/metrics/handler.py

@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Metrics handlers — control *where* metrics are written.
+
+Each handler pairs a destination (stream, file, …) with a
+:class:`~lmcache.cli.metrics.formatter.MetricsFormatter` that controls
+the rendering.
+"""
+
+# Standard
+from typing import IO, Optional, Union
+import abc
+import os
+import sys
+
+# First Party
+from lmcache.cli.metrics.formatter import (
+    JsonFormatter,
+    MetricsFormatter,
+    TerminalFormatter,
+)
+from lmcache.cli.metrics.section import Section
+
+
+class MetricsHandler(abc.ABC):
+    """Base class for metrics handlers."""
+
+    def __init__(self, formatter: MetricsFormatter) -> None:
+        self.formatter = formatter
+
+    @abc.abstractmethod
+    def emit(self, title: str, sections: list[Section]) -> None:
+        """Format and write metrics to the destination.
+
+        Args:
+            title: The report title.
+            sections: Ordered list of ``Section`` objects.
+        """
+
+
+class StreamHandler(MetricsHandler):
+    """Writes formatted metrics to a text stream.
+
+    Args:
+        formatter: The formatter to use for rendering.
+        stream: Writable text stream. Defaults to ``sys.stdout``.
+    """
+
+    def __init__(
+        self,
+        formatter: Optional[MetricsFormatter] = None,
+        stream: Optional[IO[str]] = None,
+    ) -> None:
+        super().__init__(formatter or TerminalFormatter())
+        self._stream = stream
+
+    def emit(self, title: str, sections: list[Section]) -> None:
+        """Format and write metrics to the stream.
+
+        Args:
+            title: The report title.
+            sections: Ordered list of ``Section`` objects.
+        """
+        stream = self._stream or sys.stdout
+        stream.write(self.formatter.format(title, sections))
+        stream.write("\n")
+
+
+class FileHandler(MetricsHandler):
+    """Writes formatted metrics to a file.
+
+    Args:
+        path: Destination file path.
+        formatter: The formatter to use for rendering. Defaults to
+            :class:`~lmcache.cli.metrics.formatter.JsonFormatter`.
+    """
+
+    def __init__(
+        self,
+        path: Union[str, os.PathLike],
+        formatter: Optional[MetricsFormatter] = None,
+    ) -> None:
+        super().__init__(formatter or JsonFormatter())
+        self.path = path
+
+    def emit(self, title: str, sections: list[Section]) -> None:
+        """Format and write metrics to the file.
+
+        Args:
+            title: The report title.
+            sections: Ordered list of ``Section`` objects.
+        """
+        with open(self.path, "w") as f:
+            f.write(self.formatter.format(title, sections))
+            f.write("\n")