lmcache-cli 0.5.7.dev0__py3-none-any.whl → 0.5.7rc3.dev0__py3-none-any.whl

lmcache/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.5.7.dev0'
-__version_tuple__ = version_tuple = (0, 5, 7, 'dev0')
+__version__ = version = '0.5.7rc3.dev0'
+__version_tuple__ = version_tuple = (0, 5, 7, 'rc3', 'dev0')
 
-__commit_id__ = commit_id = 'g3e11e1081'
+__commit_id__ = commit_id = 'gbf85b41fa'

lmcache/cli/commands/bench/engine_bench/command.py

@@ -107,6 +107,7 @@ def add_engine_arguments(parser: argparse.ArgumentParser) -> None:
         "--workload",
         default=None,
         choices=[
+            "kv-tier-pressure",
             "long-doc-permutator",
             "long-doc-qa",
             "multi-round-chat",
@@ -234,6 +235,83 @@ def add_engine_arguments(parser: argparse.ArgumentParser) -> None:
         "--ignore-eos for a reproducible decode phase.",
     )
 
+    # --- KV-tier-pressure workload args ---
+    ktp_group = parser.add_argument_group("kv-tier-pressure workload options")
+    ktp_group.add_argument(
+        "--ktp-overflow-factor",
+        type=float,
+        default=2.0,
+        help="Working set as a multiple of the LMCache server's L1 capacity "
+        "(default: 2.0). Used only when --ktp-pool-size is omitted: the pool "
+        "is then sized from the capacity reported by --lmcache-url, so you "
+        "do not have to convert GB to tokens to documents yourself. Above "
+        "1.0 forces eviction to the storage tier.",
+    )
+    ktp_group.add_argument(
+        "--ktp-pool-size",
+        type=int,
+        default=None,
+        help="Total documents in the corpus. Derived from the server's L1 "
+        "capacity and --ktp-overflow-factor when omitted. "
+        "This sets the working "
+        "set -- pool_size x --ktp-context-length tokens -- independently of "
+        "how large one prompt is, and only what exceeds L1 can reach the "
+        "storage tier. To overflow a cache of V GB by a factor F: "
+        "ceil(F * V * tokens_per_gb_kvcache / context_length).",
+    )
+    ktp_group.add_argument(
+        "--ktp-docs-per-request",
+        type=int,
+        default=16,
+        help="Documents sampled into each request (default: 16). Bounded by "
+        "the engine's context limit, not by the pool size.",
+    )
+    ktp_group.add_argument(
+        "--ktp-context-length",
+        type=int,
+        default=2560,
+        help="Exact token length of each document (default: 2560, which is "
+        "10 whole 256-token chunks). Requires a loadable tokenizer; pass "
+        "--model when the engine reports a name that is not a HuggingFace "
+        "repo ID or local path.",
+    )
+    ktp_group.add_argument(
+        "--ktp-system-prompt-length",
+        type=int,
+        default=256,
+        help="Exact token length of the shared system prompt (default: 256). "
+        "Use 0 for no system prompt.",
+    )
+    ktp_group.add_argument(
+        "--ktp-num-requests",
+        type=int,
+        default=200,
+        help="Number of measured requests (default: 200). The warm-up sweep "
+        "is separate and sized automatically from the pool.",
+    )
+    ktp_group.add_argument(
+        "--ktp-access-skew",
+        type=float,
+        default=0.0,
+        help="Zipf exponent for document popularity (default: 0.0 = uniform). "
+        "Larger values concentrate reads on a hot subset, raising the L1 hit "
+        "rate and lowering the L2 share.",
+    )
+    ktp_group.add_argument(
+        "--ktp-num-inflight-requests",
+        type=int,
+        default=8,
+        help="Max concurrent in-flight requests (default: 8).",
+    )
+    ktp_group.add_argument(
+        "--ktp-max-output-length",
+        type=int,
+        default=1,
+        help="Max tokens to generate per request (default: 1). The default "
+        "isolates prefill, which is the phase the cache tier affects; larger "
+        "values add decode time that dilutes the measurement.",
+    )
+
     # --- Long-doc-qa workload args ---
     group = parser.add_argument_group("long-doc-qa workload options")
     group.add_argument(
@@ -472,6 +550,20 @@ def _resolve_args(args: argparse.Namespace) -> argparse.Namespace:
             cli_val = getattr(args, attr, None)
             if cli_val is not None:
                 setattr(resolved, attr, cli_val)
+        # A config file can omit a workload-specific required value -- it is
+        # only written when the flag was supplied. Catch it here rather than
+        # letting None reach the workload and surface as a TypeError.
+        missing = [
+            flag
+            for attr, flag in _REQUIRED_WORKLOAD_ARGS.get(resolved.workload, ())
+            if getattr(resolved, attr, None) is None
+        ]
+        if missing:
+            raise SystemExit(
+                f"Config {config_path} is missing required arguments for the "
+                f"{resolved.workload!r} workload: " + ", ".join(missing) + ". "
+                "Add them to the config, or pass them on the command line."
+            )
         return resolved
 
     # Case 2: --no-interactive or --export-config — error if missing

lmcache/cli/commands/bench/engine_bench/config.py

@@ -177,6 +177,69 @@ def _find_model_meta(
     )
 
 
+def server_runs_blend_module(lmcache_url: str) -> bool:
+    """Report whether the LMCache server loaded the blend module.
+
+    Detected by the presence of ``active_cb_lookups`` in ``/status``, which
+    only :mod:`lmcache.v1.multiprocess.modules.blend` emits.  Two fields that
+    look like they would answer this do not: ``engine_type`` is
+    ``self.__class__.__name__`` and reads ``MPCacheServer`` for both engines,
+    and nothing in the codebase writes ``cb_gpu_context_meta``.  Tracked
+    upstream in #5589; switch to the reported engine once that lands.
+
+    This is a necessary but not sufficient condition for blended reuse -- it
+    says the server can blend, not that vLLM is driving it through
+    ``CBKVConnector``.  Confirm the end-to-end path by checking which lookup
+    counter family moves once traffic has run.
+
+    Args:
+        lmcache_url: URL of the LMCache HTTP server.
+
+    Returns:
+        True when the server reports the blend module.
+
+    Raises:
+        RuntimeError: If the server is unreachable.
+    """
+    return "active_cb_lookups" in _fetch_lmcache_status(lmcache_url)
+
+
+def resolve_l1_capacity_gb(lmcache_url: str) -> float:
+    """Query the LMCache server for the configured host-memory capacity.
+
+    Reads ``storage_manager.l1_manager.memory_total_bytes`` from ``/status``.
+    Workloads that size themselves against the cache need the real capacity,
+    not a figure the user retyped; a mistyped value silently produces a
+    working set that never exceeds L1, and a run that does no L2 I/O at all.
+
+    Args:
+        lmcache_url: URL of the LMCache HTTP server.
+
+    Returns:
+        Configured L1 capacity in GB.
+
+    Raises:
+        RuntimeError: If the server is unreachable, or ``/status`` does not
+            report an L1 capacity.
+    """
+    data = _fetch_lmcache_status(lmcache_url)
+    total = (
+        data.get("storage_manager", {}).get("l1_manager", {}).get("memory_total_bytes")
+    )
+    if total is None:
+        # Older servers report it directly on storage_manager.
+        total = data.get("storage_manager", {}).get("memory_total_bytes")
+    if not isinstance(total, (int, float)) or total <= 0:
+        raise RuntimeError(
+            f"LMCache at {lmcache_url} did not report an L1 capacity "
+            f"(storage_manager.l1_manager.memory_total_bytes). Pass an "
+            f"explicit pool size instead."
+        )
+    capacity_gb = float(total) / (1024**3)
+    logger.debug("Resolved L1 capacity %.2f GB from %s", capacity_gb, lmcache_url)
+    return capacity_gb
+
+
 def resolve_tokens_per_gb(lmcache_url: str, model_name: str) -> int:
     """Query the LMCache server and compute tokens per GB of KV cache.
 

lmcache/cli/commands/bench/engine_bench/interactive/schema.py

@@ -112,6 +112,11 @@ ALL_ITEMS: list[ConfigItem] = [
         default=None,
         required=True,
         choices=[
+            (
+                "kv-tier-pressure",
+                "Retrieval over a corpus too large to cache, exercising the "
+                "storage tier",
+            ),
             (
                 "long-doc-permutator",
                 "Query the same set of long documents with different orders",
@@ -234,6 +239,103 @@ ALL_ITEMS: list[ConfigItem] = [
         default=False,
         phase=PHASE_GENERAL,
     ),
+    # ── Phase 3: kv-tier-pressure ────────────────────────────────────
+    ConfigItem(
+        key="ktp_pool_size",
+        display_name="Document pool size",
+        description=(
+            "Total documents in the corpus. Sets the working set "
+            "(pool size x document length), and therefore whether the "
+            "storage tier is reached. Leave empty to size it from the "
+            "LMCache server's L1 capacity and the overflow factor."
+        ),
+        input_type="int",
+        default=None,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_overflow_factor",
+        display_name="Overflow factor",
+        description=(
+            "Working set as a multiple of the server's L1 capacity. Used "
+            "only when no pool size is given. Above 1.0 forces eviction to "
+            "the storage tier."
+        ),
+        input_type="float",
+        default=2.0,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_docs_per_request",
+        display_name="Documents per request",
+        description="Documents sampled into each request from the pool.",
+        input_type="int",
+        default=16,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_context_length",
+        display_name="Document length (tokens)",
+        description="Token length of each document in the pool.",
+        input_type="int",
+        default=2560,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_system_prompt_length",
+        display_name="System prompt length (tokens)",
+        description="Token length of the shared system prompt. Use 0 for none.",
+        input_type="int",
+        default=256,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_num_requests",
+        display_name="Measured requests",
+        description=(
+            "Requests to measure. The warm-up sweep is separate and sized "
+            "automatically from the pool."
+        ),
+        input_type="int",
+        default=200,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_access_skew",
+        display_name="Access skew",
+        description=(
+            "Zipf exponent for document popularity. 0 samples uniformly; "
+            "larger values concentrate reads on a hot subset."
+        ),
+        input_type="float",
+        default=0.0,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_num_inflight_requests",
+        display_name="Max inflight requests",
+        description="Maximum concurrent in-flight requests.",
+        input_type="int",
+        default=8,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_max_output_length",
+        display_name="Max output length (tokens)",
+        description="Max tokens per request. Use 1 to measure prefill alone.",
+        input_type="int",
+        default=1,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
     # ── Phase 3: long-doc-permutator ─────────────────────────────────
     ConfigItem(
         key="ldp_num_contexts",

lmcache/cli/commands/bench/engine_bench/quality/scoring.py

@@ -219,10 +219,12 @@ def _same_number(
 def answer_match(prediction: str, references: list[str]) -> bool:
     """Whether *prediction* states one of the gold *references*.
 
-    A reference matches when, after :func:`normalize_answer`, it equals the
-    prediction or appears in it as a whole-word phrase ("Paris" in "the
-    capital is Paris").  A reference containing numbers also matches when
-    every one of its numbers appears in the prediction, up to 0.5% rounding.
+    A reference without numeric figures matches when, after
+    :func:`normalize_answer`, it equals the prediction or appears in it as
+    a whole-word phrase ("Paris" in "the capital is Paris"). References with
+    numeric figures require every figure to appear in the prediction, up
+    to 0.5% rounding. They cannot match by a punctuation-stripped phrase:
+    removing a decimal point would equate different values ("1.5", "15").
     When either side names a scale ("million", "billion", "5B", …), a rescale
     by a power of 1000 is also accepted, so "$11588.00" (in millions) matches
     "$11,588 million" and "$11.588 billion", but not "$3,033 million"; and a
@@ -265,9 +267,9 @@ def answer_match(prediction: str, references: list[str]) -> bool:
         gold = _phrase(reference)
         if abstains and not (_NEGATIVE.search(gold) or is_abstention(reference)):
             continue
-        if gold and (gold == normalized or f" {gold} " in padded):
-            return True
         gold_numbers = _numbers(reference)
+        if not gold_numbers and gold and (gold == normalized or f" {gold} " in padded):
+            return True
         scales = (
             _UNIT_SCALES
             if _SCALE_WORD.search(prediction) or _SCALE_WORD.search(reference)

lmcache/cli/commands/bench/engine_bench/request_sender.py

@@ -36,9 +36,18 @@ def _normalize_url(engine_url: str) -> str:
 
 
 def _extract_content(chunk: object, completions_mode: bool) -> str:
-    """Return text content from a streaming chunk, or ``""`` if none.
+    """Return non-empty text from a completion or chat streaming chunk.
 
-    Ported from Tensormesh-Benchmark ``streaming_utils.py``.
+    Chat chunks fall back to ``reasoning_content`` and then ``reasoning``
+    when ``content`` is empty. Ported from Tensormesh-Benchmark
+    ``streaming_utils.py``.
+
+    Args:
+        chunk: An OpenAI-compatible streaming response chunk.
+        completions_mode: Whether the chunk uses the legacy completions API.
+
+    Returns:
+        The chunk's text, or an empty string when it contains no text.
     """
     choices = getattr(chunk, "choices", None)
     if not choices:
@@ -55,12 +64,12 @@ def _extract_content(chunk: object, completions_mode: bool) -> str:
     if delta is None:
         return ""
     content = getattr(delta, "content", None)
-    if content is not None:
+    if content:
         return content
     # Fallback for reasoning models
     for attr in ("reasoning_content", "reasoning"):
         fallback = getattr(delta, attr, None)
-        if fallback is not None:
+        if fallback:
             return fallback
     return ""
 
@@ -138,10 +147,20 @@ class RequestSender:
     ) -> RequestResult:
         """Send a single streaming request and return the result.
 
-        Streams the response via SSE, measures TTFT, decode speed, and
-        total latency.  Extracts token counts from server usage reports.
+        Streams the response via SSE, measures TTFT from the first non-empty
+        content or reasoning text, decode speed, and total latency. Extracts
+        token counts from server usage reports.
         After collecting the result, invokes all registered
         ``on_finished`` callbacks.
+
+        Args:
+            request_id: Identifier included in the result and callbacks.
+            messages: Chat messages, or the first message's content as the
+                prompt in completions mode.
+            max_tokens: Maximum number of output tokens requested.
+
+        Returns:
+            Token counts, timings, and success status for this request.
         """
         submit_time = time.time()
         first_token_time = 0.0

lmcache/cli/commands/bench/engine_bench/stats.py

@@ -243,6 +243,7 @@ class StatsCollector:
             "config": asdict(config),
             "results": asdict(final),
         }
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
         with open(path, "w") as f:
             json.dump(output, f, indent=2)
         logger.debug("Exported JSON summary to: %s", path)

lmcache/cli/commands/bench/engine_bench/workloads/__init__.py

@@ -12,13 +12,21 @@ import argparse
 import os
 
 # First Party
-from lmcache.cli.commands.bench.engine_bench.config import EngineBenchConfig
+from lmcache.cli.commands.bench.engine_bench.config import (
+    EngineBenchConfig,
+    resolve_l1_capacity_gb,
+    server_runs_blend_module,
+)
 from lmcache.cli.commands.bench.engine_bench.progress import ProgressMonitor
 from lmcache.cli.commands.bench.engine_bench.request_sender import (
     RequestSender,
 )
 from lmcache.cli.commands.bench.engine_bench.stats import StatsCollector
 from lmcache.cli.commands.bench.engine_bench.workloads.base import BaseWorkload
+from lmcache.cli.commands.bench.engine_bench.workloads.kv_tier_pressure import (
+    KVTierPressureConfig,
+    KVTierPressureWorkload,
+)
 from lmcache.cli.commands.bench.engine_bench.workloads.long_doc_permutator import (
     LongDocPermutatorConfig,
     LongDocPermutatorWorkload,
@@ -45,12 +53,15 @@ from lmcache.cli.commands.bench.engine_bench.workloads.random_prefill import (
     RandomPrefillConfig,
     RandomPrefillWorkload,
 )
+from lmcache.logging import init_logger
 
 __all__ = [
     "DEFAULT_DOC_ALIGN_TOKENS",
     "BaseWorkload",
     "LongDocPermutatorConfig",
     "LongDocPermutatorWorkload",
+    "KVTierPressureConfig",
+    "KVTierPressureWorkload",
     "LongDocQAConfig",
     "LongDocQAWorkload",
     "MultiRoundChatConfig",
@@ -66,7 +77,10 @@ __all__ = [
     "validate_max_output_length_supported",
 ]
 
+logger = init_logger(__name__)
+
 _WORKLOAD_NAMES = (
+    "kv-tier-pressure",
     "long-doc-permutator",
     "long-doc-qa",
     "multi-round-chat",
@@ -77,19 +91,19 @@ _WORKLOAD_NAMES = (
 
 # Workloads that expose a user-configurable max output length, each via its own
 # flag (``--ldqa-max-output-length`` / ``--mrc-output-length`` /
-# ``--ldp-max-output-length``).
+# ``--ldp-max-output-length`` / ``--ktp-max-output-length``).
 _WORKLOADS_WITH_MAX_OUTPUT_LENGTH: frozenset[str] = frozenset(
-    {"long-doc-permutator", "long-doc-qa", "multi-round-chat"}
+    {"long-doc-permutator", "long-doc-qa", "kv-tier-pressure", "multi-round-chat"}
 )
 
 
 def validate_max_output_length_supported(workload: str) -> None:
     """Validate that a max output length can be specified for ``workload``.
 
-    Only workloads with a max-output-length parameter (``long-doc-permutator``,
-    ``long-doc-qa``, ``multi-round-chat``) support setting it; every other
-    workload fixes its generation length internally, so requesting one is
-    rejected.
+    Only workloads with a max-output-length parameter
+    (``long-doc-permutator``, ``long-doc-qa``, ``kv-tier-pressure``,
+    ``multi-round-chat``) support setting it; every other workload fixes its
+    generation length internally, so requesting one is rejected.
 
     Args:
         workload: The selected workload name (``EngineBenchConfig.workload``).
@@ -151,6 +165,66 @@ def create_workload(
             model_name=config.model,
         )
 
+    if config.workload == "kv-tier-pressure":
+        # Query the server for its real L1 capacity so the pool can be sized
+        # against the cache rather than against a figure the user retyped.
+        # Only needed when no explicit pool size was given.
+        lmcache_url = getattr(args, "lmcache_url", None) or ""
+        l1_capacity_gb = 0.0
+        if not args.ktp_pool_size and lmcache_url:
+            try:
+                l1_capacity_gb = resolve_l1_capacity_gb(lmcache_url)
+            except RuntimeError as exc:
+                logger.warning("Could not read L1 capacity: %s", exc)
+        # A cheap early signal only. The server reporting the blend module
+        # does not mean vLLM is driving it through CBKVConnector, and the
+        # fields that look authoritative are not: engine_type is the server
+        # class name (MPCacheServer for both engines) and nothing writes
+        # cb_gpu_context_meta. The real check is which lookup counter family
+        # moves during warm-up, which the workload does itself once traffic
+        # has run -- so warn here, do not refuse.
+        if lmcache_url and args.ktp_docs_per_request > 1:
+            try:
+                runs_blend = server_runs_blend_module(lmcache_url)
+            except RuntimeError as exc:
+                logger.warning("Could not read the server status: %s", exc)
+            else:
+                if not runs_blend:
+                    logger.warning(
+                        "The LMCache server at %s does not report the blend "
+                        "module, and --ktp-docs-per-request is %d. Each "
+                        "request concatenates a random subset of the pool in "
+                        "random order, which a prefix-chained cache cannot "
+                        "reuse. The run will stop after warm-up if blended "
+                        "reuse is not confirmed; pass "
+                        "--ktp-docs-per-request 1 for the non-blend path.",
+                        lmcache_url,
+                        args.ktp_docs_per_request,
+                    )
+        ktp_workload_config = KVTierPressureConfig.resolve(
+            pool_size=args.ktp_pool_size or 0,
+            l1_capacity_gb=l1_capacity_gb,
+            tokens_per_gb_kvcache=config.tokens_per_gb_kvcache,
+            overflow_factor=args.ktp_overflow_factor,
+            docs_per_request=args.ktp_docs_per_request,
+            context_length=args.ktp_context_length,
+            system_prompt_length=args.ktp_system_prompt_length,
+            num_requests=args.ktp_num_requests,
+            access_skew=args.ktp_access_skew,
+            vocab_size=8000,
+            num_inflight_requests=args.ktp_num_inflight_requests,
+            max_output_length=args.ktp_max_output_length,
+        )
+        return KVTierPressureWorkload(
+            config=ktp_workload_config,
+            request_sender=request_sender,
+            stats_collector=stats_collector,
+            progress_monitor=progress_monitor,
+            seed=config.seed,
+            model_name=config.model,
+            lmcache_url=lmcache_url,
+        )
+
     if config.workload == "long-doc-qa":
         ld_workload_config = LongDocQAConfig.resolve(
             kv_cache_volume_gb=config.kv_cache_volume_gb,