lmcache-cli 0.5.7.dev0__py3-none-any.whl → 0.5.7.post2.dev0__py3-none-any.whl

lmcache/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.5.7.dev0'
-__version_tuple__ = version_tuple = (0, 5, 7, 'dev0')
+__version__ = version = '0.5.7.post2.dev0'
+__version_tuple__ = version_tuple = (0, 5, 7, 'post2', 'dev0')
 
-__commit_id__ = commit_id = 'g3e11e1081'
+__commit_id__ = commit_id = 'g1181bea5c'

lmcache/cli/commands/bench/engine_bench/command.py

@@ -107,6 +107,7 @@ def add_engine_arguments(parser: argparse.ArgumentParser) -> None:
         "--workload",
         default=None,
         choices=[
+            "kv-tier-pressure",
             "long-doc-permutator",
             "long-doc-qa",
             "multi-round-chat",
@@ -234,6 +235,83 @@ def add_engine_arguments(parser: argparse.ArgumentParser) -> None:
         "--ignore-eos for a reproducible decode phase.",
     )
 
+    # --- KV-tier-pressure workload args ---
+    ktp_group = parser.add_argument_group("kv-tier-pressure workload options")
+    ktp_group.add_argument(
+        "--ktp-overflow-factor",
+        type=float,
+        default=2.0,
+        help="Working set as a multiple of the LMCache server's L1 capacity "
+        "(default: 2.0). Used only when --ktp-pool-size is omitted: the pool "
+        "is then sized from the capacity reported by --lmcache-url, so you "
+        "do not have to convert GB to tokens to documents yourself. Above "
+        "1.0 forces eviction to the storage tier.",
+    )
+    ktp_group.add_argument(
+        "--ktp-pool-size",
+        type=int,
+        default=None,
+        help="Total documents in the corpus. Derived from the server's L1 "
+        "capacity and --ktp-overflow-factor when omitted. "
+        "This sets the working "
+        "set -- pool_size x --ktp-context-length tokens -- independently of "
+        "how large one prompt is, and only what exceeds L1 can reach the "
+        "storage tier. To overflow a cache of V GB by a factor F: "
+        "ceil(F * V * tokens_per_gb_kvcache / context_length).",
+    )
+    ktp_group.add_argument(
+        "--ktp-docs-per-request",
+        type=int,
+        default=16,
+        help="Documents sampled into each request (default: 16). Bounded by "
+        "the engine's context limit, not by the pool size.",
+    )
+    ktp_group.add_argument(
+        "--ktp-context-length",
+        type=int,
+        default=2560,
+        help="Exact token length of each document (default: 2560, which is "
+        "10 whole 256-token chunks). Requires a loadable tokenizer; pass "
+        "--model when the engine reports a name that is not a HuggingFace "
+        "repo ID or local path.",
+    )
+    ktp_group.add_argument(
+        "--ktp-system-prompt-length",
+        type=int,
+        default=256,
+        help="Exact token length of the shared system prompt (default: 256). "
+        "Use 0 for no system prompt.",
+    )
+    ktp_group.add_argument(
+        "--ktp-num-requests",
+        type=int,
+        default=200,
+        help="Number of measured requests (default: 200). The warm-up sweep "
+        "is separate and sized automatically from the pool.",
+    )
+    ktp_group.add_argument(
+        "--ktp-access-skew",
+        type=float,
+        default=0.0,
+        help="Zipf exponent for document popularity (default: 0.0 = uniform). "
+        "Larger values concentrate reads on a hot subset, raising the L1 hit "
+        "rate and lowering the L2 share.",
+    )
+    ktp_group.add_argument(
+        "--ktp-num-inflight-requests",
+        type=int,
+        default=8,
+        help="Max concurrent in-flight requests (default: 8).",
+    )
+    ktp_group.add_argument(
+        "--ktp-max-output-length",
+        type=int,
+        default=1,
+        help="Max tokens to generate per request (default: 1). The default "
+        "isolates prefill, which is the phase the cache tier affects; larger "
+        "values add decode time that dilutes the measurement.",
+    )
+
     # --- Long-doc-qa workload args ---
     group = parser.add_argument_group("long-doc-qa workload options")
     group.add_argument(
@@ -472,6 +550,20 @@ def _resolve_args(args: argparse.Namespace) -> argparse.Namespace:
             cli_val = getattr(args, attr, None)
             if cli_val is not None:
                 setattr(resolved, attr, cli_val)
+        # A config file can omit a workload-specific required value -- it is
+        # only written when the flag was supplied. Catch it here rather than
+        # letting None reach the workload and surface as a TypeError.
+        missing = [
+            flag
+            for attr, flag in _REQUIRED_WORKLOAD_ARGS.get(resolved.workload, ())
+            if getattr(resolved, attr, None) is None
+        ]
+        if missing:
+            raise SystemExit(
+                f"Config {config_path} is missing required arguments for the "
+                f"{resolved.workload!r} workload: " + ", ".join(missing) + ". "
+                "Add them to the config, or pass them on the command line."
+            )
         return resolved
 
     # Case 2: --no-interactive or --export-config — error if missing

lmcache/cli/commands/bench/engine_bench/config.py

@@ -177,6 +177,69 @@ def _find_model_meta(
     )
 
 
+def server_runs_blend_module(lmcache_url: str) -> bool:
+    """Report whether the LMCache server loaded the blend module.
+
+    Detected by the presence of ``active_cb_lookups`` in ``/status``, which
+    only :mod:`lmcache.v1.multiprocess.modules.blend` emits.  Two fields that
+    look like they would answer this do not: ``engine_type`` is
+    ``self.__class__.__name__`` and reads ``MPCacheServer`` for both engines,
+    and nothing in the codebase writes ``cb_gpu_context_meta``.  Tracked
+    upstream in #5589; switch to the reported engine once that lands.
+
+    This is a necessary but not sufficient condition for blended reuse -- it
+    says the server can blend, not that vLLM is driving it through
+    ``CBKVConnector``.  Confirm the end-to-end path by checking which lookup
+    counter family moves once traffic has run.
+
+    Args:
+        lmcache_url: URL of the LMCache HTTP server.
+
+    Returns:
+        True when the server reports the blend module.
+
+    Raises:
+        RuntimeError: If the server is unreachable.
+    """
+    return "active_cb_lookups" in _fetch_lmcache_status(lmcache_url)
+
+
+def resolve_l1_capacity_gb(lmcache_url: str) -> float:
+    """Query the LMCache server for the configured host-memory capacity.
+
+    Reads ``storage_manager.l1_manager.memory_total_bytes`` from ``/status``.
+    Workloads that size themselves against the cache need the real capacity,
+    not a figure the user retyped; a mistyped value silently produces a
+    working set that never exceeds L1, and a run that does no L2 I/O at all.
+
+    Args:
+        lmcache_url: URL of the LMCache HTTP server.
+
+    Returns:
+        Configured L1 capacity in GB.
+
+    Raises:
+        RuntimeError: If the server is unreachable, or ``/status`` does not
+            report an L1 capacity.
+    """
+    data = _fetch_lmcache_status(lmcache_url)
+    total = (
+        data.get("storage_manager", {}).get("l1_manager", {}).get("memory_total_bytes")
+    )
+    if total is None:
+        # Older servers report it directly on storage_manager.
+        total = data.get("storage_manager", {}).get("memory_total_bytes")
+    if not isinstance(total, (int, float)) or total <= 0:
+        raise RuntimeError(
+            f"LMCache at {lmcache_url} did not report an L1 capacity "
+            f"(storage_manager.l1_manager.memory_total_bytes). Pass an "
+            f"explicit pool size instead."
+        )
+    capacity_gb = float(total) / (1024**3)
+    logger.debug("Resolved L1 capacity %.2f GB from %s", capacity_gb, lmcache_url)
+    return capacity_gb
+
+
 def resolve_tokens_per_gb(lmcache_url: str, model_name: str) -> int:
     """Query the LMCache server and compute tokens per GB of KV cache.
 

lmcache/cli/commands/bench/engine_bench/interactive/schema.py

@@ -112,6 +112,11 @@ ALL_ITEMS: list[ConfigItem] = [
         default=None,
         required=True,
         choices=[
+            (
+                "kv-tier-pressure",
+                "Retrieval over a corpus too large to cache, exercising the "
+                "storage tier",
+            ),
             (
                 "long-doc-permutator",
                 "Query the same set of long documents with different orders",
@@ -234,6 +239,103 @@ ALL_ITEMS: list[ConfigItem] = [
         default=False,
         phase=PHASE_GENERAL,
     ),
+    # ── Phase 3: kv-tier-pressure ────────────────────────────────────
+    ConfigItem(
+        key="ktp_pool_size",
+        display_name="Document pool size",
+        description=(
+            "Total documents in the corpus. Sets the working set "
+            "(pool size x document length), and therefore whether the "
+            "storage tier is reached. Leave empty to size it from the "
+            "LMCache server's L1 capacity and the overflow factor."
+        ),
+        input_type="int",
+        default=None,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_overflow_factor",
+        display_name="Overflow factor",
+        description=(
+            "Working set as a multiple of the server's L1 capacity. Used "
+            "only when no pool size is given. Above 1.0 forces eviction to "
+            "the storage tier."
+        ),
+        input_type="float",
+        default=2.0,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_docs_per_request",
+        display_name="Documents per request",
+        description="Documents sampled into each request from the pool.",
+        input_type="int",
+        default=16,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_context_length",
+        display_name="Document length (tokens)",
+        description="Token length of each document in the pool.",
+        input_type="int",
+        default=2560,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_system_prompt_length",
+        display_name="System prompt length (tokens)",
+        description="Token length of the shared system prompt. Use 0 for none.",
+        input_type="int",
+        default=256,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_num_requests",
+        display_name="Measured requests",
+        description=(
+            "Requests to measure. The warm-up sweep is separate and sized "
+            "automatically from the pool."
+        ),
+        input_type="int",
+        default=200,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_access_skew",
+        display_name="Access skew",
+        description=(
+            "Zipf exponent for document popularity. 0 samples uniformly; "
+            "larger values concentrate reads on a hot subset."
+        ),
+        input_type="float",
+        default=0.0,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_num_inflight_requests",
+        display_name="Max inflight requests",
+        description="Maximum concurrent in-flight requests.",
+        input_type="int",
+        default=8,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
+    ConfigItem(
+        key="ktp_max_output_length",
+        display_name="Max output length (tokens)",
+        description="Max tokens per request. Use 1 to measure prefill alone.",
+        input_type="int",
+        default=1,
+        condition=_workload_is("kv-tier-pressure"),
+        phase=PHASE_WORKLOAD,
+    ),
     # ── Phase 3: long-doc-permutator ─────────────────────────────────
     ConfigItem(
         key="ldp_num_contexts",