lmcache-cli 0.4.8.dev38__py3-none-any.whl → 0.4.8.dev42__py3-none-any.whl

lmcache/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.4.8.dev38'
-__version_tuple__ = version_tuple = (0, 4, 8, 'dev38')
+__version__ = version = '0.4.8.dev42'
+__version_tuple__ = version_tuple = (0, 4, 8, 'dev42')
 
-__commit_id__ = commit_id = 'ge1fdbc634'
+__commit_id__ = commit_id = 'g9ffb104f3'

lmcache/v1/mp_coordinator/blend_client.py

@@ -14,6 +14,7 @@ It is **opt-in**: with no coordinator URL configured the blend module receives
 
 # Standard
 from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from queue import Empty, Queue
 import os
@@ -21,6 +22,7 @@ import threading
 
 # First Party
 from lmcache.logging import init_logger
+from lmcache.v1.mp_coordinator.schemas import encode_tokens
 
 logger = init_logger(__name__)
 
@@ -67,10 +69,11 @@ class _MatchItem:
 class BlendCoordinatorClient:
     """Background bridge from the blend module to the coordinator directory.
 
-    Thread-safe. Publishes and match queries are enqueued by blend handler
-    threads and serviced by one daemon thread; match results are stored in a dict
-    the handler polls. Match queries are prioritized over publishes so lookup
-    latency is not held up by best-effort store traffic.
+    Thread-safe. Handler threads enqueue publishes and match queries; one daemon
+    thread dequeues them, dispatching each match query to a thread pool so
+    round-trips run concurrently. Match results land in a dict the handler
+    polls. Match queries drain ahead of publishes so lookup latency is not held
+    up by best-effort store traffic.
     """
 
     def __init__(
@@ -80,6 +83,7 @@ class BlendCoordinatorClient:
         request_fn: _RequestFn | None = None,
         request_timeout: float = 2.0,
         match_budget_s: float = 2.0,
+        match_concurrency: int = 8,
     ) -> None:
         """Create the client and start its daemon.
 
@@ -91,10 +95,15 @@ class BlendCoordinatorClient:
                 testing). Must raise on transport failure.
             request_timeout: Per-request HTTP timeout in seconds.
             match_budget_s: Per-lookup wall-clock budget the blend module uses to
-                bound the optional global leg. Unlike ``request_timeout`` (one
-                round-trip), this bounds total time including queue wait behind
-                other match queries, since the daemon services them serially.
+                bound the optional global leg, including queue wait.
+            match_concurrency: Max match round-trips in flight at once (the
+                match dispatch pool size). Must be >= 1.
+
+        Raises:
+            ValueError: If ``match_concurrency`` is not positive.
         """
+        if match_concurrency < 1:
+            raise ValueError(f"match_concurrency must be >= 1, got {match_concurrency}")
         self.match_budget_s = match_budget_s
         self._client = None
         if request_fn is None:
@@ -119,6 +128,9 @@ class BlendCoordinatorClient:
         self._results: dict[str, object] = {}
         self._results_lock = threading.Lock()
         self._stop = threading.Event()
+        self._match_pool = ThreadPoolExecutor(
+            max_workers=match_concurrency, thread_name_prefix="cb-coord-match"
+        )
         self._worker = threading.Thread(
             target=self._run, name="cb-coordinator-client", daemon=True
         )
@@ -185,9 +197,10 @@ class BlendCoordinatorClient:
             self._results.pop(rid, None)
 
     def close(self) -> None:
-        """Stop the daemon and close the HTTP client."""
+        """Stop the daemon, drain the match pool, and close the HTTP client."""
         self._stop.set()
         self._worker.join(timeout=2.0)
+        self._match_pool.shutdown(wait=False)
         if self._client is not None:
             self._client.close()
 
@@ -211,20 +224,28 @@ class BlendCoordinatorClient:
         url = os.getenv("LMCACHE_COORDINATOR_URL", "").strip()
         if not url:
             return None
+        concurrency = int(os.getenv("LMCACHE_COORDINATOR_BLEND_MATCH_CONCURRENCY", "8"))
         timeout = float(os.getenv("LMCACHE_COORDINATOR_BLEND_TIMEOUT", "1.0"))
         logger.info("Blend coordinator client enabled -> %s", url)
-        return cls(url, request_timeout=timeout, match_budget_s=timeout)
+        return cls(
+            url,
+            request_timeout=timeout,
+            match_budget_s=timeout,
+            match_concurrency=concurrency,
+        )
 
     # -- daemon ------------------------------------------------------------
 
     def _run(self) -> None:
-        """Service match queries (priority) then publishes until stopped."""
+        """Dispatch match queries to the pool (priority), then publishes."""
         while not self._stop.is_set():
             try:
-                self._handle_match(self._match_q.get(timeout=0.05))
-                continue
+                item = self._match_q.get(timeout=0.05)
             except Empty:
-                pass
+                item = None
+            if item is not None:
+                self._match_pool.submit(self._handle_match, item)
+                continue
             try:
                 self._handle_publish(self._publish_q.get_nowait())
             except Empty:
@@ -239,7 +260,7 @@ class BlendCoordinatorClient:
                 "/blend/match",
                 {
                     "model_scope": item.model_scope,
-                    "tokens": item.tokens,
+                    "tokens_b64": encode_tokens(item.tokens),
                 },
             )
             matches = [

lmcache/v1/mp_coordinator/blend_directory.py

@@ -15,21 +15,39 @@ the request.
 - **Probe stride** (the querying server's inference block size) controls which
   request offsets can seed a match; sent per query, so servers with different
   (per-machine, dynamic) block sizes interoperate.
-- **Scope** = ``f"{model_name}@{cache_salt}"``; cross-scope content never matches.
+- **Scope** = the model name; cross-model content never matches. ``cache_salt``
+  is **not** part of the scope: matched hashes are expanded to ObjectKeys with
+  the *requester's* salt at retrieve (``ipc_key_to_object_keys``), so a
+  cross-salt match simply misses at L2 -- isolation holds with one table per
+  model instead of one per ``(model, salt)``.
 - **TP rank** is resolved at retrieve (``ipc_key_to_object_keys``), not here.
 
 ``object_key`` is the chunk's shared-L2 storage key (``th``), which is
 prefix-bound and computed by the storing server -- the coordinator cannot derive
 it, so it is supplied with each published range.
 
-Thread-safe (single lock, mirroring ``InstanceRegistry``) and ephemeral. A stale
-entry (chunk evicted from L2 but not yet removed here) only causes a wasted
-prefetch downstream, then recompute -- never wrong KV.
+Concurrency: fingerprints are partitioned per scope (``_ScopeTable``), each
+mutated and probed in place under its own lock; a top-level lock guards only the
+scope map and the eviction map. The probe is vectorized, so the locked section
+is short.
+
+The probe mirrors the local matcher: the strided rolling hashes index a
+per-scope **direct-address table** in one numpy gather, and only occupied slots
+reach a short Python loop, where a full-64-bit re-check rejects bucket
+collisions. Inserts write the table in place (O(1) per chunk); evictions
+tombstone; a rebuild -- which also compacts tombstones -- happens only on the
+write path, when the table outgrows its load factor. Tables are sized per scope
+(power of two, a few times the entry count), so small scopes stay small, unlike
+the local matcher's fixed 2^20 array.
+
+Thread-safe and ephemeral. A stale entry or a (rare) bucket collision only costs
+a wasted prefetch or a missed reuse, recomputed downstream -- never wrong KV.
 
 See ``docs/design/v1/mp_coordinator/blend_lookup.md``.
 """
 
 # Standard
+from collections import defaultdict
 from dataclasses import dataclass
 import threading
 
@@ -41,6 +59,7 @@ from lmcache.logging import init_logger
 from lmcache.v1.multiprocess.token_hasher import (
     chunk_hash_windows_numba,
     rolling_hash_windows_numba,
+    update_table_id_numba,
 )
 
 logger = init_logger(__name__)
@@ -51,13 +70,19 @@ logger = init_logger(__name__)
 # so the two align. Constant across a coordinator's lifetime.
 POLY_BASE = np.uint64(0x9E3779B97F4A7C15)
 
+# Table size = smallest power of two >= _TABLE_GROWTH * live entries, keeping
+# the load factor (and thus bucket-collision recall loss) low for a few
+# bytes/chunk.
+_TABLE_GROWTH = 4
+_MIN_TABLE_SIZE = 1 << 10
+
 
 @dataclass
 class StoreRange:
     """One stored token range published by a blend server.
 
     Attributes:
-        model_scope: Reuse-compatibility scope, ``f"{model_name}@{cache_salt}"``.
+        model_scope: Reuse-compatibility scope (the model name).
         tokens: The stored tokens (``token_ids[start:end]``). The coordinator
             chunks these at ``chunk_size`` and hashes each chunk.
         object_keys: Shared-L2 storage key (hex of the ObjectKey chunk hash) per
@@ -95,13 +120,94 @@ class GlobalMatch:
     cur_st: int
 
 
+class _ScopeTable:
+    """One scope's fingerprint table, mutated in place under ``lock``.
+
+    Mirrors the local matcher: ``slots`` direct-addresses the low bits of a poly
+    hash to a compact entry id (``-1`` = empty, last writer wins on collision);
+    ``hashes[cid]`` / ``locs[cid]`` hold the full hash (collision rejection) and
+    chunk location. ``poly_to_cid`` gives idempotent insert and eviction lookup.
+    Eviction tombstones ``locs[cid]``; a rebuild (on load-factor growth, or when
+    tombstones outnumber live entries) re-creates ``slots`` compacted.
+
+    All methods require the caller to hold ``lock``.
+    """
+
+    def __init__(self) -> None:
+        """Initialize an empty table at the minimum size."""
+        self.lock = threading.Lock()
+        self.poly_to_cid: dict[int, int] = {}
+        self.hashes: list[int] = []
+        self.locs: "list[_ChunkLoc | None]" = []
+        self.slots: np.ndarray = np.full(_MIN_TABLE_SIZE, -1, dtype=np.int64)
+        self.mask: np.uint64 = np.uint64(_MIN_TABLE_SIZE - 1)
+
+    def insert(self, poly: int, loc: _ChunkLoc) -> bool:
+        """Insert one fingerprint, growing the table when needed.
+
+        Args:
+            poly: The chunk's full 64-bit poly hash.
+            loc: The chunk's storage location.
+
+        Returns:
+            ``True`` if newly inserted, ``False`` if the hash was present.
+        """
+        if poly in self.poly_to_cid:
+            return False
+        cid = len(self.hashes)
+        self.hashes.append(poly)
+        self.locs.append(loc)
+        self.poly_to_cid[poly] = cid
+        self.slots[poly & int(self.mask)] = cid
+        if _TABLE_GROWTH * len(self.poly_to_cid) > self.slots.shape[0]:
+            self._rebuild()
+        return True
+
+    def evict(self, poly: int) -> bool:
+        """Tombstone one fingerprint, compacting when tombstones dominate.
+
+        Args:
+            poly: The chunk's full 64-bit poly hash.
+
+        Returns:
+            ``True`` if evicted, ``False`` if the hash was absent.
+        """
+        cid = self.poly_to_cid.pop(poly, None)
+        if cid is None:
+            return False
+        slot = poly & int(self.mask)
+        if self.slots[slot] == cid:  # a colliding later insert may own the slot
+            self.slots[slot] = -1
+        self.locs[cid] = None
+        if len(self.poly_to_cid) < len(self.locs) // 2:
+            self._rebuild()
+        return True
+
+    def _rebuild(self) -> None:
+        """Re-create ``slots`` sized to the live entries, dropping tombstones."""
+        live = [(poly, self.locs[cid]) for poly, cid in self.poly_to_cid.items()]
+        size = _MIN_TABLE_SIZE
+        while size < _TABLE_GROWTH * len(live):
+            size <<= 1
+        self.hashes = [poly for poly, _ in live]
+        self.locs = [loc for _, loc in live]
+        self.poly_to_cid = {poly: cid for cid, (poly, _) in enumerate(live)}
+        self.slots = np.full(size, -1, dtype=np.int64)
+        self.mask = np.uint64(size - 1)
+        if live:
+            update_table_id_numba(
+                np.array(self.hashes, dtype=np.uint64),
+                self.slots,
+                np.arange(len(live), dtype=np.int64),
+            )
+
+
 class GlobalBlendMatcher:
     """Thread-safe fleet-wide chunk fingerprint directory.
 
-    Hashes published token ranges into a poly-hash table and matches request
-    tokens by a strided rolling-hash probe. All public methods take the internal
-    lock, so the directory stays consistent under concurrent
-    publish/query/evict.
+    Hashes published token ranges into per-scope direct-address tables, mutated
+    in place, and matches request tokens with a vectorized strided rolling-hash
+    probe under the scope's lock.
     """
 
     def __init__(self, chunk_size: int = 256, probe_stride: int = 1) -> None:
@@ -123,11 +229,28 @@ class GlobalBlendMatcher:
             raise ValueError(f"probe_stride must be >= 1, got {probe_stride}")
         self._chunk_size = chunk_size
         self._probe_stride = probe_stride
+        # Top-level lock: guards _scopes and _by_key only (cheap, short holds).
         self._lock = threading.Lock()
-        self._index: dict[tuple[str, int], _ChunkLoc] = {}
+        self._scopes: dict[str, _ScopeTable] = {}
         # Reverse map for eviction: object_key -> its (scope, poly_hash) keys.
         self._by_key: dict[str, list[tuple[str, int]]] = {}
 
+    def _get_or_create_scope(self, model_scope: str) -> _ScopeTable:
+        """Return the table for ``model_scope``, creating it if absent.
+
+        Args:
+            model_scope: The reuse scope to fetch a table for.
+
+        Returns:
+            The (possibly newly created) per-scope table.
+        """
+        with self._lock:
+            table = self._scopes.get(model_scope)
+            if table is None:
+                table = _ScopeTable()
+                self._scopes[model_scope] = table
+            return table
+
     def register(self, ranges: list[StoreRange]) -> int:
         """Hash and insert published token ranges (idempotent per chunk).
 
@@ -148,10 +271,10 @@ class GlobalBlendMatcher:
             Number of chunk fingerprints newly inserted (excludes idempotent
             skips).
         """
-        # Hash every range outside the lock; keep only well-formed ranges.
+        # Hash every range outside any lock; keep only well-formed ranges.
         prepared: list[tuple[str, np.ndarray, list[str], int]] = []
         for rng in ranges:
-            arr = np.array(rng.tokens, dtype=np.uint64)
+            arr = np.asarray(rng.tokens, dtype=np.uint64)
             polys = chunk_hash_windows_numba(arr, self._chunk_size, POLY_BASE)
             n_chunks = int(polys.shape[0])
             if n_chunks != len(rng.object_keys):
@@ -169,18 +292,22 @@ class GlobalBlendMatcher:
             prepared.append((rng.model_scope, polys, rng.object_keys, rng.old_st_base))
 
         inserted = 0
-        with self._lock:
-            for model_scope, polys, object_keys, old_st_base in prepared:
+        for model_scope, polys, object_keys, old_st_base in prepared:
+            table = self._get_or_create_scope(model_scope)
+            new_keys: list[tuple[str, int]] = []
+            with table.lock:
                 for i in range(len(object_keys)):
-                    key = (model_scope, int(polys[i]))
-                    if key in self._index:
-                        continue
-                    object_key = object_keys[i]
-                    self._index[key] = _ChunkLoc(
-                        object_key, old_st_base + i * self._chunk_size
-                    )
-                    self._by_key.setdefault(object_key, []).append(key)
-                    inserted += 1
+                    poly = int(polys[i])
+                    loc = _ChunkLoc(object_keys[i], old_st_base + i * self._chunk_size)
+                    if table.insert(poly, loc):
+                        new_keys.append((object_keys[i], poly))
+            if new_keys:
+                inserted += len(new_keys)
+                with self._lock:
+                    for object_key, poly in new_keys:
+                        self._by_key.setdefault(object_key, []).append(
+                            (model_scope, poly)
+                        )
         return inserted
 
     def remove(self, object_keys: list[str]) -> int:
@@ -192,45 +319,74 @@ class GlobalBlendMatcher:
         Returns:
             Number of fingerprint entries removed.
         """
-        removed = 0
+        # Collect keys under the top-level lock, then mutate per scope.
+        by_scope: dict[str, list[int]] = defaultdict(list)
         with self._lock:
             for object_key in object_keys:
-                for key in self._by_key.pop(object_key, []):
-                    if self._index.pop(key, None) is not None:
+                for model_scope, poly in self._by_key.pop(object_key, []):
+                    by_scope[model_scope].append(poly)
+
+        removed = 0
+        for model_scope, polys in by_scope.items():
+            with self._lock:
+                table = self._scopes.get(model_scope)
+            if table is None:
+                continue
+            with table.lock:
+                for poly in polys:
+                    if table.evict(poly):
                         removed += 1
         return removed
 
-    def match(self, model_scope: str, tokens: list[int]) -> list[GlobalMatch]:
+    def match(
+        self, model_scope: str, tokens: "list[int] | np.ndarray"
+    ) -> list[GlobalMatch]:
         """Match request tokens against the directory.
 
-        Rolls a chunk-window hash over the request and probes the table every
-        ``probe_stride`` positions; a hit is an exact 64-bit poly match (the dict
-        key is the full hash). De-duplicates by ``object_key``. Mirrors
+        Rolls a chunk-window hash over the request, then probes the scope's
+        direct-address table every ``probe_stride`` positions in one numpy
+        gather; a full 64-bit re-check in the sparse hit loop rejects bucket
+        collisions. De-duplicates by ``object_key``. Mirrors the local
         ``BlendTokenRangeMatcherV3.match_sub_sequence``.
 
         Args:
-            model_scope: Scope to match within (``f"{model_name}@{cache_salt}"``).
-            tokens: The request tokens.
+            model_scope: Scope to match within (the model name).
+            tokens: The request tokens (a ``list[int]`` or a ``uint64`` array).
 
         Returns:
             Matches in ascending ``cur_st`` order; empty if nothing matched.
         """
-        if len(tokens) < self._chunk_size:
+        arr = np.asarray(tokens, dtype=np.uint64)
+        if arr.shape[0] < self._chunk_size:
             return []
-        arr = np.array(tokens, dtype=np.uint64)
+
+        with self._lock:
+            table = self._scopes.get(model_scope)
+        if table is None:
+            return []
+
         rolling = rolling_hash_windows_numba(arr, self._chunk_size, POLY_BASE)
-        n_positions = int(rolling.shape[0])
+        probe = rolling[:: self._probe_stride]
         matches: list[GlobalMatch] = []
         seen: set[str] = set()
-        with self._lock:
-            for q_pos in range(0, n_positions, self._probe_stride):
-                loc = self._index.get((model_scope, int(rolling[q_pos])))
+        stride = self._probe_stride
+        with table.lock:
+            # One gather: strided hash -> slot's entry index (-1 = empty slot).
+            entry_ids = table.slots[probe & table.mask]
+            hit_positions = np.nonzero(entry_ids >= 0)[0]
+            for p in hit_positions.tolist():
+                cid = int(entry_ids[p])
+                if int(probe[p]) != table.hashes[cid]:
+                    continue  # bucket collision: different content in this slot
+                loc = table.locs[cid]
                 if loc is None or loc.object_key in seen:
-                    continue
+                    continue  # evicted, or already matched
                 seen.add(loc.object_key)
                 matches.append(
                     GlobalMatch(
-                        object_key=loc.object_key, old_st=loc.old_st, cur_st=q_pos
+                        object_key=loc.object_key,
+                        old_st=loc.old_st,
+                        cur_st=p * stride,
                     )
                 )
         return matches

lmcache/v1/mp_coordinator/http_apis/blend_directory_api.py

@@ -24,6 +24,7 @@ from lmcache.v1.mp_coordinator.schemas import (
     BlendMatchRequest,
     BlendMatchResponse,
     GlobalMatchModel,
+    decode_tokens,
 )
 
 logger = init_logger(__name__)
@@ -89,12 +90,15 @@ def evict_fingerprints(body: BlendEvictRequest, request: Request) -> BlendEvictR
 def match_fingerprints(body: BlendMatchRequest, request: Request) -> BlendMatchResponse:
     """Match a request's rolling-hash array against the directory.
 
+    The request tokens arrive base64-packed (``tokens_b64``); they are decoded
+    to a ``uint64`` array and handed straight to the matcher.
+
     Returns:
         Matched chunks (``object_key`` / ``old_st`` / ``cur_st``), ascending by
         ``cur_st``.
     """
     directory = _directory(request)
-    matches = directory.match(body.model_scope, body.tokens)
+    matches = directory.match(body.model_scope, decode_tokens(body.tokens_b64))
     return BlendMatchResponse(
         matches=[
             GlobalMatchModel(object_key=m.object_key, old_st=m.old_st, cur_st=m.cur_st)

lmcache/v1/mp_coordinator/schemas.py

@@ -10,14 +10,58 @@ bodies and parse replies, so both sides agree on the schema in one place.
 # Standard
 from enum import Enum
 from typing import Annotated
+import base64
 
 # Third Party
-from pydantic import BaseModel, Field, StringConstraints
+from pydantic import BaseModel, Field, StringConstraints, field_validator
+import numpy as np
 
 # First Party
 from lmcache.v1.distributed.api import EncodedObjectKey  # noqa: F401  re-exported
 
 
+def encode_tokens(tokens: "list[int] | np.ndarray") -> str:
+    """Encode token ids into a compact base64 wire string.
+
+    Token ids fit in ``uint32``, so a little-endian ``uint32`` buffer is far
+    smaller than a JSON integer list and decodes in one ``np.frombuffer`` call.
+
+    Args:
+        tokens: Token ids (a ``list[int]`` or any array castable to ``uint32``).
+
+    Returns:
+        Base64 of the little-endian ``uint32`` token buffer.
+    """
+    arr = np.ascontiguousarray(np.asarray(tokens, dtype="<u4"))
+    return base64.b64encode(arr.tobytes()).decode("ascii")
+
+
+def decode_tokens(tokens_b64: str) -> np.ndarray:
+    """Decode a base64 token string produced by :func:`encode_tokens`.
+
+    Args:
+        tokens_b64: Base64 of a little-endian ``uint32`` token buffer.
+
+    Returns:
+        A ``uint64`` array of token ids (widened so it feeds the hashers
+        directly).
+
+    Raises:
+        ValueError: If ``tokens_b64`` is not valid base64 or not a multiple of
+            4 bytes.
+    """
+    try:
+        raw = base64.b64decode(tokens_b64, validate=True)
+    except Exception as exc:
+        raise ValueError(f"tokens_b64 is not valid base64: {exc}") from exc
+    if len(raw) % 4 != 0:
+        raise ValueError(
+            f"tokens_b64 byte length {len(raw)} is not a multiple of 4 "
+            "(malformed uint32 token buffer)"
+        )
+    return np.frombuffer(raw, dtype="<u4").astype(np.uint64)
+
+
 class RegisterRequest(BaseModel):
     """Body of a ``POST /instances`` registration request.
 
@@ -179,7 +223,7 @@ class StoreRangeModel(BaseModel):
     chunk ``i`` maps to ``object_keys[i]`` at ``old_st_base + i * chunk_size``.
 
     Attributes:
-        model_scope: ``f"{model_name}@{cache_salt}"`` reuse scope.
+        model_scope: Reuse scope (the model name).
         tokens: The stored tokens (``token_ids[start:end]``).
         object_keys: Shared-L2 storage key (hex) per chunk, in order.
         old_st_base: Token position of the range's first token.
@@ -236,11 +280,34 @@ class BlendMatchRequest(BaseModel):
 
     Attributes:
         model_scope: Scope to match within.
-        tokens: The request tokens (the coordinator hashes and probes them).
+        tokens_b64: The request tokens, packed via :func:`encode_tokens`
+            (base64 little-endian ``uint32``).
     """
 
     model_scope: str
-    tokens: list[int] = Field(default_factory=list)
+    tokens_b64: str = ""
+
+    @field_validator("tokens_b64")
+    @classmethod
+    def _validate_tokens_b64(cls, value: str) -> str:
+        """Reject a malformed token buffer at request validation.
+
+        Without this, ``decode_tokens`` would raise ``ValueError`` inside the
+        route handler, which FastAPI surfaces as a 500 (server error) for what
+        is really bad client input. Validating here returns a 422 instead.
+
+        Args:
+            value: The base64 ``tokens_b64`` field.
+
+        Returns:
+            The unchanged value once it is confirmed decodable.
+
+        Raises:
+            ValueError: If ``value`` is not valid base64 or not a whole number
+                of ``uint32`` tokens (surfaced by FastAPI as 422).
+        """
+        decode_tokens(value)
+        return value
 
 
 class GlobalMatchModel(BaseModel):

lmcache/v1/multiprocess/modules/blend_v3.py

@@ -1224,7 +1224,7 @@ class BlendV3Module(InstanceLivenessTarget):
         if coordinator is None or not chunk_hashes:
             return
         try:
-            model_scope = f"{key.model_name}@{key.cache_salt}"
+            model_scope = key.model_name
             store_range = {
                 "model_scope": model_scope,
                 "tokens": list(tokens_in_range),
@@ -1256,8 +1256,7 @@ class BlendV3Module(InstanceLivenessTarget):
             tokens = list(key.token_ids)
             if len(tokens) < self._ctx.chunk_size:
                 return False
-            model_scope = f"{key.model_name}@{key.cache_salt}"
-            coordinator.submit_match(key.request_id, model_scope, tokens)
+            coordinator.submit_match(key.request_id, key.model_name, tokens)
             return True
         except Exception:
             logger.warning(
@@ -1272,10 +1271,9 @@ class BlendV3Module(InstanceLivenessTarget):
 
         Mirrors the prefix/sparse legs: ``return None`` to defer while pending.
         A per-lookup wall-clock deadline (``job.coord_deadline``) bounds the
-        total wait: the daemon services match queries serially, so queue backlog
-        can keep a query ``PENDING`` well past one HTTP round-trip. Once the
-        deadline passes, the leg is abandoned and the lookup proceeds local-only
-        (the daemon's later fill, if any, is dropped via ``take_match``).
+        total wait, including queue/pool time. Past the deadline the leg is
+        abandoned and the lookup proceeds local-only (the client's later fill,
+        if any, is dropped via ``take_match``).
 
         Args:
             job: The per-request poll state.

{lmcache_cli-0.4.8.dev38.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lmcache-cli
-Version: 0.4.8.dev38
+Version: 0.4.8.dev42
 Summary: A LLM serving engine extension to reduce TTFT and increase throughput, especially under long-context scenarios.
 Author-email: LMCache Team <lmcacheteam@gmail.com>
 License-Expression: Apache-2.0

{lmcache_cli-0.4.8.dev38.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/RECORD

@@ -1,5 +1,5 @@
 lmcache/__init__.py,sha256=C7vwCTEvGEP_ca17O7YpXxz7Kqz-nPOHDPjCdzCdzRM,3958
-lmcache/_version.py,sha256=jJS7CZ5JP6NMPW0z0tMjYwZ0Dhm2V1JDscwM-L_lWlg,543
+lmcache/_version.py,sha256=GD_w2b5I14ipJ41mA7IkKCtO0jiZLuHNRDucf6M0szo,543
 lmcache/banner.py,sha256=fQFq8TX_OODhxu-zv4UAMPcbF6FizY2QF1zJPYU88mo,4070
 lmcache/connections.py,sha256=5bSguAPmkJCRpgjkZd0ugGna6Zfm0pXG9vo-RGVXeFg,4965
 lmcache/logging.py,sha256=7qOcguB6x0mr6TlyWvwcRS3Yk6DVfTq0ZM7ACh5DaT4,2662
@@ -339,14 +339,14 @@ lmcache/v1/lookup_client/record_strategies/memory_bloom_filter.py,sha256=J62TNb7
 lmcache/v1/mp_coordinator/__init__.py,sha256=Z_XPvBbsaFmUcLWipthjjBduoDkjvqD3FfcV115Im_s,943
 lmcache/v1/mp_coordinator/__main__.py,sha256=p962Qf2YipEW6xRLo3hmpzAbb5PPI9aZ269Rg_MxlnQ,769
 lmcache/v1/mp_coordinator/app.py,sha256=Zd9ffQu6GopQpq7Q7URNdJFAGJUFho1SJerTcT_elFo,6578
-lmcache/v1/mp_coordinator/blend_client.py,sha256=V5OYF6RzkTaCKdoq1U0LsIC2LjjHPoUR79PWKW3mccU,9691
-lmcache/v1/mp_coordinator/blend_directory.py,sha256=abK1SeyqWt9xsFxPcD0lCsObmPrLTEoPqnOzpEMWjCU,9517
+lmcache/v1/mp_coordinator/blend_client.py,sha256=_eMPadcPK_ID7NbU_Bvlwmdwdkz-eaJA6gzp9pBQIAU,10560
+lmcache/v1/mp_coordinator/blend_directory.py,sha256=KGzaM_os1msn41GjOQ8POG2froXwXthY3aIDuCWBYmU,15865
 lmcache/v1/mp_coordinator/config.py,sha256=Tk1tKFmBXAqcWvhfFhf6jKLU9xfO5UxrijKOcQfOodo,6416
 lmcache/v1/mp_coordinator/registrar.py,sha256=5a_tgtm3CYvYPLhLkAvAMn4b_W9swKHss8TLknEv8LA,5073
 lmcache/v1/mp_coordinator/registry.py,sha256=dYtRtW7tXuz6udGI5YfRQzBsOzY82hazPhbEU19lw9Y,5915
-lmcache/v1/mp_coordinator/schemas.py,sha256=1Z2rITRm5jrXDz1vqLDqybj49cZaEpO2nVJPgfhidXA,7392
+lmcache/v1/mp_coordinator/schemas.py,sha256=Mou0HD6cXbr26ebQrlW4HhB6VAbhmyvhxDQaab0Y8_A,9645
 lmcache/v1/mp_coordinator/http_apis/__init__.py,sha256=fRZK22mUV9kw77LXySq4cNgL7eg2JBVu8zuWhfhxREo,235
-lmcache/v1/mp_coordinator/http_apis/blend_directory_api.py,sha256=nK0Ba7s83SMV8Zj_hU7Bw7HV2b5C3CLxnHv6LFarY7E,3044
+lmcache/v1/mp_coordinator/http_apis/blend_directory_api.py,sha256=kNumJuPk7yHqf9yAFOTNeQejpduMnizrxhqAuqHILTM,3224
 lmcache/v1/mp_coordinator/http_apis/health_api.py,sha256=8BcdUAhP_wIDC4hs1eTBQ2e01U8PRlO9jwRWv6OYQ30,380
 lmcache/v1/mp_coordinator/http_apis/instances_api.py,sha256=ZQLFJ6_4QTR043CB1BmIE8rWAnlVN0OD_feWOyWagKk,4574
 lmcache/v1/mp_coordinator/http_apis/l2_api.py,sha256=H2vuwfnT-jnn-zrwd-K-bm8P5NM8s0lwWOIhY2-q9HM,6205
@@ -427,7 +427,7 @@ lmcache/v1/multiprocess/http_apis/status_api.py,sha256=0v-dqktWjs9thc5FmRykOPaGc
 lmcache/v1/multiprocess/http_apis/version_api.py,sha256=A_haZ4cMmUp37aU3Qe4QVR1rgxnJFv88aRqILhLMq5Y,151
 lmcache/v1/multiprocess/modules/__init__.py,sha256=bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E,38
 lmcache/v1/multiprocess/modules/blend.py,sha256=NcqAG6j045gVK2lUQv1hTtQfzQdoe0YPo3fMiB56kU8,46972
-lmcache/v1/multiprocess/modules/blend_v3.py,sha256=9TkVNJ60QHR_J0fhcJN4wZDHFbp6DrPcCnbrZZl8ytE,74710
+lmcache/v1/multiprocess/modules/blend_v3.py,sha256=TV4tcwmaEdQXRYMJpX-kA8bKk_DjBilung6t9c4yUSc,74518
 lmcache/v1/multiprocess/modules/engine_driven_transfer.py,sha256=sc0I40fWE9DhRoilvryb2LHOyd8vTv1dc5PM9lu5IHY,19193
 lmcache/v1/multiprocess/modules/lmcache_driven_transfer.py,sha256=lBvu_VzmuZnxryo7mjGo0dr1dfP_rbIwGK0r83Sr_lY,43244
 lmcache/v1/multiprocess/modules/lookup.py,sha256=3RKwuMLEHBHEbEyLKSD85kuRhWRNDLQS4kr_2SaTIsk,16850
@@ -572,9 +572,9 @@ lmcache/v1/utils/cache_utils.py,sha256=O4RO_BRAOXXPwc3SqV_osq66Rnf6U733gUlbiEHwE
 lmcache/v1/utils/json_utils.py,sha256=WFrdauddZP8xuOqKqkhFlYyt3cNH-krNtVCRe4jIxeY,1724
 lmcache/v1/utils/router_discovery.py,sha256=yZR3oTovQesNQzfa6EboyYPj798WVvUxZRJ6l4L-9bk,1867
 lmcache/v1/utils/subclass_discovery.py,sha256=HmvDIEf8QI7GVeNzD-06zK8McjyX8yFMJNAOaR_85GU,9159
-lmcache_cli-0.4.8.dev38.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lmcache_cli-0.4.8.dev38.dist-info/METADATA,sha256=89s3_qlQDNUmBlLf9Lg4UNJTtMDly3H_nBWiWe83tjw,11289
-lmcache_cli-0.4.8.dev38.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-lmcache_cli-0.4.8.dev38.dist-info/entry_points.txt,sha256=Axef26i3pyPHL-zuNZWCubmM74zLa9tQnoMU1JrAZUg,50
-lmcache_cli-0.4.8.dev38.dist-info/top_level.txt,sha256=7KnuVYNl7HkrnKK3wAN5qJJQSOcCv-l0TW4bzvH_SaU,8
-lmcache_cli-0.4.8.dev38.dist-info/RECORD,,
+lmcache_cli-0.4.8.dev42.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lmcache_cli-0.4.8.dev42.dist-info/METADATA,sha256=aX8MMhA_lVMrMLC8m8Wji6eT_DsHTrl6BeWQ3ZvY6MQ,11289
+lmcache_cli-0.4.8.dev42.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+lmcache_cli-0.4.8.dev42.dist-info/entry_points.txt,sha256=Axef26i3pyPHL-zuNZWCubmM74zLa9tQnoMU1JrAZUg,50
+lmcache_cli-0.4.8.dev42.dist-info/top_level.txt,sha256=7KnuVYNl7HkrnKK3wAN5qJJQSOcCv-l0TW4bzvH_SaU,8
+lmcache_cli-0.4.8.dev42.dist-info/RECORD,,