lmcache-kvblockd 0.2.0__tar.gz

lmcache_kvblockd-0.2.0/.gitignore

@@ -0,0 +1,56 @@
+# ── PRIVATE RESEARCH VAULT — never commit ─────────────────────────
+# All competitive research, strategy, business plans, drafts, and course material.
+research/
+
+# OS noise
+.DS_Store
+
+# Build artifacts
+bin/
+dist/
+*.prof
+*.test
+testdata/corpus-large/
+
+# External build dependencies fetched at build time (never vendored):
+# ai-dynamo/nixl checkout for the adapters/nixl plugin (see its README).
+build-deps/
+adapters/nixl/build/
+
+# Local env
+.env
+
+# Soak ARTIFACTS (logs, JSONL, pprof snapshots) — never the driver source
+/soak
+/soak.jsonl
+bench/rigs/aws-soak/results/
+bench/rigs/aws-dramgate/allocs-*.pb.gz
+
+# Claude session rules — private, never push
+CLAUDE.md
+
+# Compiled command binaries accidentally built into the tree
+/xferspike
+/kvblockd
+/kvbctl
+cmd/xferspike/xferspike
+
+# Python
+__pycache__/
+*.pyc
+.pytest_cache/
+.ruff_cache/
+*.egg-info/
+python/**/dist/
+
+# mutation-testing artifacts
+report.json
+bench/rigs/*/.rig-state
+
+# locally built bench binaries
+/kvbench
+/crash
+/getbench
+internal/store/modeltest/testdata/rapid/
+bench/rigs/aws-s3soak/results/
+.claude/

lmcache_kvblockd-0.2.0/PKG-INFO

@@ -0,0 +1,88 @@
+Metadata-Version: 2.5
+Name: lmcache-kvblockd
+Version: 0.2.0
+Summary: LMCache remote-storage backend backed by kvblockd
+Project-URL: Homepage, https://kvstash.com
+Project-URL: Documentation, https://kvstash.com/integrations/lmcache/
+Project-URL: Get the daemon, https://kvstash.com/docs/install/
+Author-email: Rudray Mehra <rudraymehra@gmail.com>
+License-Expression: Apache-2.0
+Requires-Python: >=3.10
+Requires-Dist: kvblockd>=0.2.0
+Requires-Dist: lmcache<0.6,>=0.5.1
+Requires-Dist: numpy
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == 'test'
+Requires-Dist: torch; extra == 'test'
+Description-Content-Type: text/markdown
+
+# lmcache-kvblockd
+
+[LMCache](https://github.com/LMCache/LMCache) remote storage plugin for
+[KVStash](https://kvstash.com). vLLM deployments that already run LMCache can
+keep KV in the KVStash daemon (`kvblockd://` scheme) and reload it after a
+restart or from another server.
+
+- Get the daemon (free download): https://kvstash.com/docs/install/
+- Full guide: https://kvstash.com/integrations/lmcache/
+
+Linux x86-64 only (LMCache ships manylinux wheels only).
+
+## Install
+
+```sh
+pip install lmcache-kvblockd
+```
+
+## Configure
+
+```sh
+PYTHONHASHSEED=0 vllm serve MODEL --kv-transfer-config '{
+  "kv_connector": "LMCacheConnectorV1", "kv_role": "kv_both",
+  "kv_connector_extra_config": {
+    "lmcache.remote_storage_plugins": "kvblockd",
+    "lmcache.local_cpu": true,
+    "lmcache.chunk_size": 256,
+    "lmcache.extra_config": {
+      "kvblockd_token": "YOUR_TOKEN",
+      "remote_storage_plugin.kvblockd.module_path": "lmcache_kvblockd.adapter",
+      "remote_storage_plugin.kvblockd.class_name": "KvblockdConnectorAdapter",
+      "remote_storage_plugin.kvblockd.url": "kvblockd://STORE_HOST:9440?namespace=lmcache&streams=4"
+    }
+  }
+}'
+```
+
+- The address goes in `remote_storage_plugin.kvblockd.url`. LMCache ignores
+  `remote_url` when `remote_storage_plugins` is set, and setting both creates
+  a second backend that doubles every write.
+- Set `PYTHONHASHSEED=0` on every worker. LMCache's chunk keys depend on it,
+  so workers with different seeds cannot share cache.
+- Use a separate namespace per model revision, dtype and cache layout.
+
+## LMCache's CUDA extension is required on vLLM 0.33 and newer
+
+Without its compiled `lmcache.cuda_ops` extension, LMCache copies KV between
+GPU and CPU with a torch fallback that does not handle the fused K/V layout of
+vLLM 0.33. The bytes it hands the plugin are then not the KV, and a reload
+gives wrong answers. When CUDA is present, the extension does not import and
+vLLM is 0.33 or newer, the plugin refuses to start and says why; LMCache keeps
+serving without the remote tier. Check before you deploy:
+
+```sh
+python -c "import lmcache.cuda_ops"
+```
+
+If that fails, install an LMCache build compiled for your torch version.
+
+## Behaviour notes
+
+- Cache tags must have string names and values without `@` or `%` (LMCache's
+  text keys do not escape them). Unsupported tags become cache misses.
+- GET and lookup deadlines include queueing and connection set-up.
+- Lookup with `pin=True` reports only the leading run of keys whose
+  time-limited leases were acknowledged. A lease can expire; LMCache's full
+  asynchronous pin/unpin lifecycle is not implemented.
+- Records with malformed headers or contradictory tensor sizes are removed so
+  they can be written again. Other read problems stay misses without deleting
+  valid records. Do not share a namespace between incompatible writers.

lmcache_kvblockd-0.2.0/README.md

@@ -0,0 +1,70 @@
+# lmcache-kvblockd
+
+[LMCache](https://github.com/LMCache/LMCache) remote storage plugin for
+[KVStash](https://kvstash.com). vLLM deployments that already run LMCache can
+keep KV in the KVStash daemon (`kvblockd://` scheme) and reload it after a
+restart or from another server.
+
+- Get the daemon (free download): https://kvstash.com/docs/install/
+- Full guide: https://kvstash.com/integrations/lmcache/
+
+Linux x86-64 only (LMCache ships manylinux wheels only).
+
+## Install
+
+```sh
+pip install lmcache-kvblockd
+```
+
+## Configure
+
+```sh
+PYTHONHASHSEED=0 vllm serve MODEL --kv-transfer-config '{
+  "kv_connector": "LMCacheConnectorV1", "kv_role": "kv_both",
+  "kv_connector_extra_config": {
+    "lmcache.remote_storage_plugins": "kvblockd",
+    "lmcache.local_cpu": true,
+    "lmcache.chunk_size": 256,
+    "lmcache.extra_config": {
+      "kvblockd_token": "YOUR_TOKEN",
+      "remote_storage_plugin.kvblockd.module_path": "lmcache_kvblockd.adapter",
+      "remote_storage_plugin.kvblockd.class_name": "KvblockdConnectorAdapter",
+      "remote_storage_plugin.kvblockd.url": "kvblockd://STORE_HOST:9440?namespace=lmcache&streams=4"
+    }
+  }
+}'
+```
+
+- The address goes in `remote_storage_plugin.kvblockd.url`. LMCache ignores
+  `remote_url` when `remote_storage_plugins` is set, and setting both creates
+  a second backend that doubles every write.
+- Set `PYTHONHASHSEED=0` on every worker. LMCache's chunk keys depend on it,
+  so workers with different seeds cannot share cache.
+- Use a separate namespace per model revision, dtype and cache layout.
+
+## LMCache's CUDA extension is required on vLLM 0.33 and newer
+
+Without its compiled `lmcache.cuda_ops` extension, LMCache copies KV between
+GPU and CPU with a torch fallback that does not handle the fused K/V layout of
+vLLM 0.33. The bytes it hands the plugin are then not the KV, and a reload
+gives wrong answers. When CUDA is present, the extension does not import and
+vLLM is 0.33 or newer, the plugin refuses to start and says why; LMCache keeps
+serving without the remote tier. Check before you deploy:
+
+```sh
+python -c "import lmcache.cuda_ops"
+```
+
+If that fails, install an LMCache build compiled for your torch version.
+
+## Behaviour notes
+
+- Cache tags must have string names and values without `@` or `%` (LMCache's
+  text keys do not escape them). Unsupported tags become cache misses.
+- GET and lookup deadlines include queueing and connection set-up.
+- Lookup with `pin=True` reports only the leading run of keys whose
+  time-limited leases were acknowledged. A lease can expire; LMCache's full
+  asynchronous pin/unpin lifecycle is not implemented.
+- Records with malformed headers or contradictory tensor sizes are removed so
+  they can be written again. Other read problems stay misses without deleting
+  valid records. Do not share a namespace between incompatible writers.

lmcache_kvblockd-0.2.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "lmcache-kvblockd"
+version = "0.2.0"
+description = "LMCache remote-storage backend backed by kvblockd"
+requires-python = ">=3.10"
+readme = "README.md"
+license = "Apache-2.0"
+authors = [{ name = "Rudray Mehra", email = "rudraymehra@gmail.com" }]
+dependencies = [
+    "kvblockd>=0.2.0",
+    "lmcache>=0.5.1,<0.6",
+    "numpy",  # zero-copy MemoryObj views via tensor.numpy() (torch does not pull it)
+]
+
+[project.urls]
+Homepage = "https://kvstash.com"
+Documentation = "https://kvstash.com/integrations/lmcache/"
+"Get the daemon" = "https://kvstash.com/docs/install/"
+
+[project.optional-dependencies]
+test = ["pytest>=8", "torch"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lmcache_kvblockd"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"

lmcache_kvblockd-0.2.0/src/lmcache_kvblockd/__init__.py

@@ -0,0 +1 @@
+"""LMCache remote-storage backend backed by kvblockd."""

lmcache_kvblockd-0.2.0/src/lmcache_kvblockd/adapter.py

@@ -0,0 +1,127 @@
+"""KvblockdConnectorAdapter — registers the kvblockd:// scheme with LMCache's
+ConnectorManager (plugin path: remote_storage_plugins + extra_config
+module_path/class_name; see docs/notes/lmcache-head-verify.md).
+
+TWO URL shapes must parse, because LMCache 0.5.x dials remote backends two
+different ways (lmcache/v1/storage_backend/remote_backend.py, init_connection):
+
+  1. legacy `remote_url` backend  → url = "kvblockd://host:port?..."
+  2. plugin backend (the `remote_storage_plugins` path) → url =
+     "plugin://kvblockd" — a VIRTUAL url carrying no endpoint at all.
+
+An adapter that only matches kvblockd:// makes the plugin backend fail
+"No adapter found for URL: plugin://kvblockd"; RemoteBackend swallows that
+(connection stays None) and every op silently degrades to a miss — the
+GPU-box zero-bytes failure. For plugin:// the real endpoint comes from
+extra_config["remote_storage_plugin.kvblockd.url"] (falling back to
+config.remote_url when it is a kvblockd:// url). This mirrors how builtin
+adapters (fs_adapter.py) treat plugin:// URLs in lmcache 0.5.1.
+
+The LMCache ConnectorAdapter base is resolved at import time; if LMCache is
+absent (CI import checks, or unit tests without lmcache) we fall back to
+`object` so the module still imports and the class still instantiates."""
+
+from __future__ import annotations
+
+from .connector import make_connector
+
+try:
+    from lmcache.v1.storage_backend.connector import ConnectorAdapter as _Base
+except Exception:  # noqa: BLE001 — availability fallback: LMCache not installed — keep the module importable
+    _Base = object
+
+SCHEME = "kvblockd://"
+PLUGIN_SCHEME = "plugin://"
+PLUGIN_TYPE = "kvblockd"
+
+
+def _plugin_type(name: str) -> str:
+    """Mirror lmcache's extract_plugin_type: '{type}' or '{type}.{instance}'."""
+    return name.split(".", 1)[0]
+
+
+def unsafe_copy_path() -> str | None:
+    """Why storing through this LMCache would write wrong KV, or None.
+
+    Without its compiled lmcache.cuda_ops extension LMCache copies between GPU
+    and CPU with a torch fallback that has no branch for the fused K/V layout
+    vLLM 0.33 allocates (LBNHC), so the bytes it hands us are not the KV. The
+    daemon never overwrites a stored key, so such blocks would stay wrong
+    after LMCache is fixed: refuse instead.
+    """
+    try:
+        import torch
+
+        if not torch.cuda.is_available():
+            return None  # CPU-only processes (tests, CI) never take the GPU copy path
+    except Exception:  # noqa: BLE001 - no torch, no GPU copy path
+        return None
+    try:
+        import lmcache.cuda_ops  # noqa: F401
+        return None
+    except ImportError:
+        pass
+    try:
+        from importlib.metadata import version
+
+        major_minor = tuple(int(x) for x in version("vllm").split(".")[:2])
+    except Exception:  # noqa: BLE001 - unknown vLLM: keep the old behaviour
+        return None
+    if major_minor >= (0, 33):
+        return ("kvblockd plugin disabled: LMCache's compiled lmcache.cuda_ops extension is not "
+                "importable, and its torch fallback copies vLLM >= 0.33's fused K/V layout into the "
+                "wrong blocks. Install an LMCache build with its CUDA extension for this torch.")
+    return None
+
+
+class KvblockdConnectorAdapter(_Base):
+    """Matches kvblockd:// and plugin://kvblockd[.instance] URLs and builds a
+    KvblockdRemoteConnector."""
+
+    def __init__(self, schema: str = SCHEME):
+        try:
+            super().__init__(schema)  # ConnectorAdapter(schema) when present
+        except TypeError:
+            self.schema = schema  # object() base: set it ourselves
+
+    def can_parse(self, url: str) -> bool:
+        if url.startswith(SCHEME):
+            return True
+        if url.startswith(PLUGIN_SCHEME):
+            return _plugin_type(url[len(PLUGIN_SCHEME):]) == PLUGIN_TYPE
+        return False
+
+    def create_connector(self, context):
+        reason = unsafe_copy_path()
+        if reason:
+            # RemoteBackend.init_connection catches this and runs without the
+            # remote tier: nothing reaches the store, so nothing wrong persists.
+            raise RuntimeError(reason)
+        return make_connector(context, url=self._resolve_url(context))
+
+    @staticmethod
+    def _resolve_url(context) -> str:
+        """The real kvblockd:// endpoint for this context.
+
+        plugin:// URLs are virtual — the endpoint lives in config. Raising here
+        is deliberate and SAFE: RemoteBackend.init_connection catches it and
+        logs a warning, and a loud misconfiguration beats a silent no-op."""
+        url = getattr(context, "url", "") or ""
+        if url.startswith(SCHEME):
+            return url
+        cfg = getattr(context, "config", None)
+        extra = getattr(cfg, "extra_config", None) or {}
+        pname = getattr(context, "plugin_name", None) or url[len(PLUGIN_SCHEME):] or PLUGIN_TYPE
+        # Full instance name first ("kvblockd.primary"), then the bare type.
+        for name in dict.fromkeys((pname, _plugin_type(pname))):
+            candidate = extra.get(f"remote_storage_plugin.{name}.url")
+            if candidate:
+                return candidate
+        remote_url = getattr(cfg, "remote_url", None)
+        if remote_url and remote_url.startswith(SCHEME):
+            return remote_url
+        raise ValueError(
+            f"kvblockd plugin selected (url={url!r}) but no endpoint configured: set "
+            f"extra_config['remote_storage_plugin.{_plugin_type(pname)}.url'] = "
+            f"'kvblockd://HOST:PORT?namespace=...&streams=...'"
+        )