lmcache-cli 0.4.5.dev0__py3-none-any.whl

lmcache/v1/storage_backend/naive_serde/__init__.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: Apache-2.0
+# Standard
+from typing import Optional, Tuple
+
+# First Party
+from lmcache.v1.config import LMCacheEngineConfig
+from lmcache.v1.metadata import LMCacheMetadata
+from lmcache.v1.storage_backend.naive_serde.cachegen_decoder import CacheGenDeserializer
+from lmcache.v1.storage_backend.naive_serde.cachegen_encoder import CacheGenSerializer
+from lmcache.v1.storage_backend.naive_serde.kivi_serde import (
+    KIVIDeserializer,
+    KIVISerializer,
+)
+from lmcache.v1.storage_backend.naive_serde.naive_serde import (
+    NaiveDeserializer,
+    NaiveSerializer,
+)
+from lmcache.v1.storage_backend.naive_serde.serde import Deserializer, Serializer
+
+
+def CreateSerde(
+    serde_type: str,
+    metadata: LMCacheMetadata,
+    config: LMCacheEngineConfig,
+) -> Tuple[Serializer, Deserializer]:
+    s: Optional[Serializer] = None
+    d: Optional[Deserializer] = None
+
+    if serde_type == "naive":
+        s, d = NaiveSerializer(), NaiveDeserializer()
+    elif serde_type == "kivi":
+        s, d = KIVISerializer(), KIVIDeserializer()
+    elif serde_type == "cachegen":
+        s, d = (
+            CacheGenSerializer(config, metadata),
+            CacheGenDeserializer(config, metadata),
+        )
+    else:
+        raise ValueError(f"Invalid type: {serde_type}")
+
+    return s, d
+
+
+__all__ = [
+    "Serializer",
+    "Deserializer",
+    "KIVISerializer",
+    "KIVIDeserializer",
+    "CreateSerde",
+]

lmcache/v1/storage_backend/naive_serde/cachegen_basics.py

@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+# Standard
+from dataclasses import dataclass
+from typing import List
+
+# Third Party
+from transformers import AutoConfig
+
+# First Party
+from lmcache.logging import init_logger
+from lmcache.storage_backend.serde.cachegen_basics import QuantizationSpec
+
+logger = init_logger(__name__)
+
+
+@dataclass
+class CacheGenConfig:
+    # TODO: move this class to another file like "cachegen_basics.py"
+    nlayers: int
+    kspecs: List[QuantizationSpec]
+    vspecs: List[QuantizationSpec]
+
+    def __getitem__(self, key: str) -> int:
+        return getattr(self, key)
+
+    @staticmethod
+    def from_model_name(model_name: str) -> "CacheGenConfig":
+        family_7b = [
+            "mistralai/Mistral-7B-Instruct-v0.2",
+            "lmsys/longchat-7b-16k",
+            "Qwen/Qwen-7B",
+        ]
+        family_8b = ["meta-llama/Llama-3.1-8B-Instruct"]
+        family_9b = ["THUDM/glm-4-9b-chat"]
+        if model_name in family_7b:
+            return CacheGenConfig(
+                nlayers=32,
+                kspecs=[
+                    QuantizationSpec(start_layer=0, end_layer=10, bins=32),
+                    QuantizationSpec(start_layer=10, end_layer=32, bins=16),
+                ],
+                vspecs=[
+                    QuantizationSpec(start_layer=0, end_layer=2, bins=32),
+                    QuantizationSpec(start_layer=2, end_layer=32, bins=16),
+                ],
+            )
+        elif model_name in family_8b:
+            return CacheGenConfig(
+                nlayers=32,
+                kspecs=[
+                    QuantizationSpec(start_layer=0, end_layer=10, bins=32),
+                    QuantizationSpec(start_layer=10, end_layer=32, bins=16),
+                ],
+                vspecs=[
+                    QuantizationSpec(start_layer=0, end_layer=2, bins=32),
+                    QuantizationSpec(start_layer=2, end_layer=32, bins=16),
+                ],
+            )
+        # TODO(Jiayi): needs tuning for better quality
+        elif model_name in family_9b:
+            return CacheGenConfig(
+                nlayers=40,
+                kspecs=[
+                    QuantizationSpec(start_layer=0, end_layer=10, bins=32),
+                    QuantizationSpec(start_layer=10, end_layer=40, bins=16),
+                ],
+                vspecs=[
+                    QuantizationSpec(start_layer=0, end_layer=2, bins=32),
+                    QuantizationSpec(start_layer=2, end_layer=40, bins=16),
+                ],
+            )
+        elif model_name == "test_model":
+            return CacheGenConfig(
+                nlayers=32,
+                kspecs=[
+                    QuantizationSpec(start_layer=0, end_layer=10, bins=32),
+                    QuantizationSpec(start_layer=10, end_layer=32, bins=16),
+                ],
+                vspecs=[
+                    QuantizationSpec(start_layer=0, end_layer=2, bins=32),
+                    QuantizationSpec(start_layer=2, end_layer=32, bins=16),
+                ],
+            )
+        else:
+            try:
+                config = AutoConfig.from_pretrained(model_name)
+                # Default name caught by num_hidden_layers
+                if config.num_hidden_layers is None:
+                    raise ValueError(
+                        f"num_hidden_layers is None for model {model_name}"
+                    )
+                if config.num_hidden_layers < 10:
+                    return CacheGenConfig(
+                        nlayers=config.num_hidden_layers,
+                        kspecs=[
+                            QuantizationSpec(
+                                start_layer=0,
+                                end_layer=config.num_hidden_layers,
+                                bins=32,
+                            ),
+                        ],
+                        vspecs=[
+                            QuantizationSpec(
+                                start_layer=0,
+                                end_layer=config.num_hidden_layers,
+                                bins=32,
+                            ),
+                        ],
+                    )
+                else:
+                    return CacheGenConfig(
+                        nlayers=config.num_hidden_layers,
+                        kspecs=[
+                            QuantizationSpec(start_layer=0, end_layer=10, bins=32),
+                            QuantizationSpec(
+                                start_layer=10,
+                                end_layer=config.num_hidden_layers,
+                                bins=16,
+                            ),
+                        ],
+                        vspecs=[
+                            QuantizationSpec(start_layer=0, end_layer=2, bins=32),
+                            QuantizationSpec(
+                                start_layer=2,
+                                end_layer=config.num_hidden_layers,
+                                bins=16,
+                            ),
+                        ],
+                    )
+            except Exception as e:
+                raise ValueError(
+                    f"Model {model_name} not supported by CacheGenConfig"
+                ) from e

lmcache/v1/storage_backend/naive_serde/cachegen_decoder.py

@@ -0,0 +1,135 @@
+# SPDX-License-Identifier: Apache-2.0
+# Standard
+from typing import Optional
+
+# Third Party
+import torch
+
+# First Party
+from lmcache.logging import init_logger
+from lmcache.storage_backend.serde.cachegen_basics import (
+    CacheGenGPUEncoderOutput,
+)
+from lmcache.storage_backend.serde.cachegen_decoder import (
+    decode_function_gpu,
+    do_dequantize,
+)
+from lmcache.utils import _lmcache_nvtx_annotate
+from lmcache.v1.config import LMCacheEngineConfig
+from lmcache.v1.memory_management import (
+    BytesBufferMemoryObj,
+    MemoryFormat,
+    MemoryObj,
+    MemoryObjMetadata,
+    TensorMemoryObj,
+)
+from lmcache.v1.metadata import LMCacheMetadata
+from lmcache.v1.storage_backend.naive_serde.cachegen_basics import CacheGenConfig
+from lmcache.v1.storage_backend.naive_serde.serde import Deserializer
+
+logger = init_logger(__name__)
+
+
+class CacheGenDeserializer(Deserializer):
+    def __init__(self, config: LMCacheEngineConfig, metadata: LMCacheMetadata):
+        self.dtype = metadata.kv_dtype
+        self.cachegen_config = CacheGenConfig.from_model_name(metadata.model_name)
+        self.chunk_size = config.chunk_size
+        self.output_buffer: Optional[torch.Tensor] = None
+        self.key_bins = self.make_key_bins(self.cachegen_config)
+        self.value_bins = self.make_value_bins(self.cachegen_config)
+
+    def make_key_bins(self, config: CacheGenConfig) -> torch.Tensor:
+        ret = torch.zeros(config.nlayers)
+        for spec in config.kspecs:
+            ret[spec.start_layer : spec.end_layer] = spec.bins
+        return ret.cuda()
+
+    def make_value_bins(self, config: CacheGenConfig) -> torch.Tensor:
+        ret = torch.zeros(config.nlayers)
+        for spec in config.vspecs:
+            ret[spec.start_layer : spec.end_layer] = spec.bins
+        return ret.cuda()
+
+    def get_output_buffer(self, nlayers: int, nchannels: int, ntokens: int):
+        if (
+            self.output_buffer is None
+            or self.output_buffer.shape[1] != 2 * nlayers * nchannels
+        ):
+            self.output_buffer = torch.zeros(
+                (self.chunk_size, 2 * nlayers * nchannels), dtype=torch.uint8
+            ).cuda()
+        return self.output_buffer[:ntokens, :]
+
+    # TODO(Jiayi): A lot of memory copies can be avoided in this function.
+    @_lmcache_nvtx_annotate
+    def deserialize(self, buffer_memory_obj: BytesBufferMemoryObj) -> MemoryObj:
+        encoder_output = CacheGenGPUEncoderOutput.from_bytes(
+            buffer_memory_obj.byte_array
+        )
+
+        encoder_output.max_tensors_key = encoder_output.max_tensors_key.cuda()
+        encoder_output.max_tensors_value = encoder_output.max_tensors_value.cuda()
+
+        ntokens = encoder_output.max_tensors_key.shape[1]
+        layers_in_key = encoder_output.max_tensors_key.shape[0]
+        key, value = decode_function_gpu(
+            encoder_output.cdf,
+            encoder_output.data_chunks,
+            layers_in_key,
+            ntokens,
+            self.get_output_buffer(
+                encoder_output.cdf.shape[0] // 2,
+                encoder_output.cdf.shape[1],
+                ntokens,
+            ),
+        )
+
+        # Temporary fix for #83: change the device of key_bins and value_bins
+        # to the device of key and value
+        # This requires a long-term fix in the future. Currently,
+        # CacheGenGPUEncoderOutput has implicit device in itself.
+        # More specifically, if the encoder encodes the tensor on GPU0, the
+        # from_bytes will also return a tensor on GPU0
+        # We may want to dynamically configure the device based on config and
+        # metadata in the future
+        if self.key_bins.device != key.device:
+            self.key_bins = self.key_bins.to(key.device)
+
+        if self.value_bins.device != value.device:
+            self.value_bins = self.value_bins.cuda()
+
+        key = do_dequantize(key, self.key_bins, encoder_output.max_tensors_key)
+        value = do_dequantize(value, self.value_bins, encoder_output.max_tensors_value)
+        """ merge key and value back and reshape """
+        nlayers, ntokens, nchannels = key.shape
+        blob = torch.stack([key, value])  # [2, nlayers, ntokens, nchannels]
+        blob = blob.reshape(
+            (
+                2,
+                nlayers,
+                ntokens,
+                encoder_output.num_heads,
+                encoder_output.head_size,
+            )
+        )
+
+        hidden_dim = blob.shape[-1] * blob.shape[-2]
+        kv_chunk = blob.reshape(*blob.shape[:-2], hidden_dim).to(
+            self.dtype
+        )  # [nlayers, 2, ntokens, num_heads, head_size]
+
+        memory_obj = TensorMemoryObj(
+            raw_data=kv_chunk,
+            metadata=MemoryObjMetadata(
+                shape=kv_chunk.shape,
+                dtype=kv_chunk.dtype,
+                address=-1,
+                phy_size=kv_chunk.numel() * kv_chunk.element_size(),
+                ref_count=-1,  # HACK: avoid mis-free
+                fmt=MemoryFormat.KV_2LTD,
+            ),
+            parent_allocator=None,
+        )
+
+        return memory_obj

lmcache/v1/storage_backend/naive_serde/cachegen_encoder.py

@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: Apache-2.0
+# Third Party
+import torch
+
+# First Party
+from lmcache.logging import init_logger
+from lmcache.storage_backend.serde.cachegen_encoder import encode_function
+from lmcache.utils import _lmcache_nvtx_annotate
+from lmcache.v1.config import LMCacheEngineConfig
+from lmcache.v1.memory_management import BytesBufferMemoryObj, MemoryObj
+from lmcache.v1.metadata import LMCacheMetadata
+from lmcache.v1.storage_backend.naive_serde.cachegen_basics import CacheGenConfig
+from lmcache.v1.storage_backend.naive_serde.serde import Serializer
+
+logger = init_logger(__name__)
+
+
+class CacheGenSerializer(Serializer):
+    def __init__(self, config: LMCacheEngineConfig, metadata: LMCacheMetadata):
+        self.cachegen_config = CacheGenConfig.from_model_name(metadata.model_name)
+        self.chunk_size = config.chunk_size
+        self.key_bins = self.make_key_bins(self.cachegen_config)
+        self.value_bins = self.make_value_bins(self.cachegen_config)
+
+        self.kv_shape = metadata.kv_shape
+
+    def make_key_bins(self, config: CacheGenConfig) -> torch.Tensor:
+        ret = torch.zeros(config.nlayers)
+        for spec in config.kspecs:
+            ret[spec.start_layer : spec.end_layer] = spec.bins
+        return ret.cuda()
+
+    def make_value_bins(self, config: CacheGenConfig) -> torch.Tensor:
+        ret = torch.zeros(config.nlayers)
+        for spec in config.vspecs:
+            ret[spec.start_layer : spec.end_layer] = spec.bins
+        return ret.cuda()
+
+    # TODO(Jiayi): A lot of memory copies can be avoided in this function.
+    @_lmcache_nvtx_annotate
+    def serialize(self, memory_obj: MemoryObj) -> BytesBufferMemoryObj:
+        """
+        Serialize a KV_2LTD MemoryObj to CACHEGEN_BINARY MemoryObj.
+
+        Input:
+            memory_obj: the memory object to be serialized.
+
+        Returns:
+            MemoryObj: the serialized binary memory object.
+        """
+
+        # TODO(Jiayi): please avoid this copy by directly performing
+        # serialization inside gpu connector.
+        assert memory_obj.tensor is not None
+        tensor = memory_obj.tensor.cuda()
+
+        # Temporary fix for issue #83: encoder will have the default device 0
+        # on all the ray workers. Need to set it to the correct device.
+        # Also need to figure out why this happens.
+        if torch.cuda.current_device != tensor.device:
+            torch.cuda.set_device(tensor.device)
+        if tensor.device != self.key_bins.device:
+            self.key_bins = self.key_bins.to(tensor.device)
+        if tensor.device != self.value_bins.device:
+            self.value_bins = self.value_bins.to(tensor.device)
+
+        # tensor is [2, num_layers, num_tokens, hidden_size]
+        tensor = tensor.view(*tensor.shape[:-1], self.kv_shape[-2], self.kv_shape[-1])
+        tensor = tensor.permute([1, 0, 2, 3, 4])
+
+        # TODO(Jiayi): remove hardcoded "2"
+        """ expecting a tensor of shape 
+        [num_layers, 2, num_tokens, num_heads, head_size] """
+        ntokens = tensor.shape[2]
+        output_dict = encode_function(
+            tensor,
+            self.cachegen_config,
+            self.key_bins,
+            self.value_bins,
+            ntokens,
+        )
+
+        return BytesBufferMemoryObj(output_dict.to_bytes())

lmcache/v1/storage_backend/naive_serde/kivi_serde.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# First Party
+from lmcache.v1.memory_management import MemoryObj
+from lmcache.v1.storage_backend.naive_serde.serde import Deserializer, Serializer
+
+
+class KIVISerializer(Serializer):
+    def __init__(self):
+        pass
+
+    def serialize(self, memory_obj: MemoryObj) -> MemoryObj:
+        # TODO(Yuhan)
+        return memory_obj
+
+
+class KIVIDeserializer(Deserializer):
+    def __init__(self):
+        pass
+
+    def deserialize(self, memory_obj: MemoryObj) -> MemoryObj:
+        # TODO(Yuhan)
+        return memory_obj

lmcache/v1/storage_backend/naive_serde/naive_serde.py

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+# First Party
+from lmcache.v1.memory_management import MemoryObj
+from lmcache.v1.storage_backend.naive_serde.serde import Deserializer, Serializer
+
+
+class NaiveSerializer(Serializer):
+    def __init__(self):
+        pass
+
+    def serialize(self, memory_obj: MemoryObj) -> MemoryObj:
+        memory_obj.ref_count_up()
+        return memory_obj
+
+
+class NaiveDeserializer(Deserializer):
+    def deserialize(self, memory_obj: MemoryObj) -> MemoryObj:
+        return memory_obj

lmcache/v1/storage_backend/naive_serde/serde.py

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: Apache-2.0
+# Standard
+import abc
+
+# First Party
+from lmcache.v1.memory_management import MemoryObj
+
+
+class Serializer(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def serialize(self, memory_obj: MemoryObj) -> MemoryObj:
+        """
+        Serialize/compress the memory object.
+
+        Input:
+            memory_obj: the memory object to be serialized/compressed.
+
+        Returns:
+            MemoryObj: the serialized/compressed memory object.
+        """
+        raise NotImplementedError
+
+
+class Deserializer(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def deserialize(self, memory_obj: MemoryObj) -> MemoryObj:
+        """
+        Deserialize/decompress the memory object.
+
+        Input:
+            memory_obj: the memory object to be deserialized/decompressed.
+
+        Returns:
+            MemoryObj: the deserialized/decompressed memory object.
+            None: if the memory allocation fails.
+        """
+        raise NotImplementedError

lmcache/v1/storage_backend/native_clients/connector_client_base.py

@@ -0,0 +1,165 @@
+# SPDX-License-Identifier: Apache-2.0
+# Standard
+from typing import Any, Dict, Generic, Optional, Tuple, TypeVar, Union
+import asyncio
+import concurrent.futures
+
+NativeClientT = TypeVar("NativeClientT")
+
+
+class ConnectorClientBase(Generic[NativeClientT]):
+    def __init__(
+        self,
+        native_client: NativeClientT,
+        loop: Optional[asyncio.AbstractEventLoop] = None,
+    ):
+        self.loop = loop or asyncio.get_running_loop()
+        self._client: NativeClientT = native_client
+        self._fd = int(self._client.event_fd())  # type: ignore[attr-defined]
+        self._closed = False
+        # Keepalive refs prevent buffers passed to native code from being
+        # garbage-collected while C++ worker threads still hold raw pointers.
+        self._pending: Dict[
+            int,
+            Tuple[
+                Union[asyncio.Future, concurrent.futures.Future], str, Tuple[Any, ...]
+            ],
+        ] = {}
+        self.loop.add_reader(self._fd, self._on_ready)
+
+    def _on_ready(self) -> None:
+        if self._closed:
+            return
+
+        try:
+            while True:
+                items = self._client.drain_completions()  # type: ignore[attr-defined]
+                if not items:
+                    return
+
+                for future_id, ok, error, result_bools in items:
+                    fid = int(future_id)
+                    entry = self._pending.pop(fid, None)
+                    if entry is None:
+                        continue
+
+                    fut, op, _keepalive = entry
+                    if fut.done():
+                        continue
+
+                    if ok:
+                        if op == "exists":
+                            if result_bools is not None and len(result_bools) > 0:
+                                fut.set_result(bool(result_bools[0]))
+                            else:
+                                fut.set_result(False)
+                        elif op == "batch_exists":
+                            if result_bools is not None:
+                                fut.set_result(list(result_bools))
+                            else:
+                                fut.set_result([])
+                        else:
+                            fut.set_result(None)
+                    else:
+                        fut.set_exception(RuntimeError(str(error)))
+        except Exception as e:
+            self._fail_all(RuntimeError(f"native drain_completions failed: {e}"))
+            self._shutdown_native(best_effort=True)
+
+    def _fail_all(self, exc: Exception) -> None:
+        for fid, (fut, _, _keepalive) in list(self._pending.items()):
+            if not fut.done():
+                fut.set_exception(exc)
+        self._pending.clear()
+
+    def _shutdown_native(self, best_effort: bool = False) -> None:
+        try:
+            self._closed = True
+            self.loop.remove_reader(self._fd)
+        except Exception:
+            if not best_effort:
+                raise
+
+    def _register_future_async(
+        self, op: str, future_id: int, keepalive: Tuple[Any, ...] = ()
+    ) -> asyncio.Future:
+        fut = self.loop.create_future()
+        self._pending[int(future_id)] = (fut, op, keepalive)
+        return fut
+
+    def _register_future_sync(
+        self, op: str, future_id: int, keepalive: Tuple[Any, ...] = ()
+    ) -> concurrent.futures.Future:
+        fut: concurrent.futures.Future = concurrent.futures.Future()
+        self._pending[int(future_id)] = (fut, op, keepalive)
+        return fut
+
+    async def get(self, key: str, buf: memoryview) -> None:
+        return await self.batch_get([key], [buf])
+
+    async def set(self, key: str, buf: memoryview) -> None:
+        return await self.batch_set([key], [buf])
+
+    async def exists(self, key: str) -> bool:
+        results = await self.batch_exists([key])
+        return results[0]
+
+    async def batch_get(self, keys: list[str], bufs: list[memoryview]) -> None:
+        if len(keys) != len(bufs):
+            raise ValueError("keys and bufs length mismatch")
+        future_id = int(self._client.submit_batch_get(keys, bufs))  # type: ignore[attr-defined]
+        fut = self._register_future_async("batch_get", future_id, (keys, tuple(bufs)))
+        return await fut
+
+    async def batch_set(self, keys: list[str], bufs: list[memoryview]) -> None:
+        if len(keys) != len(bufs):
+            raise ValueError("keys and bufs length mismatch")
+        future_id = int(self._client.submit_batch_set(keys, bufs))  # type: ignore[attr-defined]
+        fut = self._register_future_async("batch_set", future_id, (keys, tuple(bufs)))
+        return await fut
+
+    async def batch_exists(self, keys: list[str]) -> list[bool]:
+        future_id = int(self._client.submit_batch_exists(keys))  # type: ignore[attr-defined]
+        fut = self._register_future_async("batch_exists", future_id)
+        return await fut
+
+    async def batched_exists(self, keys: list[str]) -> list[bool]:
+        return await self.batch_exists(keys)
+
+    def get_sync(self, key: str, buf: memoryview) -> None:
+        return self.batch_get_sync([key], [buf])
+
+    def set_sync(self, key: str, buf: memoryview) -> None:
+        return self.batch_set_sync([key], [buf])
+
+    def exists_sync(self, key: str) -> bool:
+        results = self.batch_exists_sync([key])
+        return results[0]
+
+    def batch_get_sync(self, keys: list[str], bufs: list[memoryview]) -> None:
+        if len(keys) != len(bufs):
+            raise ValueError("keys and bufs length mismatch")
+        future_id = int(self._client.submit_batch_get(keys, bufs))  # type: ignore[attr-defined]
+        fut = self._register_future_sync("batch_get", future_id, (keys, tuple(bufs)))
+        return fut.result()
+
+    def batch_set_sync(self, keys: list[str], bufs: list[memoryview]) -> None:
+        if len(keys) != len(bufs):
+            raise ValueError("keys and bufs length mismatch")
+        future_id = int(self._client.submit_batch_set(keys, bufs))  # type: ignore[attr-defined]
+        fut = self._register_future_sync("batch_set", future_id, (keys, tuple(bufs)))
+        return fut.result()
+
+    def batch_exists_sync(self, keys: list[str]) -> list[bool]:
+        future_id = int(self._client.submit_batch_exists(keys))  # type: ignore[attr-defined]
+        fut = self._register_future_sync("batch_exists", future_id)
+        return fut.result()
+
+    def batched_exists_sync(self, keys: list[str]) -> list[bool]:
+        return self.batch_exists_sync(keys)
+
+    def close(self) -> None:
+        if not self._closed:
+            self._shutdown_native(best_effort=True)
+            self._fail_all(RuntimeError("Client closed"))
+            self._client.close()  # type: ignore[attr-defined]

lmcache/v1/storage_backend/native_clients/resp_client.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+# Standard
+from typing import Optional
+import asyncio
+
+# Local
+from .connector_client_base import ConnectorClientBase
+
+try:
+    # First Party
+    from lmcache.lmcache_redis import LMCacheRedisClient
+
+    REDIS_AVAILABLE = True
+except ImportError:
+    REDIS_AVAILABLE = False
+    LMCacheRedisClient = None  # type: ignore
+
+
+class RESPClient(ConnectorClientBase[LMCacheRedisClient]):
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        num_workers: int,
+        loop: Optional[asyncio.AbstractEventLoop] = None,
+        username: str = "",
+        password: str = "",
+    ):
+        if not REDIS_AVAILABLE:
+            raise RuntimeError(
+                "RESPClient requires the C++ Redis extension. "
+                "Build with: pip install -e ."
+            )
+        native_client = LMCacheRedisClient(host, port, num_workers, username, password)
+        super().__init__(native_client, loop)