lmcache-cli 0.4.5.dev0__py3-none-any.whl

lmcache/v1/distributed/eviction.py

@@ -0,0 +1,192 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Eviction module to determine what to evict from L1 and L2 caches.
+"""
+
+# Standard
+from abc import abstractmethod
+from collections.abc import Callable
+
+# First Party
+from lmcache.v1.distributed.api import ObjectKey
+from lmcache.v1.distributed.internal_api import (
+    EvictionAction,
+    EvictionDestination,
+    L1ManagerListener,
+    L2AdapterListener,
+)
+
+
+class EvictionPolicy:
+    """
+    Pure abstract base class for eviction policies.
+
+    Subclasses implement the LRU (or other) tracking logic via the
+    on_keys_* methods and expose eviction decisions via get_eviction_actions.
+    The binding to a specific cache tier (L1 or L2) is provided by
+    L1EvictionPolicy and L2EvictionPolicy respectively.
+    """
+
+    @property
+    def support_isolation(self) -> bool:
+        """Whether this policy supports isolation eviction (e.g., per user isolation).
+
+        When True, the eviction controller checks isolated usage (e.g., per user usage)
+        and passes ``cache_salt`` to ``get_eviction_actions()`` to scope
+        eviction to specific cache_salt. When False, the controller uses
+        aggregate usage only.
+
+        Default is False. Subclasses that support isolated eviction.
+        (e.g., ``IsolatedLRUEvictionPolicy``) should override to return True.
+        """
+        return False
+
+    @abstractmethod
+    def register_eviction_destination(self, destination: EvictionDestination):
+        """
+        Register an eviction destination for the eviction policy to use.
+
+        Args:
+            destination (EvictionDestination): The eviction destination to
+                register.
+        """
+        pass
+
+    @abstractmethod
+    def on_keys_created(self, keys: list[ObjectKey]):
+        """
+        Notify the eviction policy that new keys have been created.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been created.
+        """
+        pass
+
+    @abstractmethod
+    def on_keys_touched(self, keys: list[ObjectKey]):
+        """
+        Notify the eviction policy that keys have been accessed.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been accessed.
+        """
+        pass
+
+    @abstractmethod
+    def on_keys_removed(self, keys: list[ObjectKey]):
+        """
+        Notify the eviction policy that keys have been deleted.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been deleted.
+        """
+        pass
+
+    @abstractmethod
+    def get_eviction_actions(
+        self,
+        expected_ratio: float,
+        key_eligible_filter: Callable[[ObjectKey], bool] | None = None,
+        cache_salt: str | None = None,
+    ) -> list[EvictionAction]:
+        """
+        Get the eviction actions to evict objects from cache.
+
+        Args:
+            expected_ratio (float): A hint indicating approximately what
+                fraction of tracked keys should be evicted. Value should be
+                in range [0.0, 1.0]. For example, 0.1 means roughly 10% of
+                keys should be evicted. This is a hint and the policy may
+                return more or fewer keys.
+            key_eligible_filter: An optional callable that takes an ObjectKey
+                and returns True if the key is eligible for eviction. When
+                provided, keys for which the filter returns False will be
+                skipped. This is useful for skipping locked keys that
+                cannot be deleted.
+            cache_salt: When set, scope eviction to keys belonging to this
+                salt only (identified by ``ObjectKey.cache_salt``). When
+                None, evict globally across all salts. Only meaningful for
+                policies where ``support_isolation`` is True; other policies
+                ignore this parameter.
+
+        Returns:
+            list[EvictionAction]: The eviction actions to perform. Each
+                action contains the keys and one eviction destination.
+
+        Notes:
+            The eviction action may not be successfully executed, or it may
+            be executed asynchronously. Therefore, the eviction policy should
+            not assume that the objects are evicted immediately, but it should
+            use `on_keys_removed` to know when the objects are actually
+            deleted.
+        """
+        pass
+
+
+class L1EvictionPolicy(L1ManagerListener):
+    """
+    Bridges L1Manager lifecycle events to an EvictionPolicy instance.
+
+    The actual eviction policy is provided via the constructor, keeping
+    the policy logic decoupled from the listener interface.
+    """
+
+    def __init__(self, policy: EvictionPolicy):
+        self._policy = policy
+
+    @property
+    def policy(self) -> EvictionPolicy:
+        return self._policy
+
+    # L1ManagerListener implementations
+    def on_l1_keys_reserved_read(self, keys: list[ObjectKey]):
+        # No-op
+        pass
+
+    def on_l1_keys_read_finished(self, keys: list[ObjectKey]):
+        self._policy.on_keys_touched(keys)
+
+    def on_l1_keys_reserved_write(self, keys: list[ObjectKey]):
+        # No-op
+        pass
+
+    def on_l1_keys_write_finished(self, keys: list[ObjectKey]):
+        # TODO (ApostaC): we don't differentiate between the created keys and
+        # updated keys here. Probably need to fix that by introducing a new
+        # callback in L1ManagerListener or adding `mode` argument into
+        # on_keys_reserved_write.
+        self._policy.on_keys_created(keys)
+
+    def on_l1_keys_deleted_by_manager(self, keys: list[ObjectKey]):
+        self._policy.on_keys_removed(keys)
+
+    def on_l1_keys_finish_write_and_reserve_read(self, keys: list[ObjectKey]):
+        self._policy.on_keys_created(keys)
+
+    def on_l1_keys_accessed(self, keys: list[ObjectKey]):
+        self._policy.on_keys_touched(keys)
+
+
+class L2EvictionPolicy(L2AdapterListener):
+    """
+    Bridges L2Adapter lifecycle events to an EvictionPolicy instance.
+
+    The actual eviction policy is provided via the constructor, keeping
+    the policy logic decoupled from the listener interface.
+    """
+
+    def __init__(self, policy: EvictionPolicy):
+        self._policy = policy
+
+    @property
+    def policy(self) -> EvictionPolicy:
+        return self._policy
+
+    def on_l2_keys_stored(self, keys: list[ObjectKey]):
+        self._policy.on_keys_created(keys)
+
+    def on_l2_keys_accessed(self, keys: list[ObjectKey]):
+        self._policy.on_keys_touched(keys)
+
+    def on_l2_keys_deleted(self, keys: list[ObjectKey]):
+        self._policy.on_keys_removed(keys)

lmcache/v1/distributed/eviction_policy/__init__.py

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Eviction policies for L1 cache management
+"""
+
+# First Party
+from lmcache.v1.distributed.eviction_policy.factory import (
+    CreateEvictionPolicy,
+)
+from lmcache.v1.distributed.eviction_policy.lru import (
+    LRUEvictionPolicy,
+)
+from lmcache.v1.distributed.eviction_policy.noop import (
+    NoOpEvictionPolicy,
+)
+
+__all__ = [
+    "LRUEvictionPolicy",
+    "NoOpEvictionPolicy",
+    "CreateEvictionPolicy",
+]

lmcache/v1/distributed/eviction_policy/factory.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# First Party
+from lmcache.v1.distributed.config import EvictionConfig
+from lmcache.v1.distributed.eviction import EvictionPolicy
+from lmcache.v1.distributed.eviction_policy.lru import LRUEvictionPolicy
+from lmcache.v1.distributed.eviction_policy.noop import NoOpEvictionPolicy
+
+
+def CreateEvictionPolicy(eviction_config: EvictionConfig) -> EvictionPolicy:
+    """
+    Factory method to create an eviction policy based on the provided configuration.
+
+    Args:
+        eviction_config (EvictionConfig): The configuration for the eviction policy.
+
+    Returns:
+        An instance of the specified eviction policy.
+    """
+    if eviction_config.eviction_policy == "LRU":
+        return LRUEvictionPolicy()
+    elif eviction_config.eviction_policy == "noop":
+        return NoOpEvictionPolicy()
+    else:
+        raise ValueError(
+            f"Unsupported eviction policy: {eviction_config.eviction_policy}"
+        )

lmcache/v1/distributed/eviction_policy/lru.py

@@ -0,0 +1,244 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+LRU (Least Recently Used) eviction policy implementation
+"""
+
+# Standard
+from collections import OrderedDict
+from collections.abc import Callable
+import threading
+
+# First Party
+from lmcache.v1.distributed.api import ObjectKey
+from lmcache.v1.distributed.eviction import EvictionPolicy
+from lmcache.v1.distributed.internal_api import (
+    EvictionAction,
+    EvictionDestination,
+)
+
+
+class LRUEvictionPolicy(EvictionPolicy):
+    """
+    LRU (Least Recently Used) eviction policy.
+
+    This policy tracks the order of key accesses and evicts the least recently
+    used keys first when eviction is needed.
+
+    Thread Safety:
+        This class is thread-safe. All operations are protected by a global lock
+        to ensure eventual consistency without corrupted states.
+
+    Attributes:
+        _lock: Threading lock for thread-safe operations
+        _order: OrderedDict maintaining LRU order (most recent at end)
+        _destinations: List of registered eviction destinations
+        _default_destination: The default destination for eviction actions
+    """
+
+    def __init__(
+        self,
+        default_destination: EvictionDestination = EvictionDestination.DISCARD,
+    ):
+        """
+        Initialize the LRU eviction policy.
+
+        Args:
+            default_destination: The default destination for evicted objects.
+                Defaults to DISCARD.
+        """
+        # Lock for thread-safe operations
+        self._lock = threading.Lock()
+
+        # OrderedDict to maintain LRU order - keys at the beginning are oldest
+        self._order: OrderedDict[ObjectKey, None] = OrderedDict()
+
+        # List of registered eviction destinations
+        self._destinations: list[EvictionDestination] = []
+
+        # Default destination for eviction
+        self._default_destination = default_destination
+
+    def register_eviction_destination(self, destination: EvictionDestination):
+        """
+        Register an eviction destination for the eviction policy to use.
+
+        Args:
+            destination (EvictionDestination): The eviction destination to register
+        """
+        with self._lock:
+            if destination not in self._destinations:
+                self._destinations.append(destination)
+
+    def on_keys_created(self, keys: list[ObjectKey]):
+        """
+        Notify the eviction policy that new keys have been created.
+        New keys are added as most recently used.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been created
+        """
+        if not keys:
+            return
+        with self._lock:
+            # NOTE: for the request, the later keys should be evicted first.
+            # For example, the request has (key1, key2, key3), if we first
+            # evict key1, due to prefix match, key2 and key3 will not be hit.
+            for key in reversed(keys):
+                # If key already exists, move it to the end (most recently used)
+                if key in self._order:
+                    self._order.move_to_end(key)
+                else:
+                    # Add new key at the end (most recently used)
+                    self._order[key] = None
+
+    def on_keys_touched(self, keys: list[ObjectKey]):
+        """
+        Notify the eviction policy that keys have been accessed.
+        Touched keys are moved to the most recently used position.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been accessed
+        """
+        if not keys:
+            return
+        with self._lock:
+            # NOTE: for the request, the later keys should be evicted first.
+            # The example is the same as `on_keys_created`.
+            for key in reversed(keys):
+                if key in self._order:
+                    # Move to end (most recently used)
+                    self._order.move_to_end(key)
+
+    def on_keys_removed(self, keys: list[ObjectKey]):
+        """
+        Notify the eviction policy that keys have been deleted.
+        Deleted keys are removed from tracking.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been deleted
+        """
+        if not keys:
+            return
+        with self._lock:
+            for key in keys:
+                # Remove from LRU order tracking
+                if key in self._order:
+                    del self._order[key]
+
+    def get_eviction_actions(
+        self,
+        expected_ratio: float,
+        key_eligible_filter: Callable[[ObjectKey], bool] | None = None,
+        cache_salt: str | None = None,
+    ) -> list[EvictionAction]:
+        """
+        Get the eviction actions to evict objects from L1 cache.
+        Returns keys in LRU order (least recently used first).
+
+        Args:
+            expected_ratio (float): A hint indicating approximately what fraction
+                of tracked keys should be evicted. Value should be in range [0.0, 1.0].
+                For example, 0.1 means roughly 10% of keys should be evicted.
+            key_eligible_filter: An optional callable that takes an ObjectKey
+                and returns True if the key is eligible for eviction. When
+                provided, keys for which the filter returns False will be
+                skipped. This is useful for skipping locked keys that
+                cannot be deleted.
+            cache_salt: Ignored by LRU policy (not user-level).
+
+        Returns:
+            list[EvictionAction]: The eviction actions to perform. Each
+                action contains the keys and one eviction destination.
+
+        Notes:
+            The eviction action may not be successfully executed, or it
+            may be executed asynchronously. Therefore, the eviction policy
+            should not assume that the objects are evicted immediately, but
+            it should use `on_keys_deleted` to know when the objects are actually
+            deleted.
+        """
+        with self._lock:
+            if not self._order:
+                return []
+
+            # Clamp expected_ratio to valid range
+            expected_ratio = max(0.0, min(1.0, expected_ratio))
+
+            # Calculate target number of keys to evict based on ratio
+            target_count = int(len(self._order) * expected_ratio)
+
+            # Ensure at least 1 key if ratio > 0 and we have keys
+            if expected_ratio > 0 and target_count == 0 and len(self._order) > 0:
+                target_count = 1
+
+            if target_count == 0:
+                return []
+
+            # Get keys in LRU order (from beginning - least recently used),
+            # skipping keys that fail the filter (e.g. locked keys).
+            keys_to_evict: list[ObjectKey] = []
+
+            for key in self._order:
+                if key_eligible_filter is not None and not key_eligible_filter(key):
+                    # Skip keys that are not eligible for eviction
+                    # (e.g. currently locked by read/write operations)
+                    continue
+                keys_to_evict.append(key)
+                if len(keys_to_evict) >= target_count:
+                    break
+
+            if not keys_to_evict:
+                return []
+
+            # Determine the destination
+            destination = self._default_destination
+            if self._destinations:
+                # Use the first registered destination if available
+                destination = self._destinations[0]
+
+            return [EvictionAction(keys=keys_to_evict, destination=destination)]
+
+    # =========================================================================
+    # Methods below are NOT part of the EvictionPolicy interface.
+    # They are provided for testing and debugging purposes only.
+    # =========================================================================
+
+    def get_num_tracked_keys(self) -> int:
+        """
+        Get the number of keys currently being tracked.
+
+        Note:
+            This method is NOT part of the EvictionPolicy interface.
+            It is provided for testing and debugging purposes only.
+
+        Returns:
+            int: The number of tracked keys
+        """
+        with self._lock:
+            return len(self._order)
+
+    def get_eviction_candidates(self, count: int) -> list[ObjectKey]:
+        """
+        Get a list of eviction candidates without creating eviction actions.
+        Useful for querying what would be evicted.
+
+        Note:
+            This method is NOT part of the EvictionPolicy interface.
+            It is provided for testing and debugging purposes only.
+
+        Args:
+            count: Maximum number of candidates to return
+
+        Returns:
+            list[ObjectKey]: List of keys that are candidates for eviction,
+                in LRU order (least recently used first)
+        """
+        with self._lock:
+            candidates: list[ObjectKey] = []
+
+            for key in self._order:
+                candidates.append(key)
+                if len(candidates) >= count:
+                    break
+
+            return candidates

lmcache/v1/distributed/eviction_policy/noop.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+No-op eviction policy for buffer-only mode.
+
+When L1 is used purely as a write buffer (data is deleted from L1
+immediately after being stored to L2 by the StoreController), there
+is no need for the eviction policy to track keys at all.
+"""
+
+# Standard
+from collections.abc import Callable
+
+# First Party
+from lmcache.v1.distributed.api import ObjectKey
+from lmcache.v1.distributed.eviction import EvictionPolicy
+from lmcache.v1.distributed.internal_api import (
+    EvictionAction,
+    EvictionDestination,
+)
+
+
+class NoOpEvictionPolicy(EvictionPolicy):
+    """
+    Eviction policy that performs no tracking and never evicts.
+
+    Designed for buffer-only mode where the StoreController is
+    responsible for cleaning up L1 after successful L2 writes.
+    This avoids the overhead of maintaining an LRU OrderedDict
+    that would otherwise do pointless insert-then-remove cycles.
+    """
+
+    def register_eviction_destination(self, destination: EvictionDestination):
+        pass
+
+    def on_keys_created(self, keys: list[ObjectKey]):
+        pass
+
+    def on_keys_touched(self, keys: list[ObjectKey]):
+        pass
+
+    def on_keys_removed(self, keys: list[ObjectKey]):
+        pass
+
+    def get_eviction_actions(
+        self,
+        expected_ratio: float,
+        key_eligible_filter: Callable[[ObjectKey], bool] | None = None,
+        cache_salt: str | None = None,
+    ) -> list[EvictionAction]:
+        return []

lmcache/v1/distributed/internal_api.py

@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Class for distributed storage manager internal API data structures
+"""
+
+# Standard
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+import enum
+
+# First Party
+from lmcache.v1.distributed.api import ObjectKey
+
+
+@dataclass(frozen=True)
+class L1MemoryDesc:
+    """
+    Describes the L1 memory buffer registered with an external backend (e.g. Nixl).
+    """
+
+    ptr: int
+    size: int
+    align_bytes: int
+
+
+class EventListener(ABC):  # noqa: B024
+    pass
+
+
+# For L1 manager event notifications
+class L1ManagerListener(EventListener):
+    """
+    Listener for L1 manager events
+    """
+
+    @abstractmethod
+    def on_l1_keys_reserved_read(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that new keys have been reserved for read on L1.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been successfully reserved
+        """
+        pass
+
+    @abstractmethod
+    def on_l1_keys_read_finished(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been accessed on L1.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been successfully read
+        """
+        pass
+
+    @abstractmethod
+    def on_l1_keys_reserved_write(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been reserved for write on L1.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been successfully reserved
+        """
+        pass
+
+    @abstractmethod
+    def on_l1_keys_write_finished(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been finished for writing on L1.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been successfully written
+        """
+        pass
+
+    @abstractmethod
+    def on_l1_keys_finish_write_and_reserve_read(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been finished for writing
+        and reserved for read on L1.
+
+        This will only be trigger by the prefetch operation now.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been successfully
+                finished for writing and reserved for read
+        """
+        # NOTE (ApostaC): may consider renaming this to `on_l1_keys_finish_prefetch`
+        # for better clarity
+        pass
+
+    @abstractmethod
+    def on_l1_keys_deleted_by_manager(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been deleted from L1.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been deleted
+        """
+        pass
+
+    @abstractmethod
+    def on_l1_keys_accessed(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been accessed on L1.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been accessed
+        """
+        pass
+
+
+class L2AdapterListener(EventListener):
+    """Listener for L2 adapter events, analogous to L1ManagerListener."""
+
+    @abstractmethod
+    def on_l2_keys_stored(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been successfully stored in L2.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been stored.
+        """
+        pass
+
+    @abstractmethod
+    def on_l2_keys_accessed(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been accessed (lookup hit) in L2.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been accessed.
+        """
+        pass
+
+    @abstractmethod
+    def on_l2_keys_deleted(self, keys: list[ObjectKey]):
+        """
+        Notify the listener that keys have been deleted from L2.
+
+        Args:
+            keys (list[ObjectKey]): The keys that have been deleted.
+        """
+        pass
+
+
+# For Eviction
+class EvictionDestination(enum.Enum):
+    """
+    The destination of evicted objects
+    """
+
+    DISCARD = enum.auto()
+    """Discard the evicted objects"""
+
+    L2_CACHE = enum.auto()
+    """Evict to L2 storage"""
+
+
+@dataclass(frozen=True)
+class EvictionAction:
+    """
+    An action to be taken for eviction
+    """
+
+    destination: EvictionDestination
+    """The destination of the evicted object"""
+
+    keys: list[ObjectKey] = field(default_factory=list)
+    """The key of the object to be evicted"""