gllm-datastore-binary 0.5.45__cp311-cp311-macosx_13_0_arm64.whl

gllm_datastore/vector_data_store/mixin/cache_compatible_mixin.pyi

@@ -0,0 +1,145 @@
+from abc import ABC, abstractmethod
+from datetime import datetime
+from gllm_datastore.cache.cache import MatchingStrategy as MatchingStrategy
+from gllm_datastore.cache.vector_cache.eviction_manager.eviction_manager import BaseEvictionManager as BaseEvictionManager
+from gllm_datastore.cache.vector_cache.vector_cache import VectorCache as VectorCache
+from gllm_datastore.constants import METADATA_KEYS as METADATA_KEYS
+from typing import Any
+
+class CacheCompatibleMixin(ABC):
+    """Mixin that provides cache-specific matching operations for vector datastores.
+
+    This mixin adds methods for exact, fuzzy, and semantic matching that are
+    required by the VectorCache implementation, without forcing all vector datastores
+    to implement these methods.
+    """
+    async def store_cache(self, key: str, value: Any, metadata: dict[str, Any] | None = None) -> None:
+        """Public method to store cache data in the storage.
+
+        Args:
+            key (str): The key to store the cache data.
+            value (Any): The cache data to store.
+            metadata (dict[str, Any] | None, optional): Additional metadata to store with the cache data.
+                Defaults to None.
+        """
+    @abstractmethod
+    async def exact_match(self, key: str, **kwargs) -> Any | None:
+        """Find chunks that exactly match the given key.
+
+        This method should be implemented by subclasses.
+
+        Args:
+            key (str): The key to match.
+            **kwargs (Any): Additional parameters for the matching operation.
+
+        Returns:
+            Any: Chunks that exactly match the key.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def fuzzy_match(self, key: str, max_distance: int = 2, **kwargs) -> Any | None:
+        """Find chunks that approximately match the given key using fuzzy matching.
+
+        This method should be implemented by subclasses.
+
+        Args:
+            key (str): The key to match.
+            max_distance (int): Maximum distance for fuzzy matching. Lower values are more strict.
+                This is the maximum Levenshtein distance allowed for a match. Defaults to 2.
+            **kwargs (Any): Additional parameters for the matching operation.
+
+        Returns:
+            Any: Chunks that fuzzy match the key within the threshold.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def semantic_match(self, key: str, min_similarity: float = 0.8, metadata: dict[str, Any] | None = None, **kwargs) -> Any | None:
+        """Find chunks that semantically match the given key using vector similarity.
+
+        This method should be implemented by subclasses.
+
+        Args:
+            key (str): The key to match.
+            min_similarity (float): Minimum similarity score for semantic matching
+                (higher values are more strict). Ranges from 0 to 1. Defaults to 0.8.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                Defaults to None.
+            **kwargs (Any): Additional parameters for the matching operation.
+
+        Returns:
+            Any: Chunks that semantically match the key above the threshold.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def delete_expired_entries(self, now: datetime, max_size: int = 10000) -> None:
+        """Delete expired entries (for TTL eviction).
+
+        This method should be implemented by subclasses.
+
+        Args:
+            now (datetime): The current datetime for comparison.
+            max_size (int): The maximum number of entries to return. Defaults to 10000.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def delete_least_frequently_used_entries(self, num_entries: int) -> None:
+        """Delete least frequently used entries (for LFU eviction).
+
+        This method should be implemented by subclasses.
+
+        Args:
+            num_entries (int): Number of entries to return.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def delete_least_recently_used_entries(self, num_entries: int) -> None:
+        """Delete least recently used entries (for LRU eviction).
+
+        This method should be implemented by subclasses.
+
+        Args:
+            num_entries (int): Number of entries to return.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def delete_entries_by_key(self, key: str | list[str], metadata: dict[str, Any] | None = None) -> None:
+        '''Delete entries by key.
+
+        This method should be implemented by subclasses.
+
+        Args:
+            key (str): The key to delete entries for.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        '''
+    def as_cache(self, eviction_manager: BaseEvictionManager | None = None, matching_strategy: MatchingStrategy = 'exact', matching_config: dict[str, Any] | None = None, saving_config: dict[str, Any] | None = None) -> VectorCache:
+        """Return a cache instance that can be used to store and retrieve data.
+
+        Args:
+            eviction_manager (Optional[BaseEvictionManager], optional): The eviction manager to use for cache eviction.
+                Defaults to None. If None, no eviction will be performed.
+            matching_strategy (MatchingStrategy, optional): The strategy to use for matching keys.
+                Defaults to MatchingStrategy.EXACT.
+            matching_config (dict[str, Any] | None, optional): Configuration parameters for matching strategies.
+                Defaults to None, which means no specific configuration is provided.
+            saving_config (dict[str, Any] | None, optional): Configuration parameters for saving strategies.
+                Defaults to None, which means no specific configuration is provided.
+
+        Returns:
+            VectorCache: A cache instance that can be used to store and retrieve data.
+        """

gllm_datastore/vector_data_store/redis_vector_data_store.pyi

@@ -0,0 +1,191 @@
+from _typeshed import Incomplete
+from datetime import datetime
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.constants import DEFAULT_TOP_K as DEFAULT_TOP_K, METADATA_KEYS as METADATA_KEYS
+from gllm_datastore.utils.converter import cosine_distance_to_similarity_score as cosine_distance_to_similarity_score, similarity_score_to_cosine_distance as similarity_score_to_cosine_distance
+from gllm_datastore.vector_data_store.mixin.cache_compatible_mixin import CacheCompatibleMixin as CacheCompatibleMixin
+from gllm_datastore.vector_data_store.vector_data_store import BaseVectorDataStore as BaseVectorDataStore
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from redis import Redis as Redis
+from redisvl.query.filter import FilterExpression as FilterExpression
+from typing import Any
+
+FUZZY_MATCH_MAX_DISTANCE: int
+
+class RedisVectorDataStore(BaseVectorDataStore, CacheCompatibleMixin):
+    """Vector data store implementation that uses Redis with RedisVL for vector search.
+
+    This class provides methods to interact with Redis for vector storage and retrieval
+    using Redis Vector Search capabilities via RedisVL and langchain-redis.
+
+    Attributes:
+        redis_url (str): URL for Redis connection.
+        index_name (str): Name for the vector index.
+        search_index (SearchIndex): RedisVL SearchIndex instance.
+        cache_store (SemanticCache): RedisVL SemanticCache instance.
+        embedding (BaseEMInvoker | None): The embedding model to perform vectorization.
+    """
+    index_name: Incomplete
+    url: Incomplete
+    client: Incomplete
+    filterable_fields: Incomplete
+    cache_store: Incomplete
+    def __init__(self, index_name: str, url: str | None = None, client: Redis | None = None, embedding: BaseEMInvoker | None = None, additional_filter_fields: list[dict[str, Any]] | None = None) -> None:
+        '''Initialize Redis vector store using RedisVL and langchain-redis.
+
+        Args:
+            index_name (str): Name of the index to use.
+            url (str): URL for Redis connection.
+            client (Redis | None, optional): Redis client to use for vectorization.
+            embedding (BaseEMInvoker | None, optional): Embedding function to use for vectorization.
+                Defaults to None. If None, the default embedding model (redis/langcache-embed-v1) will be used.
+            additional_filter_fields (list[dict[str, Any]] | None, optional): Additional filterable fields to add
+                to the index. For example, to add `entry_id` as a filterable field, pass
+                `[{"name": "entry_id", "type": "text"}]`. Defaults to None.
+
+        Notes:
+            Besides the `additional_filter_fields`, the class will automatically create default filterable fields:
+                1. prompt: TEXT (default from redisvl).
+                2. response: TEXT (default from redisvl).
+                3. prompt_vector: VECTOR (default from redisvl).
+                4. chunk_id: TEXT (default additional_filter_fields).
+
+        Raises:
+            TypeError: If `embedding` is not an instance of `BaseEMInvoker`.
+        '''
+    async def get_size(self) -> int:
+        """Returns the total number of vectors in the index.
+
+        If the index is not initialized returns 0.
+
+        Returns:
+            int: The total number of vectors.
+        """
+    async def query(self, query: str, top_k: int = ..., retrieval_params: dict[str, Any] | None = None) -> list[Chunk]:
+        """Search for semantically similar documents which returns similarity scores.
+
+        Args:
+            query (str): The query text to search for.
+            top_k (int): Number of top results to return.
+            retrieval_params (dict[str, Any] | None, optional): Additional parameters for the query such as:
+                - filter: Redis filter expression to narrow results following RedisVL FilterExpression.
+
+        Returns:
+            list[Chunk]: List of chunks semantically similar to the query
+        """
+    async def query_by_id(self, id_: str | list[str]) -> list[Chunk]:
+        """Retrieve chunks by their IDs.
+
+        Args:
+            id_ (str | list[str]): A single ID or list of chunk IDs to retrieve
+
+        Returns:
+            list[Chunk]: List of retrieved chunks
+        """
+    async def add_chunks(self, chunks: Chunk | list[Chunk], **kwargs) -> list[str]:
+        """Add chunks to the vector store.
+
+        Args:
+            chunks (Chunk | list[Chunk]): A single chunk or a list of chunks to add
+            **kwargs: Additional parameters for adding chunks
+
+        Returns:
+            list[str]: List of IDs of the added chunks
+        """
+    async def delete_chunks(self, query: str, **kwargs: Any) -> None:
+        '''Delete chunks from the vector store by filter/query. Not supported for Redis backend.
+
+        Args:
+            query (str): The query to delete chunks by. For example, "user_*" would match keys
+                like "user_1", "user_2", etc.
+            **kwargs: Additional keyword arguments.
+        '''
+    async def delete_chunks_by_ids(self, ids: str | list[str], **kwargs: Any) -> None:
+        """Delete chunks from the vector store by their IDs.
+
+        Args:
+            ids (str | list[str]): A single ID or a list of IDs to delete.
+            **kwargs: Additional keyword arguments.
+        """
+    async def exact_match(self, key: str, metadata: dict[str, Any] | None = None) -> Any | None:
+        '''Find chunks that exactly match the given prompt.
+
+        Args:
+            key (str): The prompt to match.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+
+        Returns:
+            Any: The value stored with the matching prompt, or None if no match is found.
+        '''
+    async def fuzzy_match(self, key: str, max_distance: int = 2, metadata: dict[str, Any] | None = None) -> Any | None:
+        '''Find chunks that approximately match the given key using fuzzy matching.
+
+        Args:
+            key (str): The key to match
+            max_distance (int): Maximum allowed distance for fuzzy matching
+                (higher values allow for more differences). Maximum is 3. Defaults to 2.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+
+        Note:
+            Maximum fuzzy distance is 3. This is a limitation of the Redis Vector Search and the Redis Search module.
+            See [5] for more details.
+
+        Returns:
+            Any: The value with the closest fuzzy match, or None if no match is found
+        '''
+    async def semantic_match(self, key: str, min_similarity: float = 0.8, metadata: dict[str, Any] | None = None) -> Any | None:
+        '''Find chunks that semantically match the given key using vector similarity.
+
+        This method compares the vector embedding of the search key with vector embeddings
+        of stored keys to find semantically similar matches.
+
+        Args:
+            key (str): The key to match
+            min_similarity (float, optional): Minimum similarity score for semantic matching
+                (higher values are more strict). Ranges from 0 to 1. Defaults to 0.8.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+
+        Returns:
+            Any: The semantically closest value, or None if no match meets the threshold
+        '''
+    async def delete_expired_entries(self, now: datetime, max_size: int = 10000) -> None:
+        """Delete expired entries (for TTL eviction).
+
+        Args:
+            now (datetime): The current datetime for comparison.
+            max_size (int): The maximum number of entries to return. Defaults to 10000.
+
+        Raises:
+            NotImplementedError: Currently, app-level eviction is not supported for RedisVectorDataStore.
+        """
+    async def delete_least_frequently_used_entries(self, num_entries: int) -> None:
+        """Delete least frequently used entries (for LFU eviction).
+
+        Args:
+            num_entries (int): Number of entries to return.
+
+        Raises:
+            NotImplementedError: Currently, app-level eviction is not supported for RedisVectorDataStore.
+        """
+    async def delete_least_recently_used_entries(self, num_entries: int) -> None:
+        """Delete least recently used entries (for LRU eviction).
+
+        Args:
+            num_entries (int): Number of entries to return.
+
+        Raises:
+            NotImplementedError: Currently, app-level eviction is not supported for RedisVectorDataStore.
+        """
+    async def delete_entries_by_key(self, key: str | list[str], metadata: dict[str, Any] | None = None) -> None:
+        '''Delete entries by key.
+
+        Args:
+            key (str | list[str]): The key or list of keys to delete entries for.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+        '''
+    async def clear(self) -> None:
+        """Clear all entries in the storage."""

gllm_datastore/vector_data_store/vector_data_store.pyi

@@ -0,0 +1,146 @@
+from abc import ABC, abstractmethod
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.constants import DEFAULT_TOP_K as DEFAULT_TOP_K
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from langchain_core.embeddings import Embeddings
+from typing import Any
+
+class BaseVectorDataStore(ABC):
+    """Abstract base class for vector data stores in the retrieval system.
+
+    This class defines the interface for all vector data store implementations.
+    Subclasses must implement the `query` and `query_by_id` methods.
+    """
+    @property
+    def embedding(self) -> BaseEMInvoker | Embeddings | None:
+        """Returns the embedding model associated with this data store.
+
+        Returns:
+            BaseEMInvoker | Embeddings | None: The embedding model.
+        """
+    async def get_size(self) -> int:
+        """Returns the total number of vectors in the index.
+
+        If the index is not initialized returns 0.
+
+        Returns:
+            int: The total number of vectors.
+        """
+    @abstractmethod
+    async def query(self, query: str, top_k: int = ..., retrieval_params: dict[str, Any] | None = None) -> list[Chunk]:
+        """Executes a query on the data store.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            query (str): The query string to execute.
+            top_k (int, optional): The maximum number of results to return. Defaults to DEFAULT_TOP_K.
+            retrieval_params (dict[str, Any] | None, optional): Additional parameters for the query.
+                Defaults to None.
+
+        Returns:
+            list[Chunk]: A list of query results.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def query_by_id(self, id_: str | list[str]) -> list[Chunk]:
+        """Retrieves chunks by their IDs.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            id_ (str | list[str]): A single ID or a list of IDs to retrieve.
+
+        Returns:
+            list[Chunk]: A list of retrieved chunks.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def add_chunks(self, chunk: Chunk | list[Chunk], **kwargs) -> list[str]:
+        """Adds a chunk or a list of chunks in the data store.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            chunk (Chunk | list[Chunk]): A single chunk or a list of chunks to index.
+            **kwargs: Additional keyword arguments to pass to the method.
+
+        Returns:
+            list[str]: A list of unique identifiers (IDs) assigned to the added chunks.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def delete_chunks(self, **kwargs: Any) -> None:
+        """Deletes chunks from the data store by filter or query.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            **kwargs: Additional keyword arguments specifying the filter or query for deletion.
+                The exact parameters depend on the backend implementation.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def delete_chunks_by_ids(self, ids: str | list[str], **kwargs: Any) -> None:
+        """Deletes a chunk or a list of chunks from the data store by their IDs.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            ids (str | list[str]): A single ID or a list of IDs to delete.
+            **kwargs: Additional keyword arguments.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    async def clear(self) -> None:
+        """Clear all entries in the storage.
+
+        This method should be implemented by subclasses.
+        """
+    async def query_by_field(self, retrieval_params: dict[str, Any], limit: int | None = None, **kwargs) -> list[Chunk]:
+        """Retrieve documents that match specific metadata constraints.
+
+        This method filters and returns stored chunks based on metadata values
+        rather than vector similarity. It is particularly useful for structured lookups,
+        such as retrieving all chunks from a certain source, tagged with a specific label,
+        or authored by a particular user.
+
+        Unlike semantic search methods, `query_by_field` operates purely on metadata fields
+        associated with each document, allowing precise filtering based on key-value pairs.
+
+        Args:
+            retrieval_params (dict[str, Any]): A dictionary defining filter criteria.
+            limit (int | None, optional): The maximum number of results to return. If None, all matching
+                documents will be returned.
+            **kwargs: Additional arguments to support datastore-specific behavior or filtering logic.
+
+        Returns:
+            list[Chunk]: A list of `Chunk` objects that satisfy the metadata criteria.
+
+        Raises:
+            NotImplementedError: If not implemented in the subclass.
+        """
+    async def query_by_vector(self, vector: list[float], top_k: int = ..., min_similarity: float = 0.8, retrieval_params: dict | None = None) -> list[Chunk]:
+        """Search for documents that are similar to a given vector.
+
+        Args:
+            vector (list[float]): The query embedding vector to compare against stored vectors.
+            top_k (int, optional): The number of top results to return. Defaults to DEFAULT_TOP_K.
+            min_similarity (float): Minimum similarity score for vector similarity.
+            retrieval_params (dict | None, optional): Filter parameters to narrow the search:
+                - filter (Where): Metadata-based filter.
+                - where_document (WhereDocument): Content-based filter.
+                Defaults to None.
+
+        Returns:
+            list[Chunk]: A list of Chunk objects with similarity scores based on the input vector.
+        """

gllm_datastore.build/.gitignore

@@ -0,0 +1 @@
+*

gllm_datastore.pyi

@@ -0,0 +1,156 @@
+# This file was generated by Nuitka
+
+# Stubs included by default
+
+
+__name__ = ...
+
+
+
+# Modules used internally, to allow implicit dependencies to be seen:
+import os
+import abc
+import enum
+import typing
+import asyncio
+import functools
+import json
+import collections
+import collections.OrderedDict
+import datetime
+import gllm_core
+import gllm_core.schema
+import gllm_core.schema.chunk
+import gllm_core.utils
+import gllm_datastore.core.filters.FilterClause
+import gllm_datastore.core.filters.QueryFilter
+import gllm_datastore.core.filters.QueryOptions
+import gzip
+import pickle
+import shutil
+import time
+import gllm_core.utils.logger_manager
+import posixpath
+import gllm_datastore.cache.hybrid_cache.key_matcher.ExactKeyMatcher
+import gllm_datastore.utils.convert_ttl_to_seconds
+import gllm_core.utils.imports
+import Levenshtein
+import gllm_datastore.vector_data_store.ElasticsearchVectorDataStore
+import redis
+import hashlib
+import gllm_datastore.core.filters.FilterCondition
+import gllm_datastore.core.filters.FilterOperator
+import __future__
+import gllm_inference
+import gllm_inference.schema
+import pydantic
+import gllm_datastore.data_store.chroma.ChromaDataStore
+import gllm_datastore.data_store.elasticsearch.ElasticsearchDataStore
+import gllm_datastore.data_store.in_memory.InMemoryDataStore
+import gllm_datastore.data_store.redis.RedisDataStore
+import gllm_inference.em_invoker
+import gllm_inference.em_invoker.em_invoker
+import gllm_datastore.core.capabilities.FulltextCapability
+import gllm_datastore.core.capabilities.GraphCapability
+import gllm_datastore.core.capabilities.VectorCapability
+import gllm_datastore.cache.Cache
+import gllm_datastore.cache.MatchingStrategy
+import chromadb
+import sys
+import pysqlite3
+import rapidfuzz
+import rapidfuzz.distance
+import logging
+import re
+import dataclasses
+import gllm_inference.em_invoker.langchain
+import langchain_core
+import langchain_core.runnables
+import langchain_chroma
+import elasticsearch
+import elasticsearch.dsl
+import elasticsearch.dsl.query
+import gllm_core.utils.retry
+import gllm_datastore.utils.flatten_dict
+import langchain_elasticsearch
+import langchain_elasticsearch.vectorstores
+import collections.abc
+import gllm_core.utils.similarity
+import redis.asyncio
+import redis.asyncio.client
+import redis.exceptions
+import redis.commands
+import redis.commands.search
+import redis.commands.search.query
+import redis.commands.search.field
+import redis.commands.search.indexDefinition
+import numpy
+import redisvl
+import redisvl.redis
+import redisvl.redis.utils
+import redisvl.index
+import redisvl.query
+import redisvl.schema
+import base64
+import cryptography
+import cryptography.hazmat
+import cryptography.hazmat.primitives
+import cryptography.hazmat.primitives.ciphers
+import cryptography.hazmat.primitives.ciphers.aead
+import threading
+import lightrag
+import tempfile
+import contextlib
+import gllm_inference.lm_invoker
+import gllm_inference.lm_invoker.lm_invoker
+import lightrag.kg
+import lightrag.kg.shared_storage
+import llama_index
+import llama_index.core
+import llama_index.core.base
+import llama_index.core.base.embeddings
+import llama_index.core.base.embeddings.base
+import llama_index.core.graph_stores
+import llama_index.core.graph_stores.types
+import llama_index.core.llms
+import gllm_datastore.graph_data_store.utils.LlamaIndexEMInvokerAdapter
+import gllm_datastore.graph_data_store.utils.LlamaIndexLMInvokerAdapter
+import llama_index.graph_stores
+import llama_index.graph_stores.neo4j
+import nebula3
+import nebula3.Config
+import nebula3.data
+import nebula3.data.DataObject
+import nebula3.gclient
+import nebula3.gclient.net
+import textwrap
+import neo4j
+import neo4j.exceptions
+import lightrag.base
+import nest_asyncio
+import llama_index.core.bridge
+import llama_index.core.bridge.pydantic
+import llama_index.core.constants
+import llama_index.core.base.llms
+import llama_index.core.base.llms.types
+import sqlalchemy
+import sqlalchemy.engine
+import pandas
+import concurrent
+import concurrent.futures
+import concurrent.futures.Future
+import concurrent.futures.ThreadPoolExecutor
+import sqlalchemy.exc
+import sqlalchemy.orm
+import uuid
+import langchain_core.documents
+import langchain_core.embeddings
+import langchain_core.runnables.config
+import chromadb.types
+import inspect
+import redisvl.extensions
+import redisvl.extensions.cache
+import redisvl.extensions.cache.llm
+import redisvl.utils
+import redisvl.utils.vectorize
+import redisvl.query.filter