gllm-datastore-binary 0.5.50__cp312-cp312-macosx_13_0_arm64.whl

gllm_datastore/sql_data_store/types.pyi

@@ -0,0 +1,31 @@
+from pydantic import BaseModel
+from typing import Any, Sequence
+
+class QueryFilter(BaseModel):
+    '''Model for query filters.
+
+    Attributes:
+        conditions (dict[str, Any]): The conditions for filtering the query.
+
+    Example:
+        QueryFilter(conditions={"column1": "value1", "column2": "value2"})
+    '''
+    conditions: dict[str, Any]
+
+class QueryOptions(BaseModel):
+    '''Model for query options.
+
+    Attributes:
+        columns (Sequence[str] | None): The columns to include in the query result. Defaults to None.
+        fields (Sequence[str] | None): The fields to include in the query result. Defaults to None.
+        order_by (str | None): The column to order the query result by. Defaults to None.
+        order_desc (bool): Whether to order the query result in descending order. Defaults to False.
+        limit (int | None): The maximum number of rows to return. Defaults to None.
+
+    Example:
+        QueryOptions(fields=["field1", "field2"], order_by="column1", order_desc=True, limit=10)
+    '''
+    columns: Sequence[str] | None
+    order_by: str | None
+    order_desc: bool
+    limit: int | None

gllm_datastore/utils/__init__.pyi

@@ -0,0 +1,6 @@
+from gllm_datastore.utils.converter import from_langchain as from_langchain
+from gllm_datastore.utils.dict import flatten_dict as flatten_dict
+from gllm_datastore.utils.ttl import convert_ttl_to_seconds as convert_ttl_to_seconds
+from gllm_datastore.utils.types import QueryFilter as QueryFilter, QueryOptions as QueryOptions
+
+__all__ = ['from_langchain', 'convert_ttl_to_seconds', 'flatten_dict', 'QueryFilter', 'QueryOptions']

gllm_datastore/utils/converter.pyi

@@ -0,0 +1,51 @@
+from gllm_core.schema import Chunk
+from gllm_datastore.constants import SIMILARITY_SCORE as SIMILARITY_SCORE
+from langchain_core.documents import Document
+
+def from_langchain(doc: Document, score: float | None = None) -> Chunk:
+    """Create a standardized Chunk from a LangChain Document.
+
+    Args:
+        doc (Document): The document to create a Chunk from.
+        score (float | None, optional): The score to assign to the Chunk. Defaults to None, in which case it will
+            attempt to get the score from the `score` metadata.
+
+    Returns:
+        Chunk: The standardized Chunk object.
+    """
+def to_langchain(chunk: Chunk) -> Document:
+    """Create a LangChain Document from a standardized Chunk.
+
+    Args:
+        chunk (Chunk): The standardized Chunk to create a Document from.
+
+    Returns:
+        Document: The LangChain Document object.
+    """
+def l2_distance_to_similarity_score(distance: float) -> float:
+    """Convert distance to similarity.
+
+    Args:
+        distance (float): The distance value to convert. Ranges in [0, inf].
+
+    Returns:
+        float: The converted similarity value.
+    """
+def cosine_distance_to_similarity_score(distance: float) -> float:
+    """Convert cosine distance to similarity.
+
+    Args:
+        distance (float): The cosine distance value to convert. Ranges in [0, 2].
+
+    Returns:
+        float: The converted similarity value. Ranges in [0, 1].
+    """
+def similarity_score_to_cosine_distance(similarity: float) -> float:
+    """Convert similarity to cosine distance.
+
+    Args:
+        similarity (float): The similarity value to convert. Ranges in [0, 1].
+
+    Returns:
+        float: The converted cosine distance value. Ranges in [0, 2].
+    """

gllm_datastore/utils/dict.pyi

@@ -0,0 +1,21 @@
+from typing import Any
+
+def flatten_dict(nested_dict: dict[str, Any], parent_key: str = '', sep: str = '.') -> dict[str, Any]:
+    '''Flatten a nested dictionary into a single level dictionary.
+
+    Args:
+        nested_dict (dict[str, Any]): The nested dictionary to flatten.
+        parent_key (str, optional): The parent key to prepend to the keys in the flattened dictionary.
+            Defaults to empty string.
+        sep (str, optional): The separator to use between the parent key and the child key. Defaults to ".".
+
+    Returns:
+        dict[str, Any]: The flattened dictionary.
+
+    Examples:
+        ```python
+        nested = {"a": {"b": 1, "c": 2}, "d": 3}
+        flattened = flatten_dict(nested)
+        # Result: {"a.b": 1, "a.c": 2, "d": 3}
+        ```
+    '''

gllm_datastore/utils/ttl.pyi

@@ -0,0 +1,25 @@
+from _typeshed import Incomplete
+
+TIME_UNIT_TO_SECOND_MAPPING: Incomplete
+
+def convert_ttl_to_seconds(ttl: str | int) -> int:
+    '''Convert TTL (time-to-live) string with time units to seconds.
+
+    Supported units: s (seconds), m (minutes), h (hours), d (days), w (weeks), y (years).
+
+    Examples:
+        "2m" -> 120 (2 minutes in seconds)
+        "1h" -> 3600 (1 hour in seconds)
+        "1y" -> 31536000 (1 year in seconds)
+        300 -> 300 (numeric input returned as is)
+
+    Args:
+        ttl (str | int): Time to live value with optional unit suffix (e.g., "2m", "1h", "1y")
+            or numeric value in seconds.
+
+    Returns:
+        int: TTL converted to seconds.
+
+    Raises:
+        ValueError: If the input format is invalid.
+    '''

gllm_datastore/utils/types.pyi

@@ -0,0 +1,32 @@
+from pydantic import BaseModel
+from typing import Any, Sequence
+
+class QueryFilter(BaseModel):
+    '''Model for query filters.
+
+    Attributes:
+        conditions (dict[str, Any]): The conditions for filtering the query.
+
+    Example:
+        QueryFilter(conditions={"column1": "value1", "column2": "value2"})
+    '''
+    conditions: dict[str, Any]
+
+class QueryOptions(BaseModel):
+    '''Model for query options.
+
+    Attributes:
+        columns (Sequence[str] | None): The columns to include in the query result. Defaults to None.
+        fields (Sequence[str] | None): The fields to include in the query result. Defaults to None.
+        order_by (str | None): The column to order the query result by. Defaults to None.
+        order_desc (bool): Whether to order the query result in descending order. Defaults to False.
+        limit (int | None): The maximum number of rows to return. Defaults to None.
+
+    Example:
+        QueryOptions(fields=["field1", "field2"], order_by="column1", order_desc=True, limit=10)
+    '''
+    columns: Sequence[str] | None
+    fields: Sequence[str] | None
+    order_by: str | None
+    order_desc: bool
+    limit: int | None

gllm_datastore/vector_data_store/__init__.pyi

@@ -0,0 +1,6 @@
+from gllm_datastore.vector_data_store.chroma_vector_data_store import ChromaVectorDataStore as ChromaVectorDataStore
+from gllm_datastore.vector_data_store.elasticsearch_vector_data_store import ElasticsearchVectorDataStore as ElasticsearchVectorDataStore
+from gllm_datastore.vector_data_store.in_memory_vector_data_store import InMemoryVectorDataStore as InMemoryVectorDataStore
+from gllm_datastore.vector_data_store.redis_vector_data_store import RedisVectorDataStore as RedisVectorDataStore
+
+__all__ = ['ChromaVectorDataStore', 'ElasticsearchVectorDataStore', 'InMemoryVectorDataStore', 'RedisVectorDataStore']

gllm_datastore/vector_data_store/chroma_vector_data_store.pyi

@@ -0,0 +1,259 @@
+from _typeshed import Incomplete
+from chromadb.types import Where, WhereDocument
+from datetime import datetime
+from enum import Enum
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.constants import DEFAULT_TOP_K as DEFAULT_TOP_K, METADATA_KEYS as METADATA_KEYS
+from gllm_datastore.utils.converter import from_langchain as from_langchain, l2_distance_to_similarity_score as l2_distance_to_similarity_score, to_langchain as to_langchain
+from gllm_datastore.vector_data_store.mixin.cache_compatible_mixin import CacheCompatibleMixin as CacheCompatibleMixin
+from gllm_datastore.vector_data_store.vector_data_store import BaseVectorDataStore as BaseVectorDataStore
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from langchain_core.embeddings import Embeddings
+from typing import Any
+
+DEFAULT_NUM_CANDIDATES: int
+
+class ChromaClientType(str, Enum):
+    """Enum for different types of ChromaDB clients.
+
+    Attributes:
+        MEMORY (str): Client type for an in-memory data store.
+        PERSISTENT (str): Client type for a persistent data store.
+        HTTP (str): Client type for a client-server architecture.
+    """
+    MEMORY: str
+    PERSISTENT: str
+    HTTP: str
+
+class ChromaVectorDataStore(BaseVectorDataStore, CacheCompatibleMixin):
+    """Datastore for interacting with ChromaDB.
+
+    This class provides methods to interact with ChromaDB for vector storage and retrieval
+    using the langchain-chroma integration.
+
+    Attributes:
+        vector_store (Chroma): The langchain Chroma vector store instance.
+        collection_name (str): The name of the ChromaDB collection to use.
+        num_candidates (int): The maximum number of candidates to consider during search.
+        embedding (BaseEMInvoker | Embeddings | None): The embedding model to perform vectorization.
+    """
+    vector_store: Incomplete
+    collection_name: Incomplete
+    num_candidates: Incomplete
+    def __init__(self, collection_name: str, embedding: BaseEMInvoker | Embeddings | None = None, client_type: ChromaClientType = ..., persist_directory: str | None = None, host: str | None = None, port: int | None = None, headers: dict | None = None, num_candidates: int = ..., **kwargs: Any) -> None:
+        """Initialize the ChromaDB vector data store with langchain-chroma.
+
+        Args:
+            collection_name (str): Name of the collection to use in ChromaDB.
+            embedding (BaseEMInvoker | Embeddings | None, optional): The embedding model to perform vectorization.
+                Defaults to None.
+            client_type (ChromaClientType, optional): Type of ChromaDB client to use.
+                Defaults to ChromaClientType.MEMORY.
+            persist_directory (str | None, optional): Directory to persist vector store data.
+                Required for PERSISTENT client type. Defaults to None.
+            host (str | None, optional): Host address for ChromaDB server.
+                Required for HTTP client type. Defaults to None.
+            port (int | None, optional): Port for ChromaDB server.
+                Required for HTTP client type. Defaults to None.
+            headers (dict | None, optional): Headers for ChromaDB server.
+                Used for HTTP client type. Defaults to None.
+            num_candidates (int, optional): Maximum number of candidates to consider during search.
+                Defaults to DEFAULT_NUM_CANDIDATES.
+            **kwargs: Additional parameters for Chroma initialization.
+
+        Note:
+            num_candidates (int, optional): This constant affects the maximum number of results to consider
+            during the search. Index with more documents would need a higher value for the whole documents
+            to be considered during search. This happens due to a bug with Chroma's search algorithm as discussed
+            in this issue: [3] https://github.com/langchain-ai/langchain/issues/1946
+        """
+    async def get_size(self) -> int:
+        """Returns the total number of vectors in the index.
+
+        If the index is not initialized returns 0.
+
+        Returns:
+            int: The total number of vectors.
+        """
+    async def query(self, query: str, top_k: int = ..., retrieval_params: dict[str, dict[str, str]] | None = None) -> list[Chunk]:
+        '''Query the vector data store for similar chunks with similarity scores.
+
+        Args:
+            query (str): The query string to find similar chunks for.
+            top_k (int, optional): Maximum number of results to return. Defaults to DEFAULT_TOP_K.
+            retrieval_params (dict[str, Any] | None, optional): Additional parameters for retrieval.
+                - filter (Where, optional): A Where type dict used to filter the retrieval by the metadata keys.
+                    E.g. `{"$and": [{"color" : "red"}, {"price": {"$gte": 4.20}]}}`.
+                - where_document (WhereDocument, optional): A WhereDocument type dict used to filter the retrieval by
+                    the document content. E.g. `{$contains: {"text": "hello"}}`.
+                Defaults to None.
+
+        Returns:
+            list[Chunk]: A list of Chunk objects matching the query, with similarity scores.
+        '''
+    async def query_by_id(self, id: str | list[str]) -> list[Chunk]:
+        """Retrieve chunks by their IDs.
+
+        Args:
+            id (str | list[str]): A single ID or a list of IDs to retrieve.
+
+        Returns:
+            list[Chunk]: A list of retrieved Chunk objects.
+        """
+    async def add_chunks(self, chunks: Chunk | list[Chunk], **kwargs) -> list[str]:
+        """Add chunks to the vector data store.
+
+        Args:
+            chunks (Chunk | list[Chunk]): A single chunk or list of chunks to add.
+            **kwargs: Additional keyword arguments for the add operation.
+
+        Returns:
+            list[str]: List of IDs of the added chunks.
+        """
+    async def delete_chunks(self, where: Where | None = None, where_document: WhereDocument | None = None, **kwargs: Any) -> None:
+        '''Delete chunks from the vector data store.
+
+        Args:
+            where (Where | None, optional): A Where type dict used to filter the deletion by metadata.
+                E.g. `{"source": "mydoc"}`. Defaults to None.
+            where_document (WhereDocument | None, optional): A WhereDocument type dict used to filter the deletion by
+                the document content. E.g. `{$contains: {"text": "hello"}}`. Defaults to None.
+            **kwargs: Additional keyword arguments for the delete operation.
+
+        Note:
+            If no filter criteria is provided, all chunks in the collection will be deleted. Please use with caution.
+        '''
+    async def delete_chunks_by_ids(self, ids: str | list[str], **kwargs: Any) -> None:
+        """Delete chunks from the vector data store by IDs.
+
+        Args:
+            ids (str | list[str]): A single ID or a list of IDs to delete.
+            **kwargs: Additional keyword arguments.
+
+        Note:
+            If no IDs are provided, no chunks will be deleted.
+        """
+    async def exact_match(self, key: str, metadata: dict[str, Any] | None = None) -> Any | None:
+        '''Find chunks that exactly match the given key.
+
+        This method searches for documents with the exact original_key in metadata.
+
+        Args:
+            key (str): The key to match.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+
+        Returns:
+            Any: The value stored with the exact key match, or None if no match is found.
+        '''
+    async def fuzzy_match(self, key: str, max_distance: int = 2, metadata: dict[str, Any] | None = None) -> Any | None:
+        '''Find chunks that approximately match the given key using fuzzy matching.
+
+        Args:
+            key (str): The key to match.
+            max_distance (int): Maximum allowed Levenshtein distance for fuzzy matching.
+                Higher values are more lenient. Defaults to 2.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+
+        Returns:
+            Any: The value with the closest fuzzy match to the key, or None if no match meets the threshold.
+        '''
+    async def semantic_match(self, key: str, min_similarity: float = 0.2, metadata: dict[str, Any] | None = None) -> Any | None:
+        '''Find chunks that semantically match the given key using vector similarity.
+
+        Args:
+            key (str): The key to match.
+            min_similarity (float): Minimum similarity score for semantic matching
+                (higher values are more strict). Ranges from 0 to 1. Defaults to 0.8.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+
+        Returns:
+            Any: The semantically closest value, or None if no match meets the min_similarity.
+        '''
+    async def delete_expired_entries(self, now: datetime, max_size: int = 10000) -> None:
+        """Delete expired entries (for TTL eviction).
+
+        Args:
+            now (datetime): The current datetime for comparison.
+            max_size (int): The maximum number of entries to return. Defaults to 10000.
+
+        Raises:
+            NotImplementedError: Currently, app-level eviction is not supported for ChromaVectorDataStore.
+        """
+    async def delete_least_frequently_used_entries(self, num_entries: int) -> None:
+        """Delete least frequently used entries (for LFU eviction).
+
+        Args:
+            num_entries (int): Number of entries to return.
+
+        Raises:
+            NotImplementedError: Currently, app-level eviction is not supported for ChromaVectorDataStore.
+        """
+    async def delete_least_recently_used_entries(self, num_entries: int) -> None:
+        """Delete least recently used entries (for LRU eviction).
+
+        Args:
+            num_entries (int): Number of entries to return.
+
+        Raises:
+            NotImplementedError: Currently, app-level eviction is not supported for ChromaVectorDataStore.
+        """
+    async def delete_entries_by_key(self, key: str, metadata: dict[str, Any] | None = None) -> None:
+        '''Delete entries by key.
+
+        Args:
+            key (str): The key to delete entries for.
+            metadata (dict[str, Any] | None, optional): Optional metadata filter to apply to the search.
+                For example, `{"key": "value"}`. Defaults to None.
+
+        Raises:
+            NotImplementedError: Currently, app-level eviction is not supported for ChromaVectorDataStore.
+        '''
+    async def clear(self) -> None:
+        """Clear all entries in the storage.
+
+        Raises:
+            NotImplementedError: Currently, app-level eviction is not supported for ChromaVectorDataStore.
+        """
+    async def query_by_field(self, retrieval_params: dict[str, Any], limit: int | None = None, **kwargs) -> list[Chunk]:
+        """Retrieve documents that match specific metadata constraints.
+
+        This method filters and returns stored chunks based on metadata values
+        rather than vector similarity. It is particularly useful for structured lookups,
+        such as retrieving all chunks from a certain source, tagged with a specific label,
+        or authored by a particular user.
+
+        Unlike semantic search methods, `query_by_field` operates purely on metadata fields
+        associated with each document, allowing precise filtering based on key-value pairs.
+
+        Args:
+            retrieval_params (dict[str, Any]): A dictionary defining filter criteria. Common keys include:
+                - `filter` (dict): A dictionary of metadata field conditions.
+                - `where_document` (dict, optional): Conditions based on document content.
+            limit (int | None, optional): The maximum number of results to return. If None, all matching
+                documents will be returned.
+            **kwargs: Additional arguments to support datastore-specific behavior or filtering logic.
+
+        Returns:
+            list[Chunk]: A list of `Chunk` objects that satisfy the metadata criteria.
+
+        Raises:
+            NotImplementedError: If not implemented in the subclass.
+        """
+    async def query_by_vector(self, vector: list[float], top_k: int = ..., min_similarity: float = 0.8, retrieval_params: dict | None = None) -> list[Chunk]:
+        """Search for documents that are similar to a given vector.
+
+        Args:
+            vector (list[float]): The query embedding vector to compare against stored vectors.
+            top_k (int, optional): The number of top results to return. Defaults to DEFAULT_TOP_K.
+            min_similarity (float): Minimum similarity score for vector similarity.
+            retrieval_params (dict | None, optional): Filter parameters to narrow the search:
+                - filter (Where): Metadata-based filter.
+                - where_document (WhereDocument): Content-based filter.
+                Defaults to None.
+
+        Returns:
+            list[Chunk]: A list of Chunk objects with similarity scores based on the input vector.
+        """