gllm-datastore-binary 0.5.45__cp311-cp311-macosx_13_0_arm64.whl

gllm_datastore/data_store/base.pyi

@@ -0,0 +1,138 @@
+from abc import ABC, abstractmethod
+from enum import StrEnum
+from gllm_datastore.cache import Cache as Cache
+from gllm_datastore.core.capabilities import FulltextCapability as FulltextCapability, GraphCapability as GraphCapability, VectorCapability as VectorCapability
+from gllm_datastore.core.filters.schema import FilterClause as FilterClause, QueryFilter as QueryFilter
+from gllm_datastore.data_store.exceptions import NotRegisteredException as NotRegisteredException, NotSupportedException as NotSupportedException
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from typing import Any
+
+class CapabilityType(StrEnum):
+    """Enumeration of supported capability types."""
+    FULLTEXT: str
+    GRAPH: str
+    VECTOR: str
+
+class BaseDataStore(ABC):
+    """Base class for datastores with multiple capabilities.
+
+    This class provides the infrastructure for capability composition and
+    delegation. Datastores inherit from this class and register capability
+    handlers based on their configuration.
+    """
+    def __init__(self) -> None:
+        """Initialize the datastore with specified capabilities."""
+    @property
+    @abstractmethod
+    def supported_capabilities(self) -> list[CapabilityType]:
+        """Return list of currently supported capabilities.
+
+        A data store might have more capabilities than the ones that are currently registered.
+        Each data store should implement this method to return the list of supported capabilities.
+
+        Returns:
+            list[str]: List of capability names that are supported.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by subclass.
+        """
+    @property
+    def registered_capabilities(self) -> list[CapabilityType]:
+        """Return list of currently registered capabilities.
+
+        Returns:
+            list[str]: List of capability names that are registered and available.
+        """
+    @property
+    def fulltext(self) -> FulltextCapability:
+        """Access fulltext capability if supported.
+
+        Returns:
+            FulltextCapability: Fulltext capability handler.
+
+        Raises:
+            NotSupportedException: If fulltext capability is not supported.
+        """
+    @property
+    def vector(self) -> VectorCapability:
+        """Access vector capability if supported.
+
+        Returns:
+            VectorCapability: Vector capability handler.
+
+        Raises:
+            NotSupportedException: If vector capability is not supported
+        """
+    @property
+    def graph(self) -> GraphCapability:
+        """Access graph capability if supported.
+
+        Returns:
+            GraphCapability: Graph capability handler.
+
+        Raises:
+            NotSupportedException: If graph capability is not supported.
+        """
+    def with_fulltext(self, **kwargs) -> Self:
+        """Configure fulltext capability and return datastore instance.
+
+        Args:
+            **kwargs: Fulltext capability configuration parameters.
+
+        Returns:
+            Self: Self for method chaining.
+        """
+    def with_vector(self, em_invoker: BaseEMInvoker, **kwargs) -> Self:
+        """Configure vector capability and return datastore instance.
+
+        Args:
+            em_invoker (BaseEMInvoker): Embedding model invoker (required).
+            **kwargs: Vector capability configuration parameters.
+
+        Returns:
+            Self: Self for method chaining.
+        """
+    def with_graph(self, **kwargs) -> Self:
+        """Configure graph capability and return datastore instance.
+
+        Args:
+            **kwargs: Graph capability configuration parameters.
+
+        Returns:
+            Self: Self for method chaining.
+        """
+    def as_cache(self, eviction_manager: Any | None = None, matching_strategy: Any = None) -> Cache:
+        """Create a Cache instance from this datastore.
+
+        Args:
+            eviction_manager (Any | None, optional): Optional eviction manager for cache eviction.
+                Defaults to None.
+            matching_strategy (Any, optional): Default matching strategy for cache retrieval.
+                Defaults to None.
+
+        Returns:
+            Cache: Instance wrapping this datastore.
+
+        Raises:
+            ValueError: If required capabilities not registered.
+        """
+    @classmethod
+    def translate_query_filter(cls, query_filter: FilterClause | QueryFilter | None) -> Any:
+        """Translate QueryFilter or FilterClause to datastore's native filter syntax.
+
+        This method provides a public interface for converting the GLLM DataStore's
+        QueryFilter DSL into each datastore's native filter format. Subclasses must
+        implement this method to provide their specific translation logic.
+
+        Args:
+            query_filter (FilterClause | QueryFilter | None): The filter to translate.
+                Can be a single FilterClause, a QueryFilter with multiple clauses,
+                or None for empty filters.
+
+        Returns:
+            Any: The translated filter in the datastore's native format.
+                Returns None for empty filters.
+
+        Raises:
+            NotImplementedError: If not implemented by subclass.
+        """

gllm_datastore/data_store/chroma/__init__.pyi

@@ -0,0 +1,4 @@
+from gllm_datastore.data_store.chroma.data_store import ChromaDataStore as ChromaDataStore
+from gllm_datastore.data_store.chroma.fulltext import ChromaFulltextCapability as ChromaFulltextCapability
+
+__all__ = ['ChromaDataStore', 'ChromaFulltextCapability']

gllm_datastore/data_store/chroma/_chroma_import.pyi

@@ -0,0 +1,13 @@
+import chromadb
+
+def safe_import_chromadb() -> chromadb:
+    """Import and return the `chromadb` module with SQLite fallback.
+
+    This function centralizes the logic to import `chromadb`, applying the
+    `pysqlite3` fallback for environments where the built-in sqlite3 causes
+    issues. Other modules should use `safe_import_chromadb()` to
+    avoid duplication.
+
+    Returns:
+        ModuleType: The imported `chromadb` module.
+    """

gllm_datastore/data_store/chroma/data_store.pyi

@@ -0,0 +1,202 @@
+from _typeshed import Incomplete
+from enum import StrEnum
+from gllm_datastore.core.filters.schema import FilterClause as FilterClause, QueryFilter as QueryFilter
+from gllm_datastore.data_store.base import BaseDataStore as BaseDataStore, CapabilityType as CapabilityType
+from gllm_datastore.data_store.chroma._chroma_import import safe_import_chromadb as safe_import_chromadb
+from gllm_datastore.data_store.chroma.fulltext import ChromaFulltextCapability as ChromaFulltextCapability
+from gllm_datastore.data_store.chroma.query import DEFAULT_NUM_CANDIDATES as DEFAULT_NUM_CANDIDATES
+from gllm_datastore.data_store.chroma.query_translator import ChromaQueryTranslator as ChromaQueryTranslator
+from gllm_datastore.data_store.chroma.vector import ChromaVectorCapability as ChromaVectorCapability
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from typing import Any
+
+chromadb: Incomplete
+
+class ChromaClientType(StrEnum):
+    """Enum for different types of ChromaDB clients."""
+    MEMORY: str
+    PERSISTENT: str
+    HTTP: str
+
+class ChromaDataStore(BaseDataStore):
+    """ChromaDB data store with multiple capability support.
+
+    Attributes:
+        collection_name (str): The name of the ChromaDB collection.
+        client (chromadb.ClientAPI): The ChromaDB client instance.
+    """
+    collection_name: Incomplete
+    client: Incomplete
+    def __init__(self, collection_name: str, client_type: ChromaClientType = ..., persist_directory: str | None = None, host: str | None = None, port: int | None = None, headers: dict | None = None, client_settings: dict | None = None) -> None:
+        """Initialize the ChromaDB data store.
+
+        Args:
+            collection_name (str): The name of the ChromaDB collection.
+            client_type (ChromaClientType, optional): Type of ChromaDB client to use.
+                Defaults to ChromaClientType.MEMORY.
+            persist_directory (str | None, optional): Directory to persist vector store data.
+                Required for PERSISTENT client type. Defaults to None.
+            host (str | None, optional): Host address for ChromaDB server.
+                Required for HTTP client type. Defaults to None.
+            port (int | None, optional): Port for ChromaDB server.
+                Required for HTTP client type. Defaults to None.
+            headers (dict | None, optional): A dictionary of headers to send to the Chroma server.
+                Used for authentication with the Chroma server for HTTP client type. Defaults to None.
+            client_settings (dict | None, optional): A dictionary of additional settings for the Chroma client.
+                Defaults to None.
+        """
+    @property
+    def supported_capabilities(self) -> list[str]:
+        """Return list of currently supported capabilities.
+
+        Returns:
+            list[str]: List of capability names that are supported.
+        """
+    @property
+    def fulltext(self) -> ChromaFulltextCapability:
+        """Access fulltext capability if supported.
+
+        This method uses the logic of its parent class to return the fulltext capability handler.
+        This method overrides the parent class to return the ChromaFulltextCapability handler for better
+        type hinting.
+
+        Returns:
+            ChromaFulltextCapability: Fulltext capability handler.
+
+        Raises:
+            NotSupportedException: If fulltext capability is not supported.
+        """
+    @property
+    def vector(self) -> ChromaVectorCapability:
+        """Access vector capability if supported.
+
+        This method uses the logic of its parent class to return the vector capability handler.
+        This method overrides the parent class to return the ChromaVectorCapability handler for better
+        type hinting.
+
+        Returns:
+            ChromaVectorCapability: Vector capability handler.
+
+        Raises:
+            NotSupportedException: If vector capability is not supported.
+        """
+    def with_fulltext(self, collection_name: str | None = None, num_candidates: int = ...) -> ChromaDataStore:
+        """Configure fulltext capability and return datastore instance.
+
+        This method uses the logic of its parent class to configure the fulltext capability.
+        This method overrides the parent class for better type hinting.
+
+        Args:
+            collection_name (str | None, optional): Name of the collection to use in ChromaDB. Defaults to None,
+                in which case the default class attribute will be utilized.
+            num_candidates (int, optional): Maximum number of candidates to consider during search.
+                Defaults to DEFAULT_NUM_CANDIDATES.
+
+        Returns:
+            Self: Self for method chaining.
+        """
+    def with_vector(self, em_invoker: BaseEMInvoker, collection_name: str | None = None, num_candidates: int = ...) -> ChromaDataStore:
+        """Configure vector capability and return datastore instance.
+
+        This method uses the logic of its parent class to configure the vector capability.
+        This method overrides the parent class for better type hinting.
+
+        Args:
+            em_invoker (BaseEMInvoker): The embedding model to perform vectorization.
+            collection_name (str | None, optional): Name of the collection to use in ChromaDB. Defaults to None,
+                in which case the default class attribute will be utilized.
+            num_candidates (int, optional): Maximum number of candidates to consider during search.
+                Defaults to DEFAULT_NUM_CANDIDATES.
+
+        Returns:
+            Self: Self for method chaining.
+        """
+    @classmethod
+    def translate_query_filter(cls, query_filter: FilterClause | QueryFilter | None = None) -> dict[str, Any] | None:
+        '''Translate QueryFilter or FilterClause to ChromaDB native filter syntax.
+
+        This method uses ChromaQueryTranslator to translate filters and returns
+        the result as a dictionary.
+
+        Examples:
+            1. Translate a simple FilterClause:
+                ```python
+                from gllm_datastore.core.filters import filter as F
+
+                filter_clause = F.eq("metadata.status", "active")
+                result = ChromaDataStore.translate_query_filter(filter_clause)
+                # result -> {"where": {"status": "active"}}
+                ```
+
+            2. Translate QueryFilter with metadata filters:
+                ```python
+                from gllm_datastore.core.filters import filter as F
+
+                filters = F.and_(
+                    F.eq("metadata.category", "tech"),
+                    F.gte("metadata.price", 10),
+                )
+                result = ChromaDataStore.translate_query_filter(filters)
+                # result ->
+                # {
+                #   "where": {
+                #     "$and": [
+                #       {"category": "tech"},
+                #       {"price": {"$gte": 10}}
+                #     ]
+                #   }
+                # }
+                ```
+
+            3. Translate QueryFilter with content filters:
+                ```python
+                from gllm_datastore.core.filters import filter as F
+
+                filters = F.text_contains("content", "python")
+                result = ChromaDataStore.translate_query_filter(filters)
+                # result -> {"where_document": {"$contains": "python"}}
+                ```
+
+            4. Translate QueryFilter with id filters:
+                ```python
+                from gllm_datastore.core.filters import filter as F
+
+                filters = F.in_("id", ["chunk_1", "chunk_2"])
+                result = ChromaDataStore.translate_query_filter(filters)
+                # result -> {"ids": ["chunk_1", "chunk_2"]}
+                ```
+
+            5. Translate complex nested QueryFilter:
+                ```python
+                from gllm_datastore.core.filters import filter as F
+
+                filters = F.and_(
+                    F.or_(
+                        F.eq("metadata.status", "active"),
+                        F.eq("metadata.status", "pending"),
+                    ),
+                    F.text_contains("content", "machine learning"),
+                    F.in_("id", ["chunk_1", "chunk_2"]),
+                )
+                result = ChromaDataStore.translate_query_filter(filters)
+                # result ->
+                # {
+                #   "where": {
+                #     "$or": [
+                #       {"status": "active"},
+                #       {"status": "pending"}
+                #     ]
+                #   },
+                #   "where_document": {"$contains": "machine learning"},
+                #   "ids": ["chunk_1", "chunk_2"]
+                # }
+                ```
+
+        Args:
+            query_filter (FilterClause | QueryFilter | None, optional): The filter to translate.
+                Can be a single FilterClause, a QueryFilter with multiple clauses. Defaults to None.
+
+        Returns:
+            dict[str, Any] | None: The translated filter as a ChromaDB query dict.
+                Returns None for empty filters.
+        '''

gllm_datastore/data_store/chroma/fulltext.pyi

@@ -0,0 +1,134 @@
+from _typeshed import Incomplete
+from chromadb import ClientAPI
+from gllm_core.schema import Chunk
+from gllm_datastore.constants import CHUNK_KEYS as CHUNK_KEYS, METADATA_KEYS as METADATA_KEYS
+from gllm_datastore.core.filters import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_datastore.data_store.chroma._chroma_import import safe_import_chromadb as safe_import_chromadb
+from gllm_datastore.data_store.chroma.query import ChromaCollectionKeys as ChromaCollectionKeys, DEFAULT_NUM_CANDIDATES as DEFAULT_NUM_CANDIDATES, build_chroma_delete_kwargs as build_chroma_delete_kwargs, build_chroma_get_kwargs as build_chroma_get_kwargs, sanitize_metadata as sanitize_metadata
+from gllm_datastore.data_store.chroma.query_translator import ChromaQueryTranslator as ChromaQueryTranslator
+from typing import Any
+
+chromadb: Incomplete
+
+class ChromaFulltextCapability:
+    """ChromaDB implementation of FulltextCapability protocol.
+
+    This class provides document CRUD operations and text search using ChromaDB.
+
+    Attributes:
+        collection_name (str): The name of the ChromaDB collection.
+        client (ClientAPI): ChromaDB client instance.
+        collection: ChromaDB collection instance.
+        num_candidates (int): Maximum number of candidates to consider during search.
+    """
+    collection_name: Incomplete
+    client: Incomplete
+    collection: Incomplete
+    num_candidates: Incomplete
+    def __init__(self, collection_name: str, client: ClientAPI, num_candidates: int = ...) -> None:
+        """Initialize the ChromaDB fulltext capability.
+
+        Args:
+            collection_name (str): The name of the ChromaDB collection.
+            client (ClientAPI): ChromaDB client instance.
+            num_candidates (int, optional): Maximum number of candidates to consider during search.
+                Defaults to DEFAULT_NUM_CANDIDATES.
+        """
+    def get_size(self) -> int:
+        """Returns the total number of documents in the collection.
+
+        Returns:
+            int: The total number of documents.
+        """
+    async def create(self, data: Chunk | list[Chunk], **kwargs: Any) -> None:
+        """Create new records in the datastore.
+
+        Args:
+            data (Chunk | list[Chunk]): Data to create (single item or collection).
+            **kwargs: Backend-specific parameters.
+
+        Raises:
+            ValueError: If data structure is invalid.
+        """
+    async def retrieve(self, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None, **kwargs: Any) -> list[Chunk]:
+        '''Read records from the datastore with optional filtering.
+
+        Usage Example:
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            results = await fulltext_capability.retrieve(filters=F.eq("metadata.category", "tech"))
+
+            # Multiple filters
+            results = await fulltext_capability.retrieve(
+                filters=F.and_(F.eq("metadata.category", "tech"), F.eq("metadata.status", "active"))
+            )
+            ```
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options (sorting, pagination, etc.).
+                Defaults to None.
+            **kwargs: Backend-specific parameters.
+
+        Returns:
+            list[Chunk]: Query results.
+
+        Raises:
+            NotImplementedError: If unsupported operators are used for id or content filters.
+        '''
+    async def retrieve_fuzzy(self, query: str, max_distance: int = 2, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None, **kwargs: Any) -> list[Chunk]:
+        """Find records that fuzzy match the query within distance threshold.
+
+        Args:
+            query (str): Text to fuzzy match against.
+            max_distance (int): Maximum edit distance for matches. Defaults to 2.
+            filters (FilterClause | QueryFilter | None, optional): Optional metadata filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options (sorting, limit, etc.). Defaults to None.
+            **kwargs: Backend-specific parameters.
+
+        Returns:
+            list[Chunk]: Matched chunks ordered by distance (ascending) or by options.order_by if specified.
+        """
+    async def update(self, update_values: dict[str, Any], filters: FilterClause | QueryFilter | None = None) -> None:
+        '''Update existing records in the datastore.
+
+        Examples:
+            Update the content and metadata of the chunk with the id "unique_id" to "updated_content"
+            and "published" respectively.
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            await fulltext_capability.update(
+                update_values={"content": "updated_content", "metadata": {"status": "published"}},
+                filters=F.eq("id", "unique_id"),
+            )
+            ```
+
+        Args:
+            update_values (dict[str, Any]): Values to update. Supports "content" for updating document content
+                and "metadata" for updating metadata. Other keys are treated as direct metadata updates.
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+
+        Note:
+            ChromaDB doesn\'t support direct update operations. This method will
+            retrieve matching records, update them, and re-add them to the collection.
+        '''
+    async def delete(self, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> None:
+        """Delete records from the datastore.
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to delete.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None, in which case no operation is performed (no-op).
+            options (QueryOptions | None, optional): Query options for sorting and limiting deletions. Defaults to None.
+        """
+    async def clear(self) -> None:
+        """Clear all records from the datastore."""