gllm-datastore-binary 0.5.50__cp312-cp312-macosx_13_0_arm64.whl

gllm_datastore/data_store/elasticsearch/query.pyi

@@ -0,0 +1,118 @@
+import logging
+from _typeshed import Incomplete
+from elasticsearch import AsyncElasticsearch
+from elasticsearch.dsl import AsyncSearch
+from elasticsearch.dsl.query import Query
+from elasticsearch.dsl.response import Response
+from gllm_datastore.core.filters.schema import QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_datastore.data_store.elasticsearch.query_translator import ElasticsearchQueryTranslator as ElasticsearchQueryTranslator
+from gllm_datastore.utils import flatten_dict as flatten_dict
+from typing import Any
+
+VALID_FIELD_PATH: Incomplete
+
+def apply_filters_and_options(search: AsyncSearch, filters: QueryFilter | None = None, options: QueryOptions | None = None) -> AsyncSearch:
+    """Apply filters and options to an Elasticsearch search object.
+
+    Args:
+        search (AsyncSearch): Elasticsearch search object.
+        filters (QueryFilter | None, optional): New QueryFilter with filters and condition.
+        options (QueryOptions | None, optional): Query options (limit, sort, fields).
+
+    Returns:
+        AsyncSearch: Elasticsearch search object.
+    """
+def translate_filter(filters: QueryFilter | None) -> Query | None:
+    """Translate a structured QueryFilter into an Elasticsearch DSL Query.
+
+    The translation supports comparison operators (EQ, NE, GT, LT, GTE, LTE),
+    array operators (IN, NIN, ARRAY_CONTAINS, ANY, ALL), text operators (TEXT_CONTAINS),
+    and logical conditions (AND, OR, NOT), including nested filters.
+
+    Args:
+        filters (QueryFilter | None): Structured QueryFilter. If None, returns None.
+
+    Returns:
+        Query | None: An Elasticsearch Query object or None if no filters are provided.
+
+    Raises:
+        ValueError: When the filter structure is invalid.
+        TypeError: When an operator-value type combination is invalid.
+    """
+async def update_by_query(client: AsyncElasticsearch, index_name: str, update_values: dict[str, Any], filters: QueryFilter | None = None, logger: logging.Logger | None = None) -> None:
+    '''Update records in Elasticsearch using UpdateByQuery with retry logic for version conflicts.
+
+    This function builds a painless script that safely assigns each updated field.
+    When a field path contains dots (e.g. "metadata.cache_value"), we must
+    access the corresponding param using bracket syntax: params[\'metadata.cache_value\']
+    to avoid Painless treating it as nested object access (which would be None).
+
+    Args:
+        client (AsyncElasticsearch): Elasticsearch client instance.
+        index_name (str): The name of the Elasticsearch index.
+        update_values (dict[str, Any]): Values to update.
+        filters (QueryFilter | None, optional): New QueryFilter to select records to update.
+            Defaults to None.
+        logger (Any | None, optional): Logger instance. Defaults to None.
+    '''
+async def delete_by_query(client: AsyncElasticsearch, index_name: str, filters: QueryFilter | None = None) -> None:
+    """Delete records from Elasticsearch using delete_by_query.
+
+    Args:
+        client (AsyncElasticsearch): Elasticsearch client instance.
+        index_name (str): The name of the Elasticsearch index.
+        filters (QueryFilter | None, optional): New QueryFilter to select records for deletion.
+            Defaults to None, in which case no operation will be performed.
+    """
+async def delete_by_id(client: AsyncElasticsearch, index_name: str, ids: str | list[str]) -> None:
+    """Delete records from Elasticsearch by IDs using Search.delete().
+
+    Args:
+        client (AsyncElasticsearch): Elasticsearch client instance.
+        index_name (str): The name of the Elasticsearch index.
+        ids (str | list[str]): ID or list of IDs to delete.
+    """
+def validate_query_length(query: str, min_length: int = 0, max_length: int | None = None) -> bool:
+    """Validate query length against minimum and maximum constraints.
+
+    Args:
+        query (str): The query string to validate.
+        min_length (int, optional): Minimum required length. Defaults to 0.
+        max_length (int | None, optional): Maximum allowed length. Defaults to None.
+
+    Returns:
+        bool: True if query is valid, False otherwise.
+    """
+def create_search_with_filters(client: AsyncElasticsearch, index_name: str, filters: QueryFilter | None = None, exclude_fields: list[str] | None = None) -> AsyncSearch:
+    """Create an AsyncSearch object with optional filters and field exclusions.
+
+    Args:
+        client (AsyncElasticsearch): Elasticsearch client instance.
+        index_name (str): The name of the Elasticsearch index.
+        filters (QueryFilter | None, optional): New QueryFilter to apply. Defaults to None.
+        exclude_fields (list[str] | None, optional): Fields to exclude from source. Defaults to None.
+
+    Returns:
+        AsyncSearch: Configured AsyncSearch object.
+    """
+def apply_filter_query_to_search(search: AsyncSearch, main_query: Query, filters: QueryFilter | None = None) -> AsyncSearch:
+    """Apply filter query to a search with a main query.
+
+    Args:
+        search (AsyncSearch): Elasticsearch search object.
+        main_query (Query): The main query to apply.
+        filters (QueryFilter | None, optional): Query filters to apply. Defaults to None.
+
+    Returns:
+        AsyncSearch: Search object with applied queries.
+    """
+async def safe_execute(search: AsyncSearch, logger: logging.Logger | None = None) -> Response | None:
+    """Execute an Elasticsearch DSL search with unified error handling.
+
+    Args:
+        search (AsyncSearch): Elasticsearch DSL AsyncSearch object.
+        logger (logging.Logger | None, optional): Logger instance for error messages. Defaults to None.
+
+    Returns:
+        Response | None: The Elasticsearch response on success, otherwise None.
+    """

gllm_datastore/data_store/elasticsearch/query_translator.pyi

@@ -0,0 +1,18 @@
+from gllm_datastore.data_store._elastic_core.query_translator import ElasticLikeQueryTranslator as ElasticLikeQueryTranslator
+
+class ElasticsearchQueryTranslator(ElasticLikeQueryTranslator):
+    """Translates QueryFilter and FilterClause objects to Elasticsearch Query DSL.
+
+    This class extends ElasticLikeQueryTranslator and implements abstract methods
+    using Elasticsearch DSL API. It also provides QueryOptions handling
+    methods specific to Elasticsearch.
+
+    Attributes:
+        _logger (Logger): Logger instance for error messages and debugging.
+    """
+    def __init__(self) -> None:
+        """Initialize the Elasticsearch query translator.
+
+        Raises:
+            ImportError: If elasticsearch package is not installed.
+        """

gllm_datastore/data_store/elasticsearch/vector.pyi

@@ -0,0 +1,180 @@
+from _typeshed import Incomplete
+from elasticsearch import AsyncElasticsearch
+from gllm_core.schema import Chunk
+from gllm_datastore.constants import DEFAULT_FETCH_K as DEFAULT_FETCH_K, DEFAULT_TOP_K as DEFAULT_TOP_K
+from gllm_datastore.core.filters.schema import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_datastore.data_store._elastic_core.query_translator import convert_filter_clause as convert_filter_clause
+from gllm_datastore.data_store.elasticsearch.query import delete_by_id as delete_by_id, delete_by_query as delete_by_query, translate_filter as translate_filter, update_by_query as update_by_query
+from gllm_datastore.utils.converter import from_langchain as from_langchain, to_langchain as to_langchain
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from gllm_inference.schema import Vector
+from langchain_elasticsearch.vectorstores import AsyncRetrievalStrategy
+from typing import Any
+
+class ElasticsearchVectorCapability:
+    """Elasticsearch implementation of VectorCapability protocol.
+
+    This class provides document CRUD operations and vector search using Elasticsearch.
+
+    Attributes:
+        index_name (str): The name of the Elasticsearch index.
+        vector_store (AsyncElasticsearchStore): The vector store instance.
+        em_invoker (BaseEMInvoker): The embedding model to perform vectorization.
+    """
+    index_name: Incomplete
+    vector_store: Incomplete
+    def __init__(self, index_name: str, client: AsyncElasticsearch, em_invoker: BaseEMInvoker, query_field: str = 'text', vector_query_field: str = 'vector', retrieval_strategy: AsyncRetrievalStrategy | None = None, distance_strategy: str | None = None) -> None:
+        '''Initialize the Elasticsearch vector capability.
+
+        Args:
+            index_name (str): The name of the Elasticsearch index.
+            client (AsyncElasticsearch): The Elasticsearch client.
+            em_invoker (BaseEMInvoker): The embedding model to perform vectorization.
+            query_field (str, optional): The field name for text queries. Defaults to "text".
+            vector_query_field (str, optional): The field name for vector queries. Defaults to "vector".
+            retrieval_strategy (AsyncRetrievalStrategy | None, optional): The retrieval strategy for retrieval.
+                Defaults to None, in which case DenseVectorStrategy() is used.
+            distance_strategy (str | None, optional): The distance strategy for retrieval. Defaults to None.
+        '''
+    @property
+    def em_invoker(self) -> BaseEMInvoker:
+        """Returns the EM Invoker instance.
+
+        Returns:
+            BaseEMInvoker: The EM Invoker instance.
+        """
+    async def ensure_index(self, mapping: dict[str, Any] | None = None, index_settings: dict[str, Any] | None = None, dimension: int | None = None, distance_strategy: str | None = None) -> None:
+        '''Ensure Elasticsearch index exists, creating it if necessary.
+
+        This method is idempotent - if the index already exists, it will skip creation
+        and return early.
+
+        Args:
+            mapping (dict[str, Any] | None, optional): Custom mapping dictionary to use
+                for index creation. If provided, this mapping will be used directly.
+                The mapping should follow Elasticsearch mapping format. Defaults to None,
+                in which default mapping will be used.
+            index_settings (dict[str, Any] | None, optional): Custom index settings.
+                These settings will be merged with any default settings. Defaults to None.
+            dimension (int | None, optional): Vector dimension. If not provided and mapping
+                is not provided, will be inferred from em_invoker by generating a test embedding.
+            distance_strategy (str | None, optional): Distance strategy for vector similarity.
+                Supported values: "cosine", "l2_norm", "dot_product", etc.
+                Only used when building default mapping. Defaults to "cosine" if not specified.
+
+        Raises:
+            ValueError: If mapping is invalid or required parameters are missing.
+            RuntimeError: If index creation fails.
+        '''
+    async def create(self, data: Chunk | list[Chunk], **kwargs: Any) -> None:
+        """Create new records in the datastore.
+
+        Args:
+            data (Chunk | list[Chunk]): Data to create (single item or collection).
+            **kwargs: Datastore-specific parameters.
+
+        Raises:
+            ValueError: If data structure is invalid.
+        """
+    async def create_from_vector(self, chunk_vectors: list[tuple[Chunk, Vector]], **kwargs) -> list[str]:
+        """Add pre-computed embeddings directly.
+
+        Args:
+            chunk_vectors (list[tuple[Chunk, Vector]]): List of tuples containing chunks and their
+                corresponding vectors.
+            **kwargs: Datastore-specific parameters.
+
+        Returns:
+            list[str]: List of IDs assigned to added embeddings.
+        """
+    async def retrieve(self, query: str, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None, **kwargs: Any) -> list[Chunk]:
+        '''Semantic search using text query converted to vector.
+
+        Usage Example:
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            await vector_capability.retrieve(
+                query="What is the capital of France?",
+                filters=F.eq("metadata.category", "tech"),
+                options=QueryOptions(limit=10),
+            )
+
+            # Multiple filters
+            filters = F.and_(F.eq("metadata.source", "wikipedia"), F.eq("metadata.category", "tech"))
+            await vector_capability.retrieve(query="What is the capital of France?", filters=filters)
+            ```
+
+        Args:
+            query (str): Text query to embed and search for.
+            filters (FilterClause | QueryFilter | None, optional): Filters to apply to the search.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Options to apply to the search. Defaults to None.
+            **kwargs: Datastore-specific parameters.
+
+        Returns:
+            list[Chunk]: List of chunks ordered by relevance score.
+        '''
+    async def retrieve_by_vector(self, vector: Vector, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        '''Direct vector similarity search.
+
+        Usage Example:
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            await vector_capability.retrieve_by_vector(
+                vector=[0.1, 0.2, 0.3],
+                filters=F.eq("metadata.category", "tech"),
+                options=QueryOptions(limit=10),
+            )
+
+            # Multiple filters
+            filters = F.and_(F.eq("metadata.source", "wikipedia"), F.eq("metadata.category", "tech"))
+            await vector_capability.retrieve_by_vector(vector=[0.1, 0.2, 0.3], filters=filters)
+            ```
+
+        Args:
+            vector (Vector): Query embedding vector.
+            filters (FilterClause | QueryFilter | None, optional): Filters to apply to the search.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Options to apply to the search. Defaults to None.
+
+        Returns:
+            list[Chunk]: List of chunks ordered by similarity score.
+        '''
+    async def update(self, update_values: dict, filters: FilterClause | QueryFilter | None = None, **kwargs: Any) -> None:
+        """Update existing records in the datastore.
+
+        Args:
+            update_values (dict): Values to update.
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            **kwargs: Datastore-specific parameters.
+        """
+    async def delete(self, filters: FilterClause | QueryFilter | None = None, **kwargs: Any) -> None:
+        """Delete records from the data store based on filters.
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records for deletion.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            **kwargs: Datastore-specific parameters.
+        """
+    async def delete_by_id(self, id: str | list[str], **kwargs: Any) -> None:
+        """Delete records from the data store based on IDs.
+
+        Args:
+            id (str | list[str]): ID or list of IDs to delete.
+            **kwargs: Datastore-specific parameters.
+        """
+    async def clear(self, **kwargs: Any) -> None:
+        """Clear all records from the datastore.
+
+        Args:
+            **kwargs: Datastore-specific parameters.
+        """

gllm_datastore/data_store/exceptions.pyi

@@ -0,0 +1,35 @@
+from _typeshed import Incomplete
+
+class NotSupportedException(Exception):
+    """Raised when attempting to access an unsupported capability.
+
+    This exception is raised when code attempts to access a capability
+    that isn't configured for a datastore.
+    """
+    capability: Incomplete
+    class_name: Incomplete
+    class_obj: Incomplete
+    def __init__(self, capability: str, class_obj: type) -> None:
+        """Initialize the exception.
+
+        Args:
+            capability (str): The name of the unsupported capability.
+            class_obj (Type): The class object for context.
+        """
+
+class NotRegisteredException(Exception):
+    """Raised when attempting to access a capability that is not registered.
+
+    This exception is raised when code attempts to access a capability
+    that is not registered for a datastore but is supported by the datastore.
+    """
+    capability: Incomplete
+    class_name: Incomplete
+    class_obj: Incomplete
+    def __init__(self, capability: str, class_obj: type) -> None:
+        """Initialize the exception.
+
+        Args:
+            capability (str): The name of the unregistered capability.
+            class_obj (Type): The class object for context.
+        """

gllm_datastore/data_store/in_memory/__init__.pyi

@@ -0,0 +1,5 @@
+from gllm_datastore.data_store.in_memory.data_store import InMemoryDataStore as InMemoryDataStore
+from gllm_datastore.data_store.in_memory.fulltext import InMemoryFulltextCapability as InMemoryFulltextCapability
+from gllm_datastore.data_store.in_memory.vector import InMemoryVectorCapability as InMemoryVectorCapability
+
+__all__ = ['InMemoryDataStore', 'InMemoryFulltextCapability', 'InMemoryVectorCapability']

gllm_datastore/data_store/in_memory/data_store.pyi

@@ -0,0 +1,71 @@
+from gllm_core.schema import Chunk as Chunk
+from gllm_datastore.core.filters.schema import FilterClause as FilterClause, QueryFilter as QueryFilter
+from gllm_datastore.data_store.base import BaseDataStore as BaseDataStore, CapabilityType as CapabilityType
+from gllm_datastore.data_store.in_memory.fulltext import InMemoryFulltextCapability as InMemoryFulltextCapability
+from gllm_datastore.data_store.in_memory.vector import InMemoryVectorCapability as InMemoryVectorCapability
+
+class InMemoryDataStore(BaseDataStore):
+    """In-memory data store with multiple capability support.
+
+    This class provides a unified interface for accessing vector, fulltext,
+    and cache capabilities using in-memory storage optimized for development
+    and testing scenarios.
+
+    Attributes:
+        store (dict[str, Chunk]): Dictionary storing data with their IDs as keys.
+    """
+    store: dict[str, Chunk]
+    def __init__(self) -> None:
+        """Initialize the in-memory data store."""
+    @property
+    def supported_capabilities(self) -> list[CapabilityType]:
+        """Return list of currently supported capabilities.
+
+        Returns:
+            list[str]: List of capability names that are supported.
+        """
+    @property
+    def fulltext(self) -> InMemoryFulltextCapability:
+        """Access fulltext capability if registered.
+
+        This method solely uses the logic of its parent class to return the fulltext capability handler.
+        This method overrides the parent class to return the InMemoryFulltextCapability handler for better
+        type hinting.
+
+        Returns:
+            InMemoryFulltextCapability: Fulltext capability handler.
+
+        Raises:
+            NotRegisteredException: If fulltext capability is not registered.
+        """
+    @property
+    def vector(self) -> InMemoryVectorCapability:
+        """Access vector capability if registered.
+
+        This method solely uses the logic of its parent class to return the vector capability handler.
+        This method overrides the parent class to return the InMemoryVectorCapability handler for better
+        type hinting.
+
+        Returns:
+            InMemoryVectorCapability: Vector capability handler.
+
+        Raises:
+            NotRegisteredException: If vector capability is not registered.
+        """
+    @classmethod
+    def translate_query_filter(cls, query_filter: FilterClause | QueryFilter | None) -> FilterClause | QueryFilter | None:
+        """Translate QueryFilter or FilterClause to in-memory datastore filter syntax.
+
+        For the in-memory datastore, this method acts as an identity function since
+        the datastore works directly with the QueryFilter DSL without requiring
+        translation to a native format.
+
+        Args:
+            query_filter (FilterClause | QueryFilter | None): The filter to translate.
+                Can be a single FilterClause, a QueryFilter with multiple clauses,
+                or None for empty filters.
+
+        Returns:
+            FilterClause | QueryFilter | None: The same filter object that was passed in.
+                Returns None for empty filters.
+        """

gllm_datastore/data_store/in_memory/fulltext.pyi

@@ -0,0 +1,131 @@
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.core.filters import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_datastore.data_store.in_memory.query import create_updated_chunk as create_updated_chunk, delete_chunks_by_filters as delete_chunks_by_filters, get_chunks_from_store as get_chunks_from_store
+from typing import Any
+
+class InMemoryFulltextCapability:
+    """In-memory implementation of FulltextCapability protocol.
+
+    This class provides document CRUD operations and flexible querying using pure
+    Python data structures optimized for development and testing.
+
+    Attributes:
+        store (dict[str, Chunk]): Dictionary storing Chunk objects with their IDs as keys.
+    """
+    store: dict[str, Chunk]
+    def __init__(self, store: dict[str, Any] | None = None) -> None:
+        """Initialize the in-memory fulltext capability.
+
+        Args:
+            store (dict[str, Any] | None, optional): Dictionary storing Chunk objects with their IDs as keys.
+                Defaults to None.
+        """
+    async def create(self, data: Chunk | list[Chunk]) -> None:
+        '''Create new records in the datastore.
+
+        Examples:
+            Create a new chunk.
+            ```python
+            await fulltext_capability.create(Chunk(content="Test chunk", metadata={"category": "test"}))
+            ```
+
+        Args:
+            data (Chunk | list[Chunk]): Data to create (single item or collection).
+
+        Raises:
+            ValueError: If data structure is invalid.
+        '''
+    async def retrieve(self, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        '''Read records from the datastore with optional filtering.
+
+        Usage Example:
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            results = await fulltext_capability.retrieve(filters=F.eq("metadata.category", "tech"))
+
+            # Multiple filters
+            results = await fulltext_capability.retrieve(
+                filters=F.and_(F.eq("metadata.category", "tech"), F.eq("metadata.status", "active"))
+            )
+            ```
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options for sorting and pagination. Defaults to None.
+
+        Returns:
+            list[Chunk]: List of matched chunks after applying filters and options.
+        '''
+    async def retrieve_fuzzy(self, query: str, max_distance: int = 2, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        """Find records that fuzzy match the query within distance threshold.
+
+        Args:
+            query (str): Text to fuzzy match against.
+            max_distance (int, optional): Maximum edit distance for matches. Defaults to 2.
+            filters (FilterClause | QueryFilter | None, optional): Optional metadata filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options, only limit is used here. Defaults to None.
+
+        Returns:
+            list[Chunk]: Matched chunks ordered by distance (ascending), limited by options.limit.
+        """
+    async def update(self, update_values: dict[str, Any], filters: FilterClause | QueryFilter | None = None) -> None:
+        '''Update existing records in the datastore.
+
+        Examples:
+            Update certain metadata of a chunk with specific filters.
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            await fulltext_capability.update(
+                update_values={"metadata": {"status": "published"}},
+                filters=F.eq("metadata.category", "tech"),
+            )
+
+            # Multiple filters
+            await fulltext_capability.update(
+                update_values={"metadata": {"status": "published"}},
+                filters=F.and_(F.eq("metadata.status", "draft"), F.eq("metadata.category", "tech")),
+            )
+            ```
+
+        Args:
+            update_values (dict[str, Any]): Mapping of fields to new values to apply.
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+        '''
+    async def delete(self, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> None:
+        '''Delete records from the datastore.
+
+        Usage Example:
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            await fulltext_capability.delete(filters=F.eq("metadata.category", "tech"))
+
+            # Multiple filters
+            await fulltext_capability.delete(
+                filters=F.and_(F.eq("metadata.category", "tech"), F.eq("metadata.status", "draft"))
+            )
+            ```
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to delete.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options for sorting and limiting deletions
+                (for eviction-like operations). Defaults to None.
+
+        Returns:
+            None: This method performs deletions in-place.
+        '''
+    async def clear(self) -> None:
+        """Clear all records from the datastore."""