gllm-datastore-binary 0.5.50__cp312-cp312-macosx_13_0_arm64.whl

gllm_datastore/data_store/in_memory/query.pyi

@@ -0,0 +1,175 @@
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.constants import CHUNK_KEYS as CHUNK_KEYS
+from gllm_datastore.core.filters import FilterClause as FilterClause, FilterCondition as FilterCondition, FilterOperator as FilterOperator, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_inference.schema import Vector
+from typing import Any
+
+def apply_filters(chunks: list[Chunk], filters: FilterClause | QueryFilter) -> list[Chunk]:
+    '''Apply filters to chunks.
+
+    Usage Example:
+        ```python
+        from gllm_datastore.core.filters import filter as F
+
+        chunks = [
+            Chunk(id="1", content="Chunk 1", metadata={"category": "test"}),
+            Chunk(id="2", content="Chunk 2", metadata={"category": "test"}),
+            Chunk(id="3", content="Chunk 3", metadata={"category": "test"}),
+        ]
+        # Direct FilterClause usage
+        filters = F.eq("metadata.category", "test")
+        filtered_chunks = apply_filters(chunks, filters)
+
+        # Multiple filters
+        filters = F.and_(F.eq("metadata.category", "test"), F.eq("metadata.status", "active"))
+        filtered_chunks = apply_filters(chunks, filters)
+        ```
+
+    Args:
+        chunks (list[Chunk]): List of chunks to filter.
+        filters (FilterClause | QueryFilter): Filter criteria to apply.
+            FilterClause objects are automatically converted to QueryFilter internally.
+
+    Returns:
+        list[Chunk]: Filtered list of chunks.
+    '''
+def apply_options(chunks: list[Chunk], options: QueryOptions) -> list[Chunk]:
+    """Apply query options (sorting, pagination).
+
+    Note: columns filtering is not applicable to Chunk objects since they have a fixed structure
+    and we can only filter on id, content, score, and metadata.
+
+    Args:
+        chunks (list[Chunk]): List of chunks to process.
+        options (QueryOptions): Query options to apply.
+
+    Returns:
+        list[Chunk]: Processed list of chunks.
+    """
+def get_nested_value(obj: dict[str, Any], key_path: str) -> Any:
+    '''Get a nested value from a dictionary using dot notation.
+
+    Args:
+        obj (dict[str, Any]): Dictionary to traverse.
+        key_path (str): Dot-separated path to the value (e.g., "user.profile.name").
+
+    Returns:
+        Any: The value at the specified path, or None if not found.
+    '''
+def get_sort_value(chunk: Chunk, order_by: str) -> Any:
+    """Get the value to sort by.
+
+    Args:
+        chunk (Chunk): Chunk to get the value from.
+        order_by (str): The field to sort by.
+
+    Returns:
+        Any: The value to sort by.
+    """
+def validate_cache_key(key: str) -> None:
+    """Validate cache key format and content.
+
+    Args:
+        key (str): Cache key to validate.
+
+    Raises:
+        TypeError: If key is not a string.
+        ValueError: If key is empty or whitespace-only.
+    """
+def get_chunks_from_store(store: dict[str, Chunk], filters: QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+    """Get chunks from a store as a list with optional filters and options.
+
+    Args:
+        store (dict[str, Chunk]): Store containing chunks.
+        filters (QueryFilter | None, optional): Filter criteria to apply. Defaults to None.
+        options (QueryOptions | None, optional): Query options to apply. Defaults to None.
+
+    Returns:
+        list[Chunk]: List of all chunks in the store.
+    """
+def apply_filters_and_options(chunks: list[Chunk], filters: QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+    """Apply filters and options to a list of chunks.
+
+    Args:
+        chunks (list[Chunk]): List of chunks to process.
+        filters (QueryFilter | None, optional): Filter criteria to apply. Defaults to None.
+        options (QueryOptions | None, optional): Query options to apply. Defaults to None.
+
+    Returns:
+        list[Chunk]: Processed list of chunks.
+    """
+def create_updated_chunk(existing_chunk: Chunk, update_values: dict[str, Any]) -> Chunk:
+    """Create an updated chunk with new values.
+
+    Args:
+        existing_chunk (Chunk): The existing chunk to update.
+        update_values (dict[str, Any]): Values to update.
+
+    Returns:
+        Chunk: Updated chunk with new values.
+    """
+def delete_chunks_by_filters(store: dict[str, Chunk], filters: QueryFilter | None = None) -> int:
+    """Delete chunks from store based on filters.
+
+    Args:
+        store (dict[str, Chunk]): Store containing chunks.
+        filters (QueryFilter | None, optional): Filters to select chunks to delete. Defaults to None.
+
+    Returns:
+        int: Number of chunks deleted.
+    """
+def find_matching_chunk_ids(store: dict[str, Chunk], filters: QueryFilter) -> list[str]:
+    """Find chunk IDs that match the given filters.
+
+    Args:
+        store (dict[str, Chunk]): Store containing chunks.
+        filters (QueryFilter): The filters to apply.
+
+    Returns:
+        list[str]: List of chunk IDs that match the filters.
+    """
+def similarity_search(query_vector: Vector, store: dict[str, Chunk], filters: QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+    """Retrieve chunks by vector similarity from a store.
+
+    This method will only return chunks that have a vector in their metadata.
+    It will also apply the filters and options to the chunks.
+
+    Args:
+        query_vector (Vector): Query embedding vector.
+        store (dict[str, Chunk]): Store containing chunks.
+        filters (QueryFilter | None): Query filters to apply.
+        options (QueryOptions | None, optional): Query options to apply.
+
+    Returns:
+        list[Chunk]: List of chunks ordered by similarity score.
+    """
+def evaluate_filter(chunk: Chunk, filters: QueryFilter) -> bool:
+    '''Evaluate if a chunk matches the given filters.
+
+    Examples:
+        ```python
+        from gllm_datastore.core.filters import filter as F
+
+        # Simple filter
+        filters = F.and_(F.eq("metadata.category", "tech"))
+        result = evaluate_filter(chunk, filters)
+
+        # Complex nested filter
+        filters = F.and_(
+            F.gte("metadata.price", 10),
+            F.lte("metadata.price", 100),
+            F.or_(
+                F.eq("metadata.status", "active"),
+                F.eq("metadata.status", "pending")
+            )
+        )
+        result = evaluate_filter(chunk, filters)
+        ```
+
+    Args:
+        chunk (Chunk): The chunk to evaluate.
+        filters (QueryFilter): The filters to apply.
+
+    Returns:
+        bool: True if the chunk matches all filters, False otherwise.
+    '''

gllm_datastore/data_store/in_memory/vector.pyi

@@ -0,0 +1,174 @@
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.constants import CHUNK_KEYS as CHUNK_KEYS
+from gllm_datastore.core.filters import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_datastore.data_store.in_memory.query import create_updated_chunk as create_updated_chunk, delete_chunks_by_filters as delete_chunks_by_filters, get_chunks_from_store as get_chunks_from_store, similarity_search as similarity_search
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from gllm_inference.schema import Vector
+from typing import Any
+
+class InMemoryVectorCapability:
+    """In-memory implementation of VectorCapability protocol.
+
+    This class provides vector similarity search operations using pure Python
+    data structures optimized for development and testing.
+
+    Attributes:
+        store (dict[str, Chunk]): Dictionary storing Chunk objects with their IDs as keys.
+        em_invoker (BaseEMInvoker): The embedding model to perform vectorization.
+    """
+    store: dict[str, Chunk]
+    def __init__(self, em_invoker: BaseEMInvoker, store: dict[str, Any] | None = None) -> None:
+        """Initialize the in-memory vector capability.
+
+        Args:
+            em_invoker (BaseEMInvoker): em_invoker model for text-to-vector conversion.
+            store (dict[str, Any] | None, optional): Dictionary storing Chunk objects with their IDs as keys.
+                Defaults to None.
+        """
+    @property
+    def em_invoker(self) -> BaseEMInvoker:
+        """Returns the EM Invoker instance.
+
+        Returns:
+            BaseEMInvoker: The EM Invoker instance.
+        """
+    async def ensure_index(self) -> None:
+        """Ensure in-memory vector store exists, initializing it if necessary.
+
+        This method is idempotent - if the store already exists, it will skip
+        initialization and return early.
+        """
+    async def create(self, data: Chunk | list[Chunk]) -> None:
+        """Add chunks to the vector store with automatic embedding generation.
+
+        Args:
+            data (Chunk | list[Chunk]): Single chunk or list of chunks to add.
+        """
+    async def create_from_vector(self, chunk_vectors: list[tuple[Chunk, Vector]]) -> None:
+        """Add pre-computed vectors directly.
+
+        Args:
+            chunk_vectors (list[tuple[Chunk, Vector]]): List of tuples containing chunks and their
+                corresponding vectors.
+        """
+    async def retrieve(self, query: str, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        '''Read records from the datastore using text-based similarity search with optional filtering.
+
+        Usage Example:
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            await vector_capability.retrieve(
+                query="What is the capital of France?",
+                filters=F.eq("metadata.category", "tech"),
+                options=QueryOptions(limit=2),
+            )
+
+            # Multiple filters
+            filters = F.and_(F.eq("metadata.source", "wikipedia"), F.eq("metadata.category", "tech"))
+            await vector_capability.retrieve(
+                query="What is the capital of France?",
+                filters=filters,
+                options=QueryOptions(limit=2),
+            )
+            ```
+            This will retrieve the top 2 chunks by similarity score from the vector store
+            that match the query and the filters. The chunks will be sorted by score in descending order.
+
+        Args:
+            query (str): Input text to embed and search with.
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options like limit and sorting.
+                Defaults to None, in which case, no sorting is applied and top 10 chunks are returned.
+
+        Returns:
+            list[Chunk]: Top ranked chunks by similarity score.
+        '''
+    async def retrieve_by_vector(self, vector: Vector, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        """Direct vector similarity search.
+
+        Args:
+            vector (Vector): Query embedding vector.
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options like limit and sorting.
+                Defaults to None, in which case, no sorting is applied and top 10 chunks are returned.
+
+        Returns:
+            list[Chunk]: List of chunks ordered by similarity score.
+        """
+    async def update(self, update_values: dict[str, Any], filters: FilterClause | QueryFilter | None = None, **kwargs: Any) -> None:
+        '''Update existing records in the datastore.
+
+        Examples:
+            1. Update certain metadata of a chunk with specific filters.
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            await vector_capability.update(
+                update_values={"metadata": {"status": "published"}},
+                filters=F.eq("metadata.category", "tech"),
+            )
+
+            # Multiple filters
+            await vector_capability.update(
+                update_values={"metadata": {"status": "published"}},
+                filters=F.and_(F.eq("metadata.status", "draft"), F.eq("metadata.category", "tech")),
+            )
+            ```
+
+            2. Update certain content of a chunk with specific id.
+            This will also regenerate the vector of the chunk.
+            ```python
+            # Direct FilterClause usage
+            await vector_capability.update(
+                update_values={"content": "new_content"},
+                filters=F.eq("id", "unique_id"),
+            )
+
+            # Multiple filters
+            await vector_capability.update(
+                update_values={"content": "new_content"},
+                filters=F.and_(F.eq("id", "unique_id"), F.eq("metadata.category", "tech")),
+            )
+            ```
+
+        Args:
+            update_values (dict[str, Any]): Values to update.
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None, in which case no operation is performed (no-op).
+            **kwargs: Datastore-specific parameters.
+
+        Raises:
+            ValueError: If content is empty.
+        '''
+    async def delete(self, filters: FilterClause | QueryFilter | None = None) -> None:
+        '''Delete records from the datastore.
+
+        Usage Example:
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Direct FilterClause usage
+            await vector_capability.delete(filters=F.eq("metadata.category", "AI"))
+
+            # Multiple filters
+            await vector_capability.delete(
+                filters=F.and_(F.eq("metadata.category", "AI"), F.eq("metadata.status", "published")),
+            )
+            ```
+            This will delete all chunks from the vector store that match the filters.
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to delete.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None, in which case no operation is performed (no-op).
+        '''
+    async def clear(self) -> None:
+        """Clear all vectors from the store."""

gllm_datastore/data_store/opensearch/__init__.pyi

@@ -0,0 +1,5 @@
+from gllm_datastore.data_store.opensearch.data_store import OpenSearchDataStore as OpenSearchDataStore
+from gllm_datastore.data_store.opensearch.fulltext import OpenSearchFulltextCapability as OpenSearchFulltextCapability
+from gllm_datastore.data_store.opensearch.vector import OpenSearchVectorCapability as OpenSearchVectorCapability
+
+__all__ = ['OpenSearchDataStore', 'OpenSearchFulltextCapability', 'OpenSearchVectorCapability']

gllm_datastore/data_store/opensearch/data_store.pyi

@@ -0,0 +1,160 @@
+from _typeshed import Incomplete
+from gllm_datastore.constants import DEFAULT_REQUEST_TIMEOUT as DEFAULT_REQUEST_TIMEOUT
+from gllm_datastore.core.filters.schema import FilterClause as FilterClause, QueryFilter as QueryFilter
+from gllm_datastore.data_store._elastic_core.client_factory import EngineType as EngineType, create_client as create_client
+from gllm_datastore.data_store.base import BaseDataStore as BaseDataStore, CapabilityType as CapabilityType
+from gllm_datastore.data_store.opensearch.fulltext import OpenSearchFulltextCapability as OpenSearchFulltextCapability
+from gllm_datastore.data_store.opensearch.query_translator import OpenSearchQueryTranslator as OpenSearchQueryTranslator
+from gllm_datastore.data_store.opensearch.vector import OpenSearchVectorCapability as OpenSearchVectorCapability
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from opensearchpy import AsyncOpenSearch
+from typing import Any
+
+class OpenSearchDataStore(BaseDataStore):
+    '''OpenSearch data store with multiple capability support.
+
+    This is the explicit public API for OpenSearch. Users know they\'re
+    using OpenSearch, not a generic "elastic-like" datastore.
+
+    Attributes:
+        engine (str): Always "opensearch" for explicit identification.
+            This attribute ensures users know they\'re using OpenSearch, not a generic
+            "elastic-like" datastore.
+        index_name (str): The name of the OpenSearch index.
+        client (AsyncOpenSearch): AsyncOpenSearch client.
+    '''
+    engine: str
+    client: Incomplete
+    index_name: Incomplete
+    def __init__(self, index_name: str, client: AsyncOpenSearch | None = None, url: str | None = None, cloud_id: str | None = None, api_key: str | None = None, username: str | None = None, password: str | None = None, request_timeout: int = ..., connection_params: dict[str, Any] | None = None) -> None:
+        '''Initialize the OpenSearch data store.
+
+        Args:
+            index_name (str): The name of the OpenSearch index to use for operations.
+                This index name will be used for all queries and operations.
+            client (AsyncOpenSearch | None, optional): Pre-configured OpenSearch client instance.
+                If provided, it will be used instead of creating a new client from url/cloud_id.
+                Must be an instance of AsyncOpenSearch. Defaults to None.
+            url (str | None, optional): The URL of the OpenSearch server.
+                For example, "http://localhost:9200". Either url or cloud_id must be provided
+                if client is None. Defaults to None.
+            cloud_id (str | None, optional): The cloud ID of the OpenSearch cluster.
+                Used for OpenSearch Service connections. Either url or cloud_id must be provided
+                if client is None. Defaults to None.
+            api_key (str | None, optional): The API key for authentication.
+                If provided, will be used for authentication. Mutually exclusive with username/password.
+                Defaults to None.
+            username (str | None, optional): The username for basic authentication.
+                Must be provided together with password. Mutually exclusive with api_key.
+                Defaults to None.
+            password (str | None, optional): The password for basic authentication.
+                Must be provided together with username. Mutually exclusive with api_key.
+                Defaults to None.
+            request_timeout (int, optional): The request timeout in seconds.
+                Defaults to DEFAULT_REQUEST_TIMEOUT.
+            connection_params (dict[str, Any] | None, optional): Additional connection parameters
+                for OpenSearch client. These will be merged with automatically detected parameters
+                (authentication, SSL settings). User-provided params take precedence. Defaults to None.
+                Available parameters include:
+                1. http_auth (tuple[str, str] | None): HTTP authentication tuple (username, password).
+                2. use_ssl (bool): Whether to use SSL/TLS. Defaults to True for HTTPS URLs.
+                3. verify_certs (bool): Whether to verify SSL certificates. Defaults to True for HTTPS URLs.
+                    Set to False to use self-signed certificates (not recommended for production).
+                4. ssl_show_warn (bool): Whether to show SSL warnings. Defaults to True for HTTPS URLs.
+                5. ssl_assert_hostname (str | None): SSL hostname assertion. Defaults to None.
+                6. max_retries (int): Maximum number of retries for requests. Defaults to 3.
+                7. retry_on_timeout (bool): Whether to retry on timeouts. Defaults to True.
+                8. client_cert (str | None): Path to the client certificate file. Defaults to None.
+                9. client_key (str | None): Path to the client private key file. Defaults to None.
+                10. root_cert (str | None): Path to the root certificate file. Defaults to None.
+                11. Additional kwargs: Any other parameters accepted by OpenSearch client constructor.
+
+        Raises:
+            ValueError: If neither url nor cloud_id is provided when client is None.
+            TypeError: If client is provided but is not an instance of AsyncOpenSearch.
+        '''
+    @property
+    def supported_capabilities(self) -> list[str]:
+        """Return list of currently supported capabilities.
+
+        Returns:
+            list[str]: List of capability names that are supported.
+                Currently returns [CapabilityType.FULLTEXT, CapabilityType.VECTOR].
+        """
+    @property
+    def fulltext(self) -> OpenSearchFulltextCapability:
+        """Access fulltext capability if supported.
+
+        This method uses the logic of its parent class to return the fulltext capability handler.
+        This method overrides the parent class to return the OpenSearchFulltextCapability handler for better
+        type hinting.
+
+        Returns:
+            OpenSearchFulltextCapability: Fulltext capability handler.
+
+        Raises:
+            NotSupportedException: If fulltext capability is not supported.
+        """
+    @property
+    def vector(self) -> OpenSearchVectorCapability:
+        """Access vector capability if supported.
+
+        This method uses the logic of its parent class to return the vector capability handler.
+        This method overrides the parent class to return the OpenSearchVectorCapability handler for better
+        type hinting.
+
+        Returns:
+            OpenSearchVectorCapability: Vector capability handler.
+
+        Raises:
+            NotSupportedException: If vector capability is not supported.
+        """
+    def with_fulltext(self, index_name: str | None = None, query_field: str = 'text') -> OpenSearchDataStore:
+        '''Configure fulltext capability and return datastore instance.
+
+        Overrides parent for better type hinting.
+
+        Args:
+            index_name (str | None, optional): Index name for fulltext operations.
+                Uses datastore\'s default if None. Defaults to None.
+            query_field (str, optional): Field name for text queries. Defaults to "text".
+
+        Returns:
+            OpenSearchDataStore: Self for method chaining.
+        '''
+    def with_vector(self, em_invoker: BaseEMInvoker, index_name: str | None = None, query_field: str = 'text', vector_query_field: str = 'vector', retrieval_strategy: Any = None, distance_strategy: str | None = None) -> OpenSearchDataStore:
+        '''Configure vector capability and return datastore instance.
+
+        Overrides parent for better type hinting.
+
+        Args:
+            em_invoker (BaseEMInvoker): Embedding model for vectorization.
+            index_name (str | None, optional): Index name. Uses datastore\'s default if None.
+            query_field (str, optional): Field name for text queries. Defaults to "text".
+            vector_query_field (str, optional): Field name for vector queries. Defaults to "vector".
+            retrieval_strategy: Not used (kept for API compatibility). Defaults to None.
+            distance_strategy (str | None, optional): Distance strategy (e.g., "l2", "cosine"). Defaults to None.
+
+        Returns:
+            OpenSearchDataStore: Self for method chaining.
+
+        Note:
+            Connection parameters are configured at the data store level during initialization.
+            See OpenSearchDataStore.__init__ for connection_params details.
+        '''
+    @classmethod
+    def translate_query_filter(cls, query_filter: FilterClause | QueryFilter | None) -> dict[str, Any] | None:
+        """Translate QueryFilter or FilterClause to OpenSearch native filter syntax.
+
+        This method delegates to the OpenSearchQueryTranslator and returns the result as a dictionary.
+
+        Args:
+            query_filter (FilterClause | QueryFilter | None): The filter to translate.
+                Can be a single FilterClause, a QueryFilter with multiple clauses and logical conditions,
+                or None for empty filters. FilterClause objects are automatically converted to QueryFilter.
+
+        Returns:
+            dict[str, Any] | None: The translated filter as an OpenSearch DSL dictionary.
+                Returns None for empty filters or when query_filter is None.
+                The dictionary format matches OpenSearch Query DSL syntax.
+        """