gllm-datastore-binary 0.5.50__cp312-cp312-macosx_13_0_arm64.whl

gllm_datastore/data_store/redis/data_store.pyi

@@ -0,0 +1,153 @@
+from _typeshed import Incomplete
+from gllm_datastore.core.filters.schema import FilterClause as FilterClause, QueryFilter as QueryFilter
+from gllm_datastore.data_store.base import BaseDataStore as BaseDataStore, CapabilityType as CapabilityType
+from gllm_datastore.data_store.redis.fulltext import RedisFulltextCapability as RedisFulltextCapability
+from gllm_datastore.data_store.redis.query import get_filterable_fields_from_index as get_filterable_fields_from_index
+from gllm_datastore.data_store.redis.query_translator import RedisQueryTranslator as RedisQueryTranslator
+from gllm_datastore.data_store.redis.vector import RedisVectorCapability as RedisVectorCapability
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from redis.asyncio.client import Redis
+
+class RedisDataStore(BaseDataStore):
+    """Redis data store with fulltext capability support.
+
+    Attributes:
+        index_name (str): Name for the Redis index.
+        client (Redis): Redis client instance.
+    """
+    client: Incomplete
+    index_name: Incomplete
+    def __init__(self, index_name: str, url: str | None = None, client: Redis | None = None) -> None:
+        """Initialize the Redis data store.
+
+        Args:
+            index_name (str): Name of the Redis index to use.
+            url (str | None, optional): URL for Redis connection. Defaults to None.
+                Format: redis://[[username]:[password]]@host:port/database
+            client (Redis | None, optional): Redis client instance to use. Defaults to None.
+                in which case the url parameter will be used to create a new Redis client.
+
+        Raises:
+            ValueError: If neither `url` nor `client` is provided, or if URL is invalid.
+            TypeError: If `client` is not a Redis instance.
+            ConnectionError: If Redis connection fails.
+        """
+    @property
+    def supported_capabilities(self) -> list[CapabilityType]:
+        """Return list of currently supported capabilities.
+
+        Returns:
+            list[CapabilityType]: List of capability names that are supported.
+        """
+    @property
+    def fulltext(self) -> RedisFulltextCapability:
+        """Access fulltext capability if registered.
+
+        This method uses the logic of its parent class to return the fulltext capability handler.
+        This method overrides the parent class to return the RedisFulltextCapability handler for better
+        type hinting.
+
+        Returns:
+            RedisFulltextCapability: Fulltext capability handler.
+
+        Raises:
+            NotRegisteredException: If fulltext capability is not registered.
+        """
+    @property
+    def vector(self) -> RedisVectorCapability:
+        """Access vector capability if registered.
+
+        This method uses the logic of its parent class to return the vector capability handler.
+        This method overrides the parent class to return the RedisVectorCapability handler for better
+        type hinting.
+
+        Returns:
+            RedisVectorCapability: Vector capability handler.
+
+        Raises:
+            NotRegisteredException: If vector capability is not registered.
+        """
+    def with_fulltext(self, index_name: str | None = None) -> RedisDataStore:
+        """Configure fulltext capability and return datastore instance.
+
+        Schema will be automatically inferred from chunks when creating a new index,
+        or auto-detected from an existing index when performing operations.
+
+        Args:
+            index_name (str | None, optional): The name of the Redis index. Defaults to None,
+                in which case the default class attribute will be utilized.
+
+        Returns:
+            RedisDataStore: RedisDataStore instance for method chaining.
+        """
+    def with_vector(self, em_invoker: BaseEMInvoker, index_name: str | None = None) -> RedisDataStore:
+        """Configure vector capability and return datastore instance.
+
+        Schema will be automatically inferred from chunks when creating a new index,
+        or auto-detected from an existing index when performing operations.
+
+        Args:
+            em_invoker (BaseEMInvoker): The embedding model to perform vectorization.
+            index_name (str | None, optional): The name of the Redis index. Defaults to None,
+                in which case the default class attribute will be utilized.
+
+        Returns:
+            RedisDataStore: RedisDataStore instance for method chaining.
+
+        Raises:
+            ValueError: If em_invoker is not provided.
+        """
+    def translate_query_filter(self, query_filter: FilterClause | QueryFilter) -> str:
+        '''Translate QueryFilter or FilterClause to Redis native filter syntax.
+
+        This method delegates to the existing RedisQueryTranslator in the
+        redis.query_translator module and returns the result as a string.
+        It uses the instance\'s index_name and client to detect field types
+        for accurate Redis Search query syntax.
+
+        Examples:
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            # Create datastore instance
+            datastore = RedisDataStore(index_name="my_index", url="redis://localhost:6379")
+
+            # Single FilterClause (field types detected from index schema)
+            clause = F.eq("metadata.status", "active")
+            result = datastore.translate_query_filter(clause)
+            # Returns: "@metadata_status:{active}" if status is a TAG field
+            # Returns: "@metadata_status:active" if status is a TEXT field
+
+            # QueryFilter with multiple clauses (AND condition)
+            filter_obj = F.and_(
+                F.eq("metadata.status", "active"),
+                F.gt("metadata.age", 25),
+            )
+            result = datastore.translate_query_filter(filter_obj)
+            # Returns: "@metadata_status:{active} @metadata_age:[(25 +inf]"
+
+            # QueryFilter with OR condition
+            filter_obj = F.or_(
+                F.eq("metadata.status", "active"),
+                F.eq("metadata.status", "pending"),
+            )
+            result = datastore.translate_query_filter(filter_obj)
+            # Returns: "@metadata_status:{active} | @metadata_status:{pending}"
+
+            # IN operator (produces parentheses syntax)
+            filter_obj = F.in_("metadata.category", ["tech", "science"])
+            result = datastore.translate_query_filter(filter_obj)
+            # Returns: "@metadata_category:(tech|science)"
+
+            # Empty filter returns None
+            result = datastore.translate_query_filter(None)
+            # Returns: None
+            ```
+
+        Args:
+            query_filter (FilterClause | QueryFilter): The filter to translate.
+                Can be a single FilterClause or a QueryFilter with multiple clauses.
+
+        Returns:
+            str: The translated filter as a Redis Search query string.
+        '''

gllm_datastore/data_store/redis/fulltext.pyi

@@ -0,0 +1,128 @@
+from _typeshed import Incomplete
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.constants import CHUNK_KEYS as CHUNK_KEYS, FIELD_CONFIG_NAME as FIELD_CONFIG_NAME, FIELD_CONFIG_TYPE as FIELD_CONFIG_TYPE, FieldType as FieldType
+from gllm_datastore.core.filters import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_datastore.data_store.redis.query import apply_options_to_query as apply_options_to_query, check_index_exists as check_index_exists, collect_document_ids as collect_document_ids, delete_keys_batched as delete_keys_batched, execute_search_query as execute_search_query, get_doc_ids_for_deletion as get_doc_ids_for_deletion, get_filterable_fields_from_index as get_filterable_fields_from_index, infer_filterable_fields_from_chunks as infer_filterable_fields_from_chunks, normalize_field_name_for_schema as normalize_field_name_for_schema, parse_redis_documents as parse_redis_documents, prepare_chunk_document as prepare_chunk_document, process_doc_ids_in_batches as process_doc_ids_in_batches, process_update_batch as process_update_batch, sanitize_key as sanitize_key, strip_index_prefix as strip_index_prefix, validate_chunk_content as validate_chunk_content, validate_chunk_list as validate_chunk_list, validate_metadata_fields as validate_metadata_fields
+from gllm_datastore.data_store.redis.query_translator import RedisQueryTranslator as RedisQueryTranslator
+from redis.asyncio.client import Redis
+from typing import Any
+
+FUZZY_MATCH_MAX_DISTANCE: int
+
+class DefaultBatchSize:
+    """Default batch sizes for Redis operations."""
+    DELETE: int
+    UPDATE: int
+
+class RedisFulltextCapability:
+    """Redis implementation of FulltextCapability protocol.
+
+    Attributes:
+        index_name (str): Name of the Redis index.
+        client (Redis): Redis client instance.
+    """
+    index_name: Incomplete
+    client: Incomplete
+    def __init__(self, index_name: str, client: Redis) -> None:
+        """Initialize the Redis fulltext capability.
+
+        Schema will be automatically inferred from chunks when creating a new index,
+        or auto-detected from an existing index when performing operations.
+
+        Args:
+            index_name (str): Name of the Redis index.
+            client (Redis): Redis client instance.
+        """
+    async def create(self, data: Chunk | list[Chunk]) -> None:
+        '''Create new records in the datastore.
+
+        If the index does not exist and no filterable_fields were provided,
+        the schema will be inferred from the chunks being created.
+
+        Examples:
+            Create a new chunk.
+            ```python
+            await fulltext_capability.create(Chunk(content="Test chunk", metadata={"category": "test"}))
+            ```
+
+        Args:
+            data (Chunk | list[Chunk]): Data to create (single item or collection).
+
+        Raises:
+            ValueError: If data structure is invalid or chunk content is invalid.
+        '''
+    async def retrieve(self, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        """Read records from the datastore with optional filtering.
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply. Defaults to None.
+            options (QueryOptions | None, optional): Query options for sorting and pagination. Defaults to None,
+                in which case the default limit of 10 is used.
+
+        Returns:
+            list[Chunk]: List of matched chunks after applying filters and options.
+        """
+    async def retrieve_fuzzy(self, query: str, max_distance: int = 2, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        """Find records that fuzzy match the query within distance threshold.
+
+        Args:
+            query (str): Text to fuzzy match against.
+            max_distance (int): Maximum edit distance for matches. Defaults to 2.
+                Maximum value is 3 (limitation of Redis Vector Search).
+            filters (FilterClause | QueryFilter | None, optional): Optional metadata filters to apply.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options, only limit is used here. Defaults to None.
+
+        Returns:
+            list[Chunk]: Matched chunks ordered by relevance/distance.
+
+        Raises:
+            ValueError: If max_distance is greater than 3.
+
+        Note:
+            Maximum fuzzy distance is 3. This is a limitation of the Redis Search module.
+        """
+    async def update(self, update_values: dict[str, Any], filters: FilterClause | QueryFilter | None = None) -> None:
+        '''Update existing records in the datastore.
+
+        Processes updates in batches to avoid loading all matching documents into memory.
+        1. Get document IDs matching the filters.
+        2. In batch, get document data via document IDs.
+        3. In batch, update the document data.
+
+        Examples:
+            Update certain metadata of a chunk with specific filters.
+            ```python
+            from gllm_datastore.core.filters import filter as F
+
+            await fulltext_capability.update(
+                update_values={"metadata": {"status": "published"}},
+                filters=F.eq("metadata.status", "draft"),
+            )
+            ```
+
+        Args:
+            update_values (dict[str, Any]): Mapping of fields to new values to apply.
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
+                Defaults to None.
+
+        Raises:
+            Exception: If Redis operations fail.
+        '''
+    async def delete(self, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> None:
+        """Delete records from the datastore.
+
+        Processes deletions in batches to avoid loading all matching documents into memory.
+        For delete operations, only document IDs are retrieved (not full content) to minimize memory usage.
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to delete.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options for sorting and limiting deletions
+                (for eviction-like operations). Defaults to None.
+
+        Raises:
+            Exception: If Redis operations fail.
+        """
+    async def clear(self) -> None:
+        """Clear all records from the datastore."""