gllm-datastore-binary 0.5.45__cp311-cp311-macosx_13_0_arm64.whl

gllm_datastore/data_store/redis/vector.pyi

@@ -0,0 +1,131 @@
+from _typeshed import Incomplete
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.constants import CHUNK_KEYS as CHUNK_KEYS, FIELD_CONFIG_NAME as FIELD_CONFIG_NAME, FIELD_CONFIG_TYPE as FIELD_CONFIG_TYPE
+from gllm_datastore.core.filters import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_datastore.data_store.redis.query import build_filter_expression as build_filter_expression, check_index_exists as check_index_exists, execute_update as execute_update, fetch_hash_data_batch as fetch_hash_data_batch, get_filterable_fields_from_index as get_filterable_fields_from_index, infer_filterable_fields_from_chunks as infer_filterable_fields_from_chunks, normalize_field_name_for_schema as normalize_field_name_for_schema, parse_redisvl_result_to_chunks as parse_redisvl_result_to_chunks, prepare_chunk_document as prepare_chunk_document, strip_index_prefix as strip_index_prefix, validate_chunk_content as validate_chunk_content, validate_chunk_list as validate_chunk_list, validate_metadata_fields as validate_metadata_fields
+from gllm_datastore.data_store.redis.query_translator import RedisQueryTranslator as RedisQueryTranslator
+from gllm_datastore.utils.converter import cosine_distance_to_similarity_score as cosine_distance_to_similarity_score
+from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
+from gllm_inference.schema import Vector
+from redis.asyncio.client import Redis
+from typing import Any
+
+class RedisVectorCapability:
+    """Redis implementation of VectorCapability protocol.
+
+    This class provides vector similarity search operations using RedisVL
+    AsyncSearchIndex for vector storage and retrieval.
+
+    Attributes:
+        index_name (str): Name of the Redis index.
+        client (Redis): Redis async client instance.
+        em_invoker (BaseEMInvoker): Embedding model for vectorization.
+        index (Any): RedisVL AsyncSearchIndex instance.
+    """
+    index_name: Incomplete
+    client: Incomplete
+    index: Any
+    def __init__(self, index_name: str, client: Redis, em_invoker: BaseEMInvoker) -> None:
+        """Initialize the Redis vector capability.
+
+        Schema will be automatically inferred from chunks when creating a new index,
+        or auto-detected from an existing index when performing operations.
+
+        Args:
+            index_name (str): Name of the Redis index.
+            client (Redis): Redis async client instance.
+            em_invoker (BaseEMInvoker): Embedding model for vectorization.
+        """
+    @property
+    def em_invoker(self) -> BaseEMInvoker:
+        """Returns the EM Invoker instance.
+
+        Returns:
+            BaseEMInvoker: The EM Invoker instance.
+        """
+    async def ensure_index(self, filterable_fields: list[dict[str, Any]] | None = None) -> None:
+        '''Ensure Redis vector index exists, creating it if necessary.
+
+        This method is idempotent - if the index already exists, it will skip creation
+        and return early.
+
+        Args:
+            filterable_fields (list[dict[str, Any]] | None, optional): List of filterable field
+                configurations to use when creating a new index. Each field should be a dictionary
+                with "name" and "type" keys. For example:
+                [{"name": "metadata.category", "type": "tag"}, {"name": "metadata.score", "type": "numeric"}]
+                If not provided and index doesn\'t exist, a default schema will be created with
+                only basic fields (id, content, metadata, vector). Defaults to None.
+
+        Raises:
+            RuntimeError: If index creation fails.
+        '''
+    async def create(self, data: Chunk | list[Chunk]) -> None:
+        """Add chunks to the vector store with automatic embedding generation.
+
+        If the index does not exist, the schema will be inferred from the chunks being created.
+
+        Args:
+            data (Chunk | list[Chunk]): Single chunk or list of chunks to add.
+
+        Raises:
+            ValueError: If data structure is invalid or chunk content is invalid.
+        """
+    async def create_from_vector(self, chunk_vectors: list[tuple[Chunk, Vector]]) -> None:
+        """Add pre-computed vectors directly.
+
+        If the index does not exist, the schema will be inferred from the chunks being created.
+
+        Args:
+            chunk_vectors (list[tuple[Chunk, Vector]]): List of tuples containing chunks
+                and their corresponding vectors.
+
+        Raises:
+            ValueError: If chunk content is invalid.
+        """
+    async def retrieve(self, query: str, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        """Read records from the datastore using text-based similarity search with optional filtering.
+
+        Args:
+            query (str): Input text to embed and search with.
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options like limit and sorting. Defaults to None.
+
+        Returns:
+            list[Chunk]: Query results ordered by similarity score.
+        """
+    async def retrieve_by_vector(self, vector: Vector, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
+        """Direct vector similarity search.
+
+        Args:
+            vector (Vector): Query embedding vector.
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options like limit and sorting. Defaults to None.
+
+        Returns:
+            list[Chunk]: List of chunks ordered by similarity score.
+        """
+    async def update(self, update_values: dict[str, Any], filters: FilterClause | QueryFilter | None = None) -> None:
+        """Update existing records in the datastore.
+
+        Args:
+            update_values (dict[str, Any]): Values to update.
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+        """
+    async def delete(self, filters: FilterClause | QueryFilter | None = None) -> None:
+        """Delete records from the datastore.
+
+        Processes deletions in batches to avoid loading all matching documents into memory.
+        If filters is None, no operation is performed (no-op).
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to delete.
+                Defaults to None.
+        """
+    async def clear(self) -> None:
+        """Clear all records from the datastore."""

gllm_datastore/encryptor/__init__.pyi

@@ -0,0 +1,4 @@
+from gllm_datastore.encryptor.aes_gcm_encryptor import AESGCMEncryptor as AESGCMEncryptor
+from gllm_datastore.encryptor.key_rotating_encryptor import KeyRotatingEncryptor as KeyRotatingEncryptor
+
+__all__ = ['AESGCMEncryptor', 'KeyRotatingEncryptor']

gllm_datastore/encryptor/aes_gcm_encryptor.pyi

@@ -0,0 +1,45 @@
+from _typeshed import Incomplete
+from gllm_datastore.encryptor.encryptor import BaseEncryptor as BaseEncryptor
+
+KEY_LENGTH_BYTES: int
+NONCE_LENGTH_BYTES: int
+
+class AESGCMEncryptor(BaseEncryptor):
+    """AES-GCM 256 Encryptor that accepts keys directly.
+
+    This class provides AES-GCM symmetric encryption and decryption methods
+    with a 256-bit key provided directly by the client.
+
+    Attributes:
+        key (bytes): 256-bit encryption key.
+        aesgcm (AESGCM): AES-GCM instance.
+    """
+    key: Incomplete
+    aesgcm: Incomplete
+    def __init__(self, key: bytes) -> None:
+        """Initialize AESGCMEncryptor with a direct key.
+
+        Args:
+            key (bytes): 256-bit encryption key.
+
+        Raises:
+            ValueError: If key length is not 256 bits.
+        """
+    def encrypt(self, plaintext: str) -> str:
+        """Encrypts the plaintext using AES-GCM with a random nonce.
+
+        Args:
+            plaintext (str): The plaintext data to be encrypted.
+
+        Returns:
+            str: The encrypted data, encoded in base64 format.
+        """
+    def decrypt(self, ciphertext: str) -> str:
+        """Decrypts the AES-GCM ciphertext.
+
+        Args:
+            ciphertext (str): The ciphertext in base64 format to be decrypted.
+
+        Returns:
+            str: The decrypted plaintext data.
+        """

gllm_datastore/encryptor/encryptor.pyi

@@ -0,0 +1,52 @@
+from abc import ABC, abstractmethod
+
+class BaseEncryptor(ABC):
+    """Abstract base class defining the interface for encryption implementations.
+
+    This abstract base class ensures that all encryptors implement the required
+    encrypt and decrypt methods with consistent signatures.
+
+    Thread-safety requirement:
+        Implementations MUST be thread-safe. The client may
+        invoke `encrypt` and `decrypt` concurrently from multiple threads, so
+        any internal state (e.g., buffers, nonces, cipher instances) must be
+        protected or designed to avoid race conditions.
+    """
+    @abstractmethod
+    def encrypt(self, plaintext: str) -> str:
+        """Encrypt plain text into cipher text.
+
+        This method should be implemented by subclasses to provide the encryption functionality.
+
+        Note:
+            The implementation must be thread-safe and must not mutate shared state
+            without proper synchronization.
+
+        Args:
+            plaintext (str): The raw plain text to encrypt.
+
+        Returns:
+            str: The encrypted cipher text.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by the subclass.
+        """
+    @abstractmethod
+    def decrypt(self, ciphertext: str) -> str:
+        """Decrypt cipher text back into plain text.
+
+        This method should be implemented by subclasses to provide the decryption functionality.
+
+        Note:
+            The implementation must be thread-safe and must not mutate shared state
+            without proper synchronization.
+
+        Args:
+            ciphertext (str): The ciphertext to decrypt.
+
+        Returns:
+            str: The decrypted plain text.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by the subclass.
+        """

gllm_datastore/encryptor/key_ring/__init__.pyi

@@ -0,0 +1,3 @@
+from gllm_datastore.encryptor.key_ring.in_memory_key_ring import InMemoryKeyRing as InMemoryKeyRing
+
+__all__ = ['InMemoryKeyRing']

gllm_datastore/encryptor/key_ring/in_memory_key_ring.pyi

@@ -0,0 +1,52 @@
+from gllm_datastore.encryptor.encryptor import BaseEncryptor as BaseEncryptor
+from gllm_datastore.encryptor.key_ring.key_ring import BaseKeyRing as BaseKeyRing
+
+class InMemoryKeyRing(BaseKeyRing):
+    """In-memory implementation of BaseKeyRing.
+
+    This class provides a simple in-memory storage for encryption keys and
+    their associated encryptors. All keys are stored in memory and will be
+    lost when the application terminates.
+
+    Attributes:
+        encryptors (dict[str, BaseEncryptor]): A dictionary to store the keys and their associated encryptors.
+    """
+    encryptors: dict[str, BaseEncryptor]
+    def __init__(self, encryptors: dict[str, BaseEncryptor] | None = None) -> None:
+        """Initialize the InMemoryKeyRing.
+
+        Args:
+            encryptors (dict[str, BaseEncryptor] | None, optional): A dictionary to store the keys and
+                their associated encryptors. Defaults to None.
+        """
+    def get(self, key_id: str) -> BaseEncryptor:
+        """Get an encryptor by key ID.
+
+        Args:
+            key_id (str): ID of the key to retrieve.
+
+        Returns:
+            BaseEncryptor: The encryptor for the specified key.
+
+        Raises:
+            KeyError: If key_id does not exist.
+        """
+    def add(self, key_id: str, encryptor: BaseEncryptor) -> None:
+        """Add a new key to the key ring.
+
+        Args:
+            key_id (str): Unique identifier for the key.
+            encryptor (BaseEncryptor): The encryptor instance for this key.
+
+        Raises:
+            KeyError: If key_id already exists.
+        """
+    def remove(self, key_id: str) -> None:
+        """Remove a key from the key ring.
+
+        Args:
+            key_id (str): ID of the key to remove.
+
+        Raises:
+            KeyError: If key_id does not exist.
+        """

gllm_datastore/encryptor/key_ring/key_ring.pyi

@@ -0,0 +1,45 @@
+from abc import ABC, abstractmethod
+from gllm_datastore.encryptor.encryptor import BaseEncryptor as BaseEncryptor
+
+class BaseKeyRing(ABC):
+    """Abstract base class defining the interface for managing multiple encryption keys."""
+    @abstractmethod
+    def get(self, key_id: str) -> BaseEncryptor:
+        """Get an encryptor by key ID.
+
+        This method should be implemented by subclasses to provide the getting functionality.
+
+        Args:
+            key_id (str): ID of the key to retrieve.
+
+        Returns:
+            BaseEncryptor: The encryptor for the specified key.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by the subclass.
+        """
+    @abstractmethod
+    def add(self, key_id: str, encryptor: BaseEncryptor) -> None:
+        """Add a new key to the key ring.
+
+        This method should be implemented by subclasses to provide the adding functionality.
+
+        Args:
+            key_id (str): Unique identifier for the key.
+            encryptor (BaseEncryptor): The encryptor instance for this key.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by the subclass.
+        """
+    @abstractmethod
+    def remove(self, key_id: str) -> None:
+        """Remove a key from the key ring.
+
+        This method should be implemented by subclasses to provide the removing functionality.
+
+        Args:
+            key_id (str): ID of the key to remove.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by the subclass.
+        """

gllm_datastore/encryptor/key_rotating_encryptor.pyi

@@ -0,0 +1,60 @@
+from _typeshed import Incomplete
+from gllm_datastore.encryptor.encryptor import BaseEncryptor as BaseEncryptor
+from gllm_datastore.encryptor.key_ring.key_ring import BaseKeyRing as BaseKeyRing
+
+class KeyRotatingEncryptor(BaseEncryptor):
+    """Encryptor that supports key rotation through a key ring.
+
+    This encryptor uses a BaseKeyRing to manage multiple encryption keys.
+    Users must specify which key to use for encryption and decryption operations.
+
+    Attributes:
+        key_ring (BaseKeyRing): The key ring managing encryption keys.
+        active_key_id (str): The ID of the current key to use for encryption.
+    """
+    key_ring: Incomplete
+    def __init__(self, key_ring: BaseKeyRing, active_key_id: str) -> None:
+        """Initialize KeyRotatingEncryptor with a key ring.
+
+        Args:
+            key_ring (BaseKeyRing): The key ring to use for key management.
+            active_key_id (str): The ID of the current key to use for encryption.
+        """
+    @property
+    def active_key_id(self) -> str:
+        """Get the ID of the current key to use for encryption."""
+    @active_key_id.setter
+    def active_key_id(self, value: str) -> None:
+        """Set the ID of the current key to use for encryption.
+
+        Args:
+            value (str): The ID of the current key to use for encryption.
+
+        Raises:
+            KeyError: If the specified key does not exist.
+        """
+    def encrypt(self, plaintext: str) -> str:
+        """Encrypt plaintext using the specified key.
+
+        Args:
+            plaintext (str): The plaintext to encrypt.
+
+        Returns:
+            str: The encrypted data with key metadata, encoded in base64.
+
+        Raises:
+            KeyError: If the specified key does not exist.
+        """
+    def decrypt(self, ciphertext: str) -> str:
+        """Decrypt ciphertext the key detected from metadata.
+
+        Args:
+            ciphertext (str): The encrypted data with key metadata.
+
+        Returns:
+            str: The decrypted plaintext.
+
+        Raises:
+            ValueError: If the data format is invalid or decryption fails.
+            KeyError: If the required key is not available.
+        """

gllm_datastore/graph_data_store/__init__.pyi

@@ -0,0 +1,6 @@
+from gllm_datastore.graph_data_store.light_rag_postgres_data_store import LightRAGPostgresDataStore as LightRAGPostgresDataStore
+from gllm_datastore.graph_data_store.llama_index_neo4j_graph_rag_data_store import LlamaIndexNeo4jGraphRAGDataStore as LlamaIndexNeo4jGraphRAGDataStore
+from gllm_datastore.graph_data_store.nebula_graph_data_store import NebulaGraphDataStore as NebulaGraphDataStore
+from gllm_datastore.graph_data_store.neo4j_graph_data_store import Neo4jGraphDataStore as Neo4jGraphDataStore
+
+__all__ = ['LightRAGPostgresDataStore', 'LlamaIndexNeo4jGraphRAGDataStore', 'NebulaGraphDataStore', 'Neo4jGraphDataStore']

gllm_datastore/graph_data_store/graph_data_store.pyi

@@ -0,0 +1,151 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseGraphDataStore(ABC):
+    """Abstract base class for an async graph data store interface.
+
+    This class defines the asynchronous interface for all graph data store implementations.
+    It provides methods for creating, updating, and querying graph data.
+    """
+    @abstractmethod
+    async def upsert_node(self, label: str, identifier_key: str, identifier_value: str, properties: dict[str, Any] | None) -> Any:
+        """Upsert a node in the graph.
+
+        Args:
+            label (str): The label of the node.
+            identifier_key (str): The key of the identifier.
+            identifier_value (str): The value of the identifier.
+            properties (dict[str, Any] | None, optional): The properties of the node. Defaults to None.
+
+        Returns:
+            Any: The result of the operation.
+        """
+    @abstractmethod
+    async def upsert_relationship(self, node_source_key: str, node_source_value: str, relation: str, node_target_key: str, node_target_value: str, properties: dict[str, Any] | None) -> Any:
+        """Upsert a relationship between two nodes in the graph.
+
+        Args:
+            node_source_key (str): The key of the source node.
+            node_source_value (str): The value of the source node.
+            relation (str): The type of the relationship.
+            node_target_key (str): The key of the target node.
+            node_target_value (str): The value of the target node.
+            properties (dict[str, Any] | None, optional): The properties of the relationship. Defaults to None.
+
+        Returns:
+            Any: The result of the operation.
+        """
+    @abstractmethod
+    async def delete_node(self, label: str, identifier_key: str, identifier_value: str) -> Any:
+        """Delete a node from the graph.
+
+        Args:
+            label (str): The label of the node.
+            identifier_key (str): The key of the identifier.
+            identifier_value (str): The identifier of the node.
+
+        Returns:
+            Any: The result of the operation.
+        """
+    @abstractmethod
+    async def delete_relationship(self, node_source_key: str, node_source_value: str, relation: str, node_target_key: str, node_target_value: str) -> Any:
+        """Delete a relationship between two nodes in the graph.
+
+        Args:
+            node_source_key (str): The key of the source node.
+            node_source_value (str): The identifier of the source node.
+            relation (str): The type of the relationship.
+            node_target_key (str): The key of the target node.
+            node_target_value (str): The identifier of the target node.
+
+        Returns:
+            Any: The result of the operation.
+        """
+    @abstractmethod
+    async def query(self, query: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
+        """Query the graph data store.
+
+        Args:
+            query (str): The query to be executed.
+            parameters (dict[str, Any] | None, optional): The parameters of the query. Defaults to None.
+
+        Returns:
+            list[dict[str, Any]]: The result of the query as a list of dictionaries.
+        """
+    @abstractmethod
+    async def traverse_graph(self, node_properties: dict[str, Any], extracted_node_properties: list[str] | None = None, extracted_relationship_properties: list[str] | None = None, depth: int = 3) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+        '''Traverse graph from a node with specified properties, ignoring relationship\'s direction, up to a given depth.
+
+        Example:
+            ```python
+            nodes, relationships = await graph_data_store.traverse_graph(
+                node_properties={"name": "John Doe"},
+                extracted_node_properties=["name", "age"],
+                extracted_relationship_properties=["since"],
+                depth=1
+            )
+            ```
+            Means starting from the node with property `name` equal to "John Doe", traverse
+            the graph up to depth 1, extracting the `name` and `age` properties from nodes
+            and the `since` property from relationships.
+
+            ```python
+            nodes, relationships = await graph_data_store.traverse_graph(
+                node_properties={"name": "John Doe"},
+                depth=2
+            )
+            ```
+            Means starting from the node with property `name` equal to "John Doe", traverse
+            the graph up to depth 2, extracting all properties from nodes and relationships.
+
+        Args:
+            node_properties (dict[str, Any]): The properties of the starting node.
+            extracted_node_properties (list[str] | None, optional): The properties to extract from nodes during
+                traversal. If None or empty list, all node properties will be returned. Defaults to None.
+            extracted_relationship_properties (list[str] | None, optional): The properties to extract from relationships
+                during traversal. If None or empty list, all relationship properties will be returned. Defaults to None.
+            depth (int, optional): The depth of traversal. Defaults to 3.
+
+        Returns:
+            tuple[list[dict[str, Any]], list[dict[str, Any]]]: A tuple containing two lists:
+                - List of nodes with their extracted properties.
+                - List of relationships with their extracted properties.
+
+            Example return value:
+            nodes = [
+                {
+                    "id": 1001,
+                    "labels": ["Person"],
+                    "properties": {
+                        "name": "John Doe",
+                        "age": 30,
+                        "occupation": "Engineer"
+                    }
+                },
+                {
+                    "id": 2001,
+                    "labels": ["Company"],
+                    "properties": {
+                        "name": "TechCorp",
+                        "industry": "Technology",
+                        "employees": 500
+                    }
+                }
+            ]
+
+            relationships = [
+                {
+                    "id": 5002,
+                    "type": "FRIEND_OF",
+                    "start_node": 1001,
+                    "end_node": 1002,
+                    "properties": {
+                        "since": "2018-05-20",
+                        "closeness": 8
+                    }
+                }
+            ]
+        '''
+    @abstractmethod
+    async def close(self) -> None:
+        """Close the graph data store."""

gllm_datastore/graph_data_store/graph_rag_data_store.pyi

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseGraphRAGDataStore(ABC):
+    """Abstract base class for graph RAG data stores in the retrieval system.
+
+    This class defines the interface for all graph-based Retrieval-Augmented
+    Generation (RAG) implementations. It provides methods for querying the graph with
+    natural language and managing document-related data.
+    """
+    @abstractmethod
+    async def query(self, query: str, **kwargs: Any) -> list[dict[str, Any]]:
+        """Query the graph RAG data store.
+
+        Args:
+            query (str): The query to be executed.
+            **kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            list[dict[str, Any]]: The result of the query as a list of dictionaries.
+        """
+    @abstractmethod
+    async def delete_by_document_id(self, document_id: str, **kwargs: Any) -> None:
+        """Delete nodes and edges by document ID.
+
+        Args:
+            document_id (str): The document ID.
+            **kwargs (Any): Additional keyword arguments.
+        """