gllm-datastore-binary 0.5.45__cp311-cp311-macosx_13_0_arm64.whl

gllm_datastore/core/capabilities/graph_capability.pyi

@@ -0,0 +1,70 @@
+from typing import Any, Protocol
+
+class GraphCapability(Protocol):
+    """Protocol for graph database operations.
+
+    This protocol defines the interface for datastores that support graph-based
+    data operations. This includes node and relationship management as well as graph queries.
+    """
+    async def upsert_node(self, label: str, identifier_key: str, identifier_value: str, properties: dict[str, Any] | None = None) -> Any:
+        """Create or update a node in the graph.
+
+        Args:
+            label (str): Node label/type.
+            identifier_key (str): Key field for node identification.
+            identifier_value (str): Value for node identification.
+            properties (dict[str, Any] | None, optional): Additional node properties.
+                Defaults to None.
+
+        Returns:
+            Any: Created/updated node information.
+        """
+    async def upsert_relationship(self, node_source_key: str, node_source_value: str, relation: str, node_target_key: str, node_target_value: str, properties: dict[str, Any] | None = None) -> Any:
+        """Create or update a relationship between nodes.
+
+        Args:
+            node_source_key (str): Source node identifier key.
+            node_source_value (str): Source node identifier value.
+            relation (str): Relationship type.
+            node_target_key (str): Target node identifier key.
+            node_target_value (str): Target node identifier value.
+            properties (dict[str, Any] | None, optional): Relationship properties.
+                Defaults to None.
+
+        Returns:
+            Any: Created/updated relationship information.
+        """
+    async def retrieve(self, query: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
+        """Retrieve data from the graph with specific query.
+
+        Args:
+            query (str): Query to retrieve data from the graph.
+            parameters (dict[str, Any] | None, optional): Query parameters. Defaults to None.
+
+        Returns:
+            list[dict[str, Any]]: Query results as list of dictionaries.
+        """
+    async def delete_node(self, label: str, identifier_key: str, identifier_value: str) -> Any:
+        """Delete a node and its relationships.
+
+        Args:
+            label (str): Node label/type.
+            identifier_key (str): Node identifier key.
+            identifier_value (str): Node identifier value.
+
+        Returns:
+            Any: Deletion result information.
+        """
+    async def delete_relationship(self, node_source_key: str, node_source_value: str, relation: str, node_target_key: str, node_target_value: str) -> Any:
+        """Delete a relationship between nodes.
+
+        Args:
+            node_source_key (str): Source node identifier key.
+            node_source_value (str): Source node identifier value.
+            relation (str): Relationship type.
+            node_target_key (str): Target node identifier key.
+            node_target_value (str): Target node identifier value.
+
+        Returns:
+            Any: Deletion result information.
+        """

gllm_datastore/core/capabilities/vector_capability.pyi

@@ -0,0 +1,90 @@
+from gllm_core.schema.chunk import Chunk
+from gllm_datastore.core.filters import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+from gllm_inference.schema import Vector
+from typing import Any, Protocol
+
+class VectorCapability(Protocol):
+    """Protocol for vector similarity search operations.
+
+    This protocol defines the interface for datastores that support vector-based
+    retrieval operations. This includes similarity search, ID-based lookup as well as
+    vector storage.
+    """
+    async def create(self, data: Chunk | list[Chunk]) -> None:
+        """Add chunks to the vector store with automatic embedding generation.
+
+        Args:
+            data (Chunk | list[Chunk]): Single chunk or list of chunks to add.
+        """
+    async def create_from_vector(self, chunk_vectors: list[tuple[Chunk, Vector]], **kwargs: Any) -> None:
+        """Add pre-computed vectors directly.
+
+        Args:
+            chunk_vectors (list[tuple[Chunk, Vector]]): List of tuples containing chunks and their
+                corresponding vectors.
+            **kwargs: Datastore-specific parameters.
+        """
+    async def retrieve(self, query: str, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None, **kwargs: Any) -> list[Chunk]:
+        """Read records from the datastore using text-based similarity search with optional filtering.
+
+        Args:
+            query (str): Input text to embed and search with.
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options like limit and sorting.
+                Defaults to None.
+            **kwargs: Datastore-specific parameters.
+
+        Returns:
+            list[Chunk]: Query results.
+        """
+    async def retrieve_by_vector(self, vector: Vector, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None, **kwargs: Any) -> list[Chunk]:
+        """Direct vector similarity search.
+
+        Args:
+            vector (Vector): Query embedding vector.
+            filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            options (QueryOptions | None, optional): Query options like limit and sorting.
+                Defaults to None.
+            **kwargs: Datastore-specific parameters.
+
+        Returns:
+            list[Chunk]: List of chunks ordered by similarity score.
+        """
+    async def update(self, update_values: dict[str, Any], filters: FilterClause | QueryFilter | None = None, **kwargs: Any) -> None:
+        """Update existing records in the datastore.
+
+        Args:
+            update_values (dict[str, Any]): Values to update.
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            **kwargs: Datastore-specific parameters.
+        """
+    async def delete(self, filters: FilterClause | QueryFilter | None = None, **kwargs: Any) -> None:
+        """Delete records from the datastore.
+
+        Args:
+            filters (FilterClause | QueryFilter | None, optional): Filters to select records to delete.
+                FilterClause objects are automatically converted to QueryFilter internally.
+                Defaults to None.
+            **kwargs: Datastore-specific parameters
+
+        Note:
+            If filters is None, no operation is performed (no-op).
+        """
+    async def clear(self) -> None:
+        """Clear all records from the datastore."""
+    async def ensure_index(self, **kwargs: Any) -> None:
+        """Ensure vector index exists, creating it if necessary.
+
+        This method ensures that the vector index required for similarity search
+        operations is created. If the index already exists, this method performs
+        no operation (idempotent).
+
+        Args:
+            **kwargs (Any): Datastore-specific parameters for index configuration.
+        """

gllm_datastore/core/filters/__init__.pyi

@@ -0,0 +1,4 @@
+from gllm_datastore.core.filters.filter import all_ as all_, and_ as and_, any_ as any_, array_contains as array_contains, eq as eq, gt as gt, gte as gte, in_ as in_, lt as lt, lte as lte, ne as ne, nin as nin, not_ as not_, or_ as or_, text_contains as text_contains
+from gllm_datastore.core.filters.schema import FilterClause as FilterClause, FilterCondition as FilterCondition, FilterOperator as FilterOperator, QueryFilter as QueryFilter, QueryOptions as QueryOptions
+
+__all__ = ['FilterCondition', 'FilterOperator', 'FilterClause', 'QueryFilter', 'QueryOptions', 'all_', 'and_', 'any_', 'array_contains', 'eq', 'gt', 'gte', 'in_', 'lt', 'lte', 'ne', 'nin', 'not_', 'or_', 'text_contains']

gllm_datastore/core/filters/filter.pyi

@@ -0,0 +1,340 @@
+from gllm_datastore.core.filters.schema import FilterClause as FilterClause, FilterCondition as FilterCondition, FilterOperator as FilterOperator, QueryFilter as QueryFilter
+from typing import Any
+
+def eq(key: str, value: Any) -> FilterClause:
+    '''Create an equality filter.
+
+    This operator checks if the field value is exactly equal to the specified value.
+    Works with strings, numbers, booleans, and other scalar types.
+
+    Example:
+        Filter for documents where `metadata.status == active`.
+        ```python
+        from gllm_datastore.core.filters import eq
+
+        filter = eq("metadata.status", "active")
+        ```
+
+    Args:
+        key (str): Field path to filter on.
+        value (Any): Value to compare. Matches field values exactly equal to this value.
+
+    Returns:
+        FilterClause: Equality filter.
+    '''
+def ne(key: str, value: Any) -> FilterClause:
+    '''Create a not-equal filter.
+
+    This operator checks if the field value is not equal to the specified value.
+    Works with strings, numbers, booleans, and other scalar types.
+
+    Example:
+        Filter for documents where `metadata.status != active`.
+        ```python
+        from gllm_datastore.core.filters import ne
+
+        filter = ne("metadata.status", "active")
+        ```
+
+    Args:
+        key (str): Field path to filter on.
+        value (Any): Value to exclude. Matches all values except this one.
+
+    Returns:
+        FilterClause: Not-equal filter.
+    '''
+def gt(key: str, value: int | float) -> FilterClause:
+    '''Create a greater-than filter.
+
+    This operator checks if the field value is strictly greater than the specified value.
+    Only works with numeric fields (int or float).
+
+    Example:
+        Filter for documents where `metadata.price > 100`.
+        ```python
+        from gllm_datastore.core.filters import gt
+
+        filter = gt("metadata.price", 100)
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be numeric).
+        value (int | float): Threshold value. Matches field values greater than this.
+
+    Returns:
+        FilterClause: Greater-than filter.
+    '''
+def lt(key: str, value: int | float) -> FilterClause:
+    '''Create a less-than filter.
+
+    This operator checks if the field value is strictly less than the specified value.
+    Only works with numeric fields (int or float).
+
+    Example:
+        Filter for documents where `metadata.price < 100`.
+        ```python
+        from gllm_datastore.core.filters import lt
+
+        filter = lt("metadata.price", 100)
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be numeric).
+        value (int | float): Threshold value. Matches field values less than this.
+
+    Returns:
+        FilterClause: Less-than filter.
+    '''
+def gte(key: str, value: int | float) -> FilterClause:
+    '''Create a greater-than-or-equal filter.
+
+    This operator checks if the field value is greater than or equal to the specified value.
+    Only works with numeric fields (int or float).
+
+    Example:
+        Filter for documents where `metadata.price >= 100`.
+        ```python
+        from gllm_datastore.core.filters import gte
+
+        filter = gte("metadata.price", 100)
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be numeric).
+        value (int | float): Threshold value. Matches field values greater than or equal to this.
+
+    Returns:
+        FilterClause: Greater-than-or-equal filter.
+    '''
+def lte(key: str, value: int | float) -> FilterClause:
+    '''Create a less-than-or-equal filter.
+
+    This operator checks if the field value is less than or equal to the specified value.
+    Only works with numeric fields (int or float).
+
+    Example:
+        Filter for documents where `metadata.price <= 100`.
+        ```python
+        from gllm_datastore.core.filters import lte
+
+        filter = lte("metadata.price", 100)
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be numeric).
+        value (int | float): Threshold value. Matches field values less than or equal to this.
+
+    Returns:
+        FilterClause: Less-than-or-equal filter.
+    '''
+def in_(key: str, values: list) -> FilterClause:
+    '''Create an IN filter.
+
+    This operator checks if the field value is one of the values in the provided list.
+    Works with scalar fields (string, number, boolean). The field value must exactly
+    match one of the values in the list.
+
+    Example:
+        Filter for documents where `metadata.status in ["active", "pending"]`.
+        ```python
+        from gllm_datastore.core.filters import in_
+
+        filter = in_("metadata.status", ["active", "pending"])
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be a scalar field).
+        values (list): List of possible values. Matches field values that match one of these exactly.
+
+    Returns:
+        FilterClause: IN filter.
+    '''
+def nin(key: str, values: list) -> FilterClause:
+    '''Create a NOT IN filter.
+
+    This operator checks if the field value is not in the provided list.
+    Works with scalar fields (string, number, boolean). The field value must not
+    match any of the values in the list.
+
+    Example:
+        Filter for documents where `metadata.status not in ["deleted", "archived"]`.
+        ```python
+        from gllm_datastore.core.filters import nin
+
+        filter = nin("metadata.status", ["deleted", "archived"])
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be a scalar field).
+        values (list): List of excluded values. Matches field values that do not match any of these.
+
+    Returns:
+        FilterClause: NOT IN filter.
+    '''
+def array_contains(key: str, value: Any) -> FilterClause:
+    '''Create an ARRAY_CONTAINS filter (array field contains value).
+
+    This operator checks if an array field contains the specified value as an element.
+    The field must be an array/list, and the value must be present in that array.
+    Use this for checking array membership.
+
+    Example:
+        Filter for documents where the tags array contains "python".
+        This will match documents where "python" is an element in metadata.tags.
+        For example, if metadata.tags = ["python", "javascript"], this will match.
+        ```python
+        from gllm_datastore.core.filters import array_contains
+
+        filter = array_contains("metadata.tags", "python")
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be an array field).
+        value (Any): Value to check if it exists as an element in the array.
+
+    Returns:
+        FilterClause: ARRAY_CONTAINS filter.
+    '''
+def text_contains(key: str, value: str) -> FilterClause:
+    '''Create a TEXT_CONTAINS filter (text field contains substring).
+
+    This operator checks if a text/string field contains the specified substring.
+    The field must be a string, and the value must appear as a substring within that string.
+    Use this for substring matching in text content.
+
+    Example:
+        Filter for documents where the content field contains "machine learning".
+        This will match documents where "machine learning" appears anywhere in the content.
+        For example, if content = "This is about machine learning algorithms", this will match.
+        ```python
+        from gllm_datastore.core.filters import text_contains
+
+        filter = text_contains("content", "machine learning")
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be a string/text field).
+        value (str): Substring to search for in the text.
+
+    Returns:
+        FilterClause: TEXT_CONTAINS filter.
+    '''
+def any_(key: str, values: list) -> FilterClause:
+    '''Create an ANY filter (array field contains any of the values).
+
+    This operator checks if an array field contains at least one of the values in the provided list.
+    The field must be an array/list, and at least one element from the values list must be
+    present in the array. This is similar to checking if the arrays have any intersection.
+
+    Example:
+        Filter for documents where the tags array contains at least one of "python" or "javascript".
+        This will match if metadata.tags contains "python", "javascript", or both.
+        For example, if metadata.tags = ["python", "rust"], this will match (because of "python").
+        ```python
+        from gllm_datastore.core.filters import any_
+
+        filter = any_("metadata.tags", ["python", "javascript"])
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be an array field).
+        values (list): List of values. At least one must be present in the array.
+
+    Returns:
+        FilterClause: ANY filter.
+    '''
+def all_(key: str, values: list) -> FilterClause:
+    '''Create an ALL filter (array field contains all of the values).
+
+    This operator checks if an array field contains all of the values in the provided list.
+    The field must be an array/list, and every value in the values list must be present
+    as an element in the array. The array may contain additional elements.
+
+    Example:
+        Filter for documents where the tags array contains both "python" and "javascript".
+        This will match only if metadata.tags contains both values.
+        For example, if metadata.tags = ["python", "javascript", "rust"], this will match.
+        If metadata.tags = ["python", "rust"], this will not match (missing "javascript").
+        ```python
+        from gllm_datastore.core.filters import all_
+
+        filter = all_("metadata.tags", ["python", "javascript"])
+        ```
+
+    Args:
+        key (str): Field path to filter on (must be an array field).
+        values (list): List of values. All must be present in the array.
+
+    Returns:
+        FilterClause: ALL filter.
+    '''
+def and_(*filters: FilterClause | QueryFilter) -> QueryFilter:
+    '''Combine filters with AND condition.
+
+    This logical operator combines multiple filters such that all conditions must be satisfied.
+    A document matches only if it satisfies every filter in the list.
+
+    Example:
+        Filter for documents where status is "active" AND age is at least 18.
+        This will match documents that satisfy both conditions simultaneously.
+        ```python
+        from gllm_datastore.core.filters import and_, eq, gte
+
+        filter = and_(eq("metadata.status", "active"), gte("metadata.age", 18))
+        ```
+
+    Args:
+        *filters (FilterClause | QueryFilter): Variable number of filters to combine.
+            All filters must match for a document to be included.
+
+    Returns:
+        QueryFilter: Combined filter with AND condition.
+    '''
+def or_(*filters: FilterClause | QueryFilter) -> QueryFilter:
+    '''Combine filters with OR condition.
+
+    This logical operator combines multiple filters such that at least one condition must be satisfied.
+    A document matches if it satisfies any of the filters in the list.
+
+    Example:
+        Filter for documents where status is "active" OR status is "pending".
+        This will match documents that satisfy either condition (or both).
+        ```python
+        from gllm_datastore.core.filters import or_, eq
+
+        filter = or_(eq("metadata.status", "active"), eq("metadata.status", "pending"))
+        ```
+
+    Args:
+        *filters (FilterClause | QueryFilter): Variable number of filters to combine.
+            At least one filter must match for a document to be included.
+
+    Returns:
+        QueryFilter: Combined filter with OR condition.
+    '''
+def not_(filter: FilterClause | QueryFilter) -> QueryFilter:
+    '''Negate a filter.
+
+    This logical operator inverts the result of a filter. A document matches if it does
+    not satisfy the specified filter condition. Useful for exclusion criteria.
+
+    This operator only supports NOT with a single filter. Multiple filters in NOT condition are not supported.
+
+    Example:
+        Filter for documents where status is NOT "deleted".
+        This will match all documents except those with status == "deleted".
+        Can also be used with other operators, e.g., not_(text_contains("content", "spam"))
+        to exclude documents containing a specific substring.
+        ```python
+        from gllm_datastore.core.filters import not_, eq
+
+        filter = not_(eq("metadata.status", "deleted"))
+        ```
+
+    Args:
+        filter (FilterClause | QueryFilter): Filter to negate. Documents matching this
+            filter will be excluded from results.
+
+    Returns:
+        QueryFilter: Negated filter.
+    '''

gllm_datastore/core/filters/schema.pyi

@@ -0,0 +1,149 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from typing import Any, Sequence
+
+class FilterOperator(StrEnum):
+    """Operators for comparing field values."""
+    EQ: str
+    NE: str
+    GT: str
+    LT: str
+    GTE: str
+    LTE: str
+    IN: str
+    NIN: str
+    ANY: str
+    ALL: str
+    ARRAY_CONTAINS: str
+    TEXT_CONTAINS: str
+
+class FilterCondition(StrEnum):
+    """Logical conditions for combining filters."""
+    AND: str
+    OR: str
+    NOT: str
+
+class FilterClause(BaseModel):
+    '''Single filter criterion with operator support.
+
+    Examples:
+        ```python
+        FilterClause(key="metadata.age", value=25, operator=FilterOperator.GT)
+        FilterClause(key="metadata.status", value=["active", "pending"], operator=FilterOperator.IN)
+        ```
+
+    Attributes:
+        key (str): The field path to filter on (supports dot notation for nested fields).
+        value (int | float | str | bool | list[str] | list[float] | list[int] | list[bool] | None):
+            The value to compare against.
+        operator (FilterOperator): The comparison operator.
+    '''
+    key: str
+    value: bool | int | float | str | list[str] | list[float] | list[int] | list[bool] | None
+    operator: FilterOperator
+    def to_query_filter(self) -> QueryFilter:
+        '''Convert FilterClause to QueryFilter.
+
+        This method enables automatic conversion of FilterClause to QueryFilter.
+
+        Example:
+            ```python
+            clause = FilterClause(key="metadata.status", value="active", operator=FilterOperator.EQ)
+            query_filter = clause.to_query_filter()
+            # Results in: QueryFilter(filters=[clause], condition=FilterCondition.AND)
+            ```
+
+        Returns:
+            QueryFilter: A QueryFilter wrapping this FilterClause with AND condition.
+        '''
+
+class QueryFilter(BaseModel):
+    '''Composite filter supporting multiple conditions and logical operators.
+
+    Attributes:
+        filters (list[FilterClause | QueryFilter]): List of filters to combine.
+            Can include nested QueryFilter for complex logic.
+        condition (FilterCondition): Logical operator to combine filters. Defaults to AND.
+
+    Examples:
+        1. Simple AND: age > 25 AND status == "active"
+            ```python
+            QueryFilter(
+                filters=[
+                    FilterClause(key="metadata.age", value=25, operator=FilterOperator.GT),
+                    FilterClause(key="metadata.status", value="active", operator=FilterOperator.EQ)
+                ],
+                condition=FilterCondition.AND
+            )
+            ```
+
+        2. Complex OR: (status == "active" OR status == "pending") AND age >= 18
+            ```python
+            QueryFilter(
+                filters=[
+                    QueryFilter(
+                        filters=[
+                            FilterClause(key="metadata.status", value="active"),
+                            FilterClause(key="metadata.status", value="pending")
+                        ],
+                        condition=FilterCondition.OR
+                    ),
+                    FilterClause(key="metadata.age", value=18, operator=FilterOperator.GTE)
+                ],
+                condition=FilterCondition.AND
+            )
+            ```
+
+        3. NOT: NOT (status == "deleted")
+            ```python
+            QueryFilter(
+                filters=[
+                    FilterClause(key="metadata.status", value="deleted")
+                ],
+                condition=FilterCondition.NOT
+            )
+            ```
+    '''
+    filters: list[FilterClause | QueryFilter]
+    condition: FilterCondition
+    @classmethod
+    def from_dicts(cls, filter_dicts: list[dict[str, Any]], condition: FilterCondition = ...) -> QueryFilter:
+        '''Create QueryFilter from list of filter dictionaries.
+
+        Example:
+            ```python
+            QueryFilter.from_dicts(
+                [
+                    {"key": "metadata.age", "value": 25, "operator": ">"},
+                    {"key": "metadata.status", "value": "active"}
+                ],
+                condition=FilterCondition.AND
+            )
+            ```
+
+        Args:
+            filter_dicts (list[dict[str, Any]]): List of filter dictionaries. Contains the key, value, and operator.
+            condition (FilterCondition, optional): Logical operator to combine filters. Defaults to AND.
+
+        Returns:
+            QueryFilter: Composite filter instance.
+        '''
+
+class QueryOptions(BaseModel):
+    '''Model for query options.
+
+    Attributes:
+        include_fields (Sequence[str] | None): The fields to include in the query result. Defaults to None.
+        order_by (str | None): The column to order the query result by. Defaults to None.
+        order_desc (bool): Whether to order the query result in descending order. Defaults to False.
+        limit (int | None): The maximum number of rows to return. Must be >= 0. Defaults to None.
+
+    Example:
+        ```python
+        QueryOptions(include_fields=["field1", "field2"], order_by="column1", order_desc=True, limit=10)
+        ```
+    '''
+    include_fields: Sequence[str] | None
+    order_by: str | None
+    order_desc: bool
+    limit: int | None

gllm_datastore/data_store/__init__.pyi

@@ -0,0 +1,7 @@
+from gllm_datastore.data_store.chroma import ChromaDataStore as ChromaDataStore
+from gllm_datastore.data_store.elasticsearch import ElasticsearchDataStore as ElasticsearchDataStore
+from gllm_datastore.data_store.exceptions import NotRegisteredException as NotRegisteredException, NotSupportedException as NotSupportedException
+from gllm_datastore.data_store.in_memory import InMemoryDataStore as InMemoryDataStore
+from gllm_datastore.data_store.redis import RedisDataStore as RedisDataStore
+
+__all__ = ['ChromaDataStore', 'ElasticsearchDataStore', 'InMemoryDataStore', 'NotRegisteredException', 'NotSupportedException', 'RedisDataStore']