llama-index-vector-stores-opensearch 0.1.12__tar.gz → 0.1.14__tar.gz

{llama_index_vector_stores_opensearch-0.1.12 → llama_index_vector_stores_opensearch-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama-index-vector-stores-opensearch
-Version: 0.1.12
+Version: 0.1.14
 Summary: llama-index vector_stores opensearch integration
 License: MIT
 Author: Your Name

{llama_index_vector_stores_opensearch-0.1.12 → llama_index_vector_stores_opensearch-0.1.14}/llama_index/vector_stores/opensearch/base.py

@@ -1,14 +1,17 @@
 """Elasticsearch/Opensearch vector store."""
 
 import asyncio
-import json
 import uuid
+from datetime import datetime
 from typing import Any, Dict, Iterable, List, Optional, Union, cast
 
 from llama_index.core.bridge.pydantic import PrivateAttr
 
 from llama_index.core.schema import BaseNode, MetadataMode, TextNode
 from llama_index.core.vector_stores.types import (
+    FilterCondition,
+    FilterOperator,
+    MetadataFilter,
     MetadataFilters,
     BasePydanticVectorStore,
     VectorStoreQuery,
@@ -210,13 +213,89 @@ class OpensearchVectorClient:
             "query": {"knn": {vector_field: {"vector": query_vector, "k": k}}},
         }
 
-    def _parse_filters(self, filters: Optional[MetadataFilters]) -> Any:
-        pre_filter = []
-        if filters is not None:
-            for f in filters.legacy_filters():
-                pre_filter.append({f.key: json.loads(str(f.value))})
+    def _is_text_field(self, value: Any) -> bool:
+        """Check if value is a string and keyword filtering needs to be performed.
 
-        return pre_filter
+        Not applied to datetime strings.
+        """
+        if isinstance(value, str):
+            try:
+                datetime.fromisoformat(value)
+                return False
+            except ValueError as e:
+                return True
+        else:
+            return False
+
+    def _parse_filter(self, filter: MetadataFilter) -> dict:
+        """Parse a single MetadataFilter to equivalent OpenSearch expression.
+
+        As Opensearch does not differentiate between scalar/array keyword fields, IN and ANY are equivalent.
+        """
+        key = f"metadata.{filter.key}"
+        op = filter.operator
+
+        equality_postfix = ".keyword" if self._is_text_field(value=filter.value) else ""
+
+        if op == FilterOperator.EQ:
+            return {"term": {f"{key}{equality_postfix}": filter.value}}
+        elif op in [
+            FilterOperator.GT,
+            FilterOperator.GTE,
+            FilterOperator.LT,
+            FilterOperator.LTE,
+        ]:
+            return {"range": {key: {filter.operator.name.lower(): filter.value}}}
+        elif op == FilterOperator.NE:
+            return {
+                "bool": {
+                    "must_not": {"term": {f"{key}{equality_postfix}": filter.value}}
+                }
+            }
+        elif op in [FilterOperator.IN, FilterOperator.ANY]:
+            return {"terms": {key: filter.value}}
+        elif op == FilterOperator.NIN:
+            return {"bool": {"must_not": {"terms": {key: filter.value}}}}
+        elif op == FilterOperator.ALL:
+            return {
+                "terms_set": {
+                    key: {
+                        "terms": filter.value,
+                        "minimum_should_match_script": {"source": "params.num_terms"},
+                    }
+                }
+            }
+        elif op == FilterOperator.TEXT_MATCH:
+            return {"match": {key: {"query": filter.value, "fuzziness": "AUTO"}}}
+        elif op == FilterOperator.CONTAINS:
+            return {"wildcard": {key: f"*{filter.value}*"}}
+        else:
+            raise ValueError(f"Unsupported filter operator: {filter.operator}")
+
+    def _parse_filters_recursively(self, filters: MetadataFilters) -> dict:
+        """Parse (possibly nested) MetadataFilters to equivalent OpenSearch expression."""
+        condition_map = {FilterCondition.AND: "must", FilterCondition.OR: "should"}
+
+        bool_clause = condition_map[filters.condition]
+        bool_query: dict[str, dict[str, list[dict]]] = {"bool": {bool_clause: []}}
+
+        for filter_item in filters.filters:
+            if isinstance(filter_item, MetadataFilter):
+                bool_query["bool"][bool_clause].append(self._parse_filter(filter_item))
+            elif isinstance(filter_item, MetadataFilters):
+                bool_query["bool"][bool_clause].append(
+                    self._parse_filters_recursively(filter_item)
+                )
+            else:
+                raise ValueError(f"Unsupported filter type: {type(filter_item)}")
+
+        return bool_query
+
+    def _parse_filters(self, filters: Optional[MetadataFilters]) -> List[dict]:
+        """Parse MetadataFilters to equivalent OpenSearch expression."""
+        if filters is None:
+            return []
+        return [self._parse_filters_recursively(filters=filters)]
 
     def _knn_search_query(
         self,
@@ -389,6 +468,35 @@ class OpensearchVectorClient:
         }
         await self._os_client.delete_by_query(index=self._index, body=search_query)
 
+    async def delete_nodes(
+        self,
+        node_ids: Optional[List[str]] = None,
+        filters: Optional[MetadataFilters] = None,
+        **delete_kwargs: Any,
+    ) -> None:
+        """Deletes nodes.
+
+        Args:
+            node_ids (Optional[List[str]], optional): IDs of nodes to delete. Defaults to None.
+            filters (Optional[MetadataFilters], optional): Metadata filters. Defaults to None.
+        """
+        if not node_ids and not filters:
+            return
+
+        query = {"query": {"bool": {"filter": []}}}
+        if node_ids:
+            query["query"]["bool"]["filter"].append({"terms": {"_id": node_ids or []}})
+
+        if filters:
+            query["query"]["bool"]["filter"].extend(self._parse_filters(filters))
+
+        await self._os_client.delete_by_query(index=self._index, body=query)
+
+    async def clear(self) -> None:
+        """Clears index."""
+        query = {"query": {"bool": {"filter": []}}}
+        await self._os_client.delete_by_query(index=self._index, body=query)
+
     async def aquery(
         self,
         query_mode: VectorStoreQueryMode,
@@ -574,6 +682,44 @@ class OpensearchVectorStore(BasePydanticVectorStore):
         """
         await self._client.delete_by_doc_id(ref_doc_id)
 
+    async def adelete_nodes(
+        self,
+        node_ids: Optional[List[str]] = None,
+        filters: Optional[MetadataFilters] = None,
+        **delete_kwargs: Any,
+    ) -> None:
+        """Deletes nodes async.
+
+        Args:
+            node_ids (Optional[List[str]], optional): IDs of nodes to delete. Defaults to None.
+            filters (Optional[MetadataFilters], optional): Metadata filters. Defaults to None.
+        """
+        await self._client.delete_nodes(node_ids, filters, **delete_kwargs)
+
+    def delete_nodes(
+        self,
+        node_ids: Optional[List[str]] = None,
+        filters: Optional[MetadataFilters] = None,
+        **delete_kwargs: Any,
+    ) -> None:
+        """Deletes nodes.
+
+        Args:
+            node_ids (Optional[List[str]], optional): IDs of nodes to delete. Defaults to None.
+            filters (Optional[MetadataFilters], optional): Metadata filters. Defaults to None.
+        """
+        asyncio.get_event_loop().run_until_complete(
+            self.adelete_nodes(node_ids, filters, **delete_kwargs)
+        )
+
+    async def aclear(self) -> None:
+        """Clears index."""
+        await self._client.clear()
+
+    def clear(self) -> None:
+        """Clears index."""
+        asyncio.get_event_loop().run_until_complete(self.aclear())
+
     def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult:
         """
         Query index for top k most similar nodes.

{llama_index_vector_stores_opensearch-0.1.12 → llama_index_vector_stores_opensearch-0.1.14}/pyproject.toml

@@ -27,7 +27,7 @@ exclude = ["**/BUILD"]
 license = "MIT"
 name = "llama-index-vector-stores-opensearch"
 readme = "README.md"
-version = "0.1.12"
+version = "0.1.14"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"