alita-sdk 0.3.370__tar.gz → 0.3.371__tar.gz

{alita_sdk-0.3.370 → alita_sdk-0.3.371}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.370
+Version: 0.3.371
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.370 → alita_sdk-0.3.371}/alita_sdk/runtime/tools/vectorstore_base.py

@@ -185,6 +185,12 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
             return "No indexed collections"
         return collections
 
+    def get_index_meta(self, collection_suffix: str):
+        index_metas = self.vector_adapter.get_index_meta(self, collection_suffix)
+        if len(index_metas) > 1:
+            raise RuntimeError(f"Multiple index_meta documents found: {index_metas}")
+        return index_metas[0] if index_metas else None
+
     def _clean_collection(self, collection_suffix: str = ''):
         """
         Clean the vectorstore collection by deleting all indexed data.

{alita_sdk-0.3.370 → alita_sdk-0.3.371}/alita_sdk/tools/base_indexer_toolkit.py

@@ -1,3 +1,4 @@
+import copy
 import json
 import logging
 import time
@@ -148,7 +149,6 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         yield from ()
 
     def index_data(self, **kwargs):
-        from ..runtime.langchain.interfaces.llm_processor import add_documents
         collection_suffix = kwargs.get("collection_suffix")
         progress_step = kwargs.get("progress_step")
         clean_index = kwargs.get("clean_index")
@@ -158,17 +158,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         if clean_index:
             self._clean_index(collection_suffix)
         #
-        # create and add initial index meta document
-        index_meta_doc = Document(page_content=f"{IndexerKeywords.INDEX_META_TYPE.value}_{collection_suffix}", metadata={
-            "collection": collection_suffix,
-            "type": IndexerKeywords.INDEX_META_TYPE.value,
-            "indexed": 0,
-            "state": IndexerKeywords.INDEX_META_IN_PROGRESS.value,
-            "index_configuration": kwargs,
-            "created_on": time.time(),
-            "updated_on": time.time(),
-        })
-        index_meta_ids = add_documents(vectorstore=self.vectorstore, documents=[index_meta_doc])
+        self.index_meta_init(collection_suffix, kwargs)
         #
         self._log_tool_event(f"Indexing data into collection with suffix '{collection_suffix}'. It can take some time...")
         self._log_tool_event(f"Loading the documents to index...{kwargs}")
@@ -183,11 +173,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
                              f"Processing documents to collect dependencies and prepare them for indexing...")
         result = self._save_index_generator(documents, documents_count, chunking_tool, chunking_config, collection_suffix=collection_suffix, progress_step=progress_step)
         #
-        # update index meta document
-        index_meta_doc.metadata["indexed"] = result
-        index_meta_doc.metadata["state"] = IndexerKeywords.INDEX_META_COMPLETED.value
-        index_meta_doc.metadata["updated_on"] = time.time()
-        add_documents(vectorstore=self.vectorstore, documents=[index_meta_doc], ids=index_meta_ids)
+        self.index_meta_update(collection_suffix, IndexerKeywords.INDEX_META_COMPLETED.value, result)
         #
         return {"status": "ok", "message": f"successfully indexed {result} documents"}
 
@@ -377,30 +363,6 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             }
         return filter
 
-    def index_meta_read(self):
-        from sqlalchemy import func
-        from sqlalchemy.orm import Session
-
-        store = self.vectorstore
-        try:
-            with Session(store.session_maker.bind) as session:
-                meta = session.query(
-                        store.EmbeddingStore.id,
-                        store.EmbeddingStore.cmetadata
-                    ).filter(
-                        func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'type') == IndexerKeywords.INDEX_META_TYPE.value
-                    ).all()
-                return [
-                    {"id": id_, "metadata": cmetadata}
-                    for id_, cmetadata in meta
-                ]
-        except Exception as e:
-            logger.error(f"Failed to get index_meta from PGVector: {str(e)}")
-            return []
-
-    def index_meta_delete(self, index_meta_ids: list[str]):
-        self.vectorstore.delete(ids=index_meta_ids)
-
     def search_index(self,
                      query: str,
                      collection_suffix: str = "",
@@ -480,6 +442,51 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             reranking_config=reranking_config, 
             extended_search=extended_search
         )
+    
+    def index_meta_init(self, collection_suffix: str, index_configuration: dict[str, Any]):
+        index_meta_raw = super().get_index_meta(collection_suffix)
+        from ..runtime.langchain.interfaces.llm_processor import add_documents
+        created_on = time.time()
+        metadata = {
+            "collection": collection_suffix,
+            "type": IndexerKeywords.INDEX_META_TYPE.value,
+            "indexed": 0,
+            "state": IndexerKeywords.INDEX_META_IN_PROGRESS.value,
+            "index_configuration": index_configuration,
+            "created_on": created_on,
+            "updated_on": created_on,
+            "history": "[]",
+        }
+        index_meta_ids = None
+        #
+        if index_meta_raw:
+            history_raw = index_meta_raw.get("metadata", {}).get("history", "[]")
+            if isinstance(history_raw, str) and history_raw.strip():
+                try:
+                    history = json.loads(history_raw)
+                except (json.JSONDecodeError, TypeError):
+                    history = []
+            else:
+                history = []
+            new_history_item = {k: v for k, v in index_meta_raw.get("metadata", {}).items() if k != "history"}
+            history.append(new_history_item)
+            metadata["history"] = json.dumps(history)
+            index_meta_ids = [index_meta_raw.get("id")]
+        #
+        index_meta_doc = Document(page_content=f"{IndexerKeywords.INDEX_META_TYPE.value}_{collection_suffix}", metadata=metadata)
+        add_documents(vectorstore=self.vectorstore, documents=[index_meta_doc], ids=index_meta_ids)
+
+    def index_meta_update(self, collection_suffix: str, state: str, result: int):
+        index_meta_raw = super().get_index_meta(collection_suffix)
+        from ..runtime.langchain.interfaces.llm_processor import add_documents
+        #
+        if index_meta_raw:
+            metadata = copy.deepcopy(index_meta_raw.get("metadata", {}))
+            metadata["indexed"] = result
+            metadata["state"] = state
+            metadata["updated_on"] = time.time()
+            index_meta_doc = Document(page_content=index_meta_raw.get("content", ""), metadata=metadata)
+            add_documents(vectorstore=self.vectorstore, documents=[index_meta_doc], ids=[index_meta_raw.get("id")])
 
     def get_available_tools(self):
         """

{alita_sdk-0.3.370 → alita_sdk-0.3.371}/alita_sdk/tools/vector_adapters/VectorStoreAdapter.py

@@ -2,6 +2,8 @@ from abc import ABC, abstractmethod
 from typing import Any, Dict, Optional, List
 from logging import getLogger
 
+from ...runtime.utils.utils import IndexerKeywords
+
 logger = getLogger(__name__)
 
 
@@ -48,6 +50,11 @@ class VectorStoreAdapter(ABC):
         """Add a new collection name to the metadata"""
         pass
 
+    @abstractmethod
+    def get_index_meta(self, vectorstore_wrapper, collection_suffix: str) -> List[Dict[str, Any]]:
+        """Get all index_meta entries from the vector store."""
+        pass
+
 
 class PGVectorAdapter(VectorStoreAdapter):
     """Adapter for PGVector database operations."""
@@ -265,6 +272,29 @@ class PGVectorAdapter(VectorStoreAdapter):
         except Exception as e:
             logger.error(f"Failed to update collection for entry ID {entry_id}: {str(e)}")
 
+    def get_index_meta(self, vectorstore_wrapper, collection_suffix: str) -> List[Dict[str, Any]]:
+        from sqlalchemy.orm import Session
+        from sqlalchemy import func
+
+        store = vectorstore_wrapper.vectorstore
+        try:
+            with Session(store.session_maker.bind) as session:
+                meta = session.query(
+                    store.EmbeddingStore.id,
+                    store.EmbeddingStore.document,
+                    store.EmbeddingStore.cmetadata
+                ).filter(
+                    store.EmbeddingStore.cmetadata['type'].astext == IndexerKeywords.INDEX_META_TYPE.value,
+                    func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == collection_suffix
+                ).all()
+                result = []
+                for id, document, cmetadata in meta:
+                    result.append({"id": id, "content": document, "metadata": cmetadata})
+                return result
+        except Exception as e:
+            logger.error(f"Failed to get index_meta from PGVector: {str(e)}")
+            raise e
+
 
 class ChromaAdapter(VectorStoreAdapter):
     """Adapter for Chroma database operations."""
@@ -361,6 +391,9 @@ class ChromaAdapter(VectorStoreAdapter):
         # This is a simplified implementation - in practice, you might need more complex logic
         logger.warning("add_to_collection for Chroma is not fully implemented yet")
 
+    def get_index_meta(self, vectorstore_wrapper, collection_suffix: str) -> List[Dict[str, Any]]:
+        logger.warning("get_index_meta for Chroma is not implemented yet")
+
 
 class VectorStoreAdapterFactory:
     """Factory for creating vector store adapters."""

{alita_sdk-0.3.370 → alita_sdk-0.3.371}/alita_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.370
+Version: 0.3.371
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.370 → alita_sdk-0.3.371}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "alita_sdk"
-version = "0.3.370"
+version = "0.3.371"
 description = "SDK for building langchain agents using resources from Alita"
 readme = "README.md"
 requires-python = ">=3.10"