alita-sdk 0.3.208__tar.gz → 0.3.210__tar.gz

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.208
+Version: 0.3.210
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/clients/artifact.py

@@ -24,7 +24,14 @@ class Artifact:
             logger.error(f"Error: {e}")
             return f"Error: {e}"
     
-    def get(self, artifact_name: str, bucket_name: str = None, is_capture_image: bool = False, page_number: int = None, sheet_name: str = None):
+    def get(self,
+            artifact_name: str,
+            bucket_name: str = None,
+            is_capture_image: bool = False,
+            page_number: int = None,
+            sheet_name: str = None,
+            excel_by_sheets: bool = False,
+            llm = None):
         if not bucket_name:
             bucket_name = self.bucket_name
         data = self.client.download_artifact(bucket_name, artifact_name)
@@ -37,17 +44,24 @@ class Artifact:
         if detected['encoding'] is not None:
             return data.decode(detected['encoding'])
         else:
-            return parse_file_content(artifact_name, data, is_capture_image, page_number, sheet_name)
+            return parse_file_content(file_name=artifact_name,
+                                  file_content=data,
+                                  is_capture_image=is_capture_image,
+                                  page_number=page_number,
+                                  sheet_name=sheet_name,
+                                  excel_by_sheets=excel_by_sheets,
+                                  llm=llm)
 
     def delete(self, artifact_name: str, bucket_name = None):
         if not bucket_name:
             bucket_name = self.bucket_name
         self.client.delete_artifact(bucket_name, artifact_name)
     
-    def list(self, bucket_name: str = None) -> str:
+    def list(self, bucket_name: str = None, return_as_string = True) -> str|dict:
         if not bucket_name:
             bucket_name = self.bucket_name
-        return str(self.client.list_artifacts(bucket_name))
+        artifacts = self.client.list_artifacts(bucket_name)
+        return str(artifacts) if return_as_string else artifacts
 
     def append(self, artifact_name: str, additional_data: Any, bucket_name: str = None):
         if not bucket_name:

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/langchain/document_loaders/AlitaCSVLoader.py

@@ -26,7 +26,8 @@ class AlitaCSVLoader(AlitaTableLoader):
                  json_documents: bool = True,
                  raw_content: bool = False,
                  columns: Optional[List[str]] = None,
-                 cleanse: bool = True):
+                 cleanse: bool = True,
+                 **kwargs):
         super().__init__(file_path=file_path, json_documents=json_documents, columns=columns, raw_content=raw_content, cleanse=cleanse)
         self.encoding = encoding
         self.autodetect_encoding = autodetect_encoding

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/langchain/document_loaders/AlitaDocxMammothLoader.py

@@ -18,19 +18,19 @@ class AlitaDocxMammothLoader(BaseLoader):
     Loader for Docx files using Mammoth to convert to HTML, with image handling,
     and then Markdownify to convert HTML to markdown.
     """
-    def __init__(self, path: str, **kwargs):
+    def __init__(self, file_path: str, **kwargs):
         """
         Initializes AlitaDocxMammothLoader.
 
         Args:
             **kwargs: Keyword arguments, including:
-                path (str): Path to the Docx file. Required.
+                file_path (str): Path to the Docx file. Required.
                 llm (LLM, optional): Language model for processing images.
                 prompt (str, optional): Prompt for the language model.
         Raises:
             ValueError: If the 'path' parameter is not provided.
         """
-        self.path = path
+        self.path = file_path
         self.llm = kwargs.get("llm")
         self.prompt = kwargs.get("prompt")
 

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/langchain/document_loaders/AlitaImageLoader.py

@@ -19,11 +19,15 @@ Image.MAX_IMAGE_PIXELS = 300_000_000
 class AlitaImageLoader(BaseLoader):
     """Loads image files using pytesseract for OCR or optionally LLM for advanced analysis, including SVG support."""
 
-    def __init__(self, **kwargs):
-        if not kwargs.get('path'):
-            raise ValueError("Path parameter 'path' is required")
-        else:
+    def __init__(self, file_path=None, **kwargs):
+        # Handle both positional and keyword arguments for file_path
+        if file_path is not None:
+            self.file_path = file_path
+        elif kwargs.get('path'):
             self.file_path = kwargs['path']
+        else:
+            raise ValueError(
+                "Path parameter is required (either as 'file_path' positional argument or 'path' keyword argument)")
         self.llm = kwargs.get('llm', None)
         self.ocr_language = kwargs.get('ocr_language', None)
         self.prompt = kwargs.get('prompt') if kwargs.get(

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/langchain/document_loaders/AlitaTableLoader.py

@@ -26,7 +26,7 @@ class AlitaTableLoader(BaseLoader):
                  json_documents: bool = True,
                  raw_content: bool = False,
                  columns: Optional[List[str]] = None,
-                 cleanse: bool = True):
+                 cleanse: bool = True, **kwargs):
 
         self.raw_content = raw_content
         self.file_path = file_path

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/langchain/langraph_agent.py

@@ -9,7 +9,7 @@ from langchain_core.callbacks import dispatch_custom_event
 from langchain_core.messages import HumanMessage, AIMessage, SystemMessage, BaseMessage
 from langchain_core.runnables import Runnable
 from langchain_core.runnables import RunnableConfig
-from langchain_core.tools import BaseTool
+from langchain_core.tools import BaseTool, ToolException
 from langgraph.channels.ephemeral_value import EphemeralValue
 from langgraph.graph import StateGraph
 from langgraph.graph.graph import END, START
@@ -505,11 +505,11 @@ def create_graph(
                 if isinstance(connected_tools, dict):
                     for toolkit, selected_tools in connected_tools.items():
                         for tool in selected_tools:
-                            tool_names.append(f"{toolkit}___{tool}")
+                            tool_names.append(f"{toolkit}{TOOLKIT_SPLITTER}{tool}")
+                elif isinstance(connected_tools, list):
+                    # for cases when tools are provided as a list of names with already bound toolkit_name
+                    tool_names = connected_tools
                 
-                # Filter tools if specific tool names are provided
-                available_tools = []
-            
                 if tool_names:
                     # Filter tools by name
                     tool_dict = {tool.name: tool for tool in tools if isinstance(tool, BaseTool)}
@@ -580,7 +580,10 @@ def create_graph(
                     default_output=node['condition'].get('default_output', 'END')))
 
         # set default value for state variable at START
-        entry_point = clean_string(schema['entry_point'])
+        try:
+            entry_point = clean_string(schema['entry_point'])
+        except KeyError:
+            raise ToolException("Entry point is not defined in the schema. Please define 'entry_point' in the schema.")
         for key, value in state.items():
             if 'type' in value and 'value' in value:
                 # set default value for state variable if it is defined in the schema

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/toolkits/artifact.py

@@ -3,7 +3,7 @@ from typing import List, Any, Literal, Optional
 from alita_sdk.tools.utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
 from langchain_community.agent_toolkits.base import BaseToolkit
 from langchain_core.tools import BaseTool
-from pydantic import create_model, BaseModel, ConfigDict, Field
+from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
 from pydantic.fields import FieldInfo
 from ..tools.artifact import ArtifactWrapper
 from alita_sdk.tools.base.tool import BaseAction
@@ -22,15 +22,19 @@ class ArtifactToolkit(BaseToolkit):
             # client = (Any, FieldInfo(description="Client object", required=True, autopopulate=True)),
             bucket = (str, FieldInfo(description="Bucket name", json_schema_extra={'toolkit_name': True, 'max_toolkit_length': ArtifactToolkit.toolkit_max_length})),
             selected_tools=(List[Literal[tuple(selected_tools)]], Field(default=[], json_schema_extra={'args_schemas': selected_tools})),
+            # indexer settings
+            connection_string = (Optional[SecretStr], Field(description="Connection string for vectorstore",
+                                                            default=None,
+                                                            json_schema_extra={'secret': True})),
             __config__=ConfigDict(json_schema_extra={'metadata': {"label": "Artifact", "icon_url": None}})
         )
     
     @classmethod
-    def get_toolkit(cls, client: Any, bucket: str, toolkit_name: Optional[str] = None, selected_tools: list[str] = []):
+    def get_toolkit(cls, client: Any, bucket: str, toolkit_name: Optional[str] = None, selected_tools: list[str] = [], **kwargs):
         if selected_tools is None:
             selected_tools = []
         tools = []
-        artifact_wrapper = ArtifactWrapper(client=client, bucket=bucket)
+        artifact_wrapper = ArtifactWrapper(client=client, bucket=bucket, **kwargs)
         prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         available_tools = artifact_wrapper.get_available_tools()
         for tool in available_tools:

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/toolkits/tools.py

@@ -71,7 +71,14 @@ def get_tools(tools_list: list, alita_client, llm, memory_store: BaseStore = Non
                 client=alita_client,
                 bucket=tool['settings']['bucket'],
                 toolkit_name=tool.get('toolkit_name', ''),
-                selected_tools=tool['settings'].get('selected_tools', [])
+                selected_tools=tool['settings'].get('selected_tools', []),
+                llm=tool['settings'].get('llm'),
+                # indexer settings
+                connection_string=tool['settings'].get('connection_string', None),
+                collection_name=f"{tool.get('toolkit_name')}_{str(tool['id'])}",
+                embedding_model="HuggingFaceEmbeddings",
+                embedding_model_params={"model_name": "sentence-transformers/all-MiniLM-L6-v2"},
+                vectorstore_type="PGVector"
             ).get_tools())
         elif tool['type'] == 'vectorstore':
             tools.extend(VectorStoreToolkit.get_toolkit(

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/tools/application.py

@@ -56,6 +56,8 @@ class Application(BaseTool):
         schema_values = self.args_schema(**input).model_dump() if self.args_schema else {}
         extras = {k: v for k, v in input.items() if k not in schema_values}
         all_kwargs = {**kwargs, **extras, **schema_values}
+        if config is None:
+            config = {}
         return self._run(*config, **all_kwargs)
 
     def _run(self, *args, **kwargs):

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/tools/artifact.py

@@ -1,8 +1,19 @@
-from alita_sdk.tools.elitea_base import BaseToolApiWrapper
-from typing import Any, Optional
+import hashlib
+import json
+from typing import Any, Optional, Generator, List
+
+from langchain_core.documents import Document
+from langchain_core.tools import ToolException
 from pydantic import create_model, Field, model_validator
 
-class ArtifactWrapper(BaseToolApiWrapper):
+from alita_sdk.tools.elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
+
+try:
+    from alita_sdk.runtime.langchain.interfaces.llm_processor import get_embeddings
+except ImportError:
+    from alita_sdk.langchain.interfaces.llm_processor import get_embeddings
+
+class ArtifactWrapper(BaseVectorStoreToolApiWrapper):
     client: Any
     bucket: str
     artifact: Optional[Any] = None
@@ -17,14 +28,26 @@ class ArtifactWrapper(BaseToolApiWrapper):
         values["artifact"] = values['client'].artifact(values['bucket'])
         return values
 
-    def list_files(self, bucket_name = None):
-        return self.artifact.list(bucket_name)
+    def list_files(self, bucket_name = None, return_as_string = True):
+        return self.artifact.list(bucket_name, return_as_string)
 
     def create_file(self, filename: str, filedata: str, bucket_name = None):
         return self.artifact.create(filename, filedata, bucket_name)
 
-    def read_file(self, filename: str, bucket_name = None, is_capture_image: bool = False, page_number: int = None, sheet_name: str = None):
-        return self.artifact.get(filename, bucket_name, is_capture_image, page_number, sheet_name)
+    def read_file(self,
+                  filename: str,
+                  bucket_name = None,
+                  is_capture_image: bool = False,
+                  page_number: int = None,
+                  sheet_name: str = None,
+                  excel_by_sheets: bool = False):
+        return self.artifact.get(artifact_name=filename,
+                                 bucket_name=bucket_name,
+                                  is_capture_image=is_capture_image,
+                                  page_number=page_number,
+                                  sheet_name=sheet_name,
+                                  excel_by_sheets=excel_by_sheets,
+                                  llm=self.llm)
 
     def delete_file(self, filename: str, bucket_name = None):
         return self.artifact.delete(filename, bucket_name)
@@ -38,6 +61,40 @@ class ArtifactWrapper(BaseToolApiWrapper):
     def create_new_bucket(self, bucket_name: str, expiration_measure = "weeks", expiration_value = 1):
         return self.artifact.client.create_bucket(bucket_name, expiration_measure, expiration_value)
 
+    def _base_loader(self, **kwargs) -> List[Document]:
+        try:
+            all_files = self.list_files(self.bucket, False)
+        except Exception as e:
+            raise ToolException(f"Unable to extract files: {e}")
+
+        docs: List[Document] = []
+        for file in all_files['rows']:
+            metadata = {
+                ("updated_on" if k == "modified" else k): str(v)
+                for k, v in file.items()
+            }
+            metadata['id'] = self.get_hash_from_bucket_and_file_name(self.bucket, file['name'])
+            docs.append(Document(page_content="", metadata=metadata))
+        return docs
+
+    def get_hash_from_bucket_and_file_name(self, bucket, file_name):
+        hasher = hashlib.sha256()
+        hasher.update(bucket.encode('utf-8'))
+        hasher.update(file_name.encode('utf-8'))
+        return hasher.hexdigest()
+
+    def _process_document(self, document: Document) -> Generator[Document, None, None]:
+        page_content = self.read_file(document.metadata['name'], is_capture_image=True, excel_by_sheets=True)
+        if isinstance(page_content, dict):
+            for key, value in page_content.items():
+                metadata = document.metadata
+                metadata['page'] = key
+                yield Document(page_content=str(value), metadata=metadata)
+        else:
+            document.page_content = json.dumps(str(page_content))
+            yield document
+
+    @extend_with_vector_tools
     def get_available_tools(self):
         bucket_name = (Optional[str], Field(description="Name of the bucket to work with."
                                                         "If bucket is not specified by user directly, the name should be taken from chat history."
@@ -125,5 +182,5 @@ class ArtifactWrapper(BaseToolApiWrapper):
                                                              default="weeks")),
                     expiration_value=(Optional[int], Field(description="Expiration time values.", default=1))
                 )
-            },
+            }
         ]

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/tools/vectorstore.py

@@ -188,63 +188,108 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             except Exception as e:
                 logger.error(f"Failed to initialize PGVectorSearch: {str(e)}")
 
+    def _clean_collection(self):
+        """
+        Clean the vectorstore collection by deleting all indexed data.
+        """
+        self._log_data(
+            f"Cleaning collection '{self.dataset}'",
+            tool_name="_clean_collection"
+        )
+        data = self.vectoradapter.vectorstore.get(include=['metadatas'])
+        self.vectoradapter.vectorstore.delete(ids=data['ids'])
+        self._log_data(
+            f"Collection '{self.dataset}' has been cleaned. ",
+            tool_name="_clean_collection"
+        )
+
     def _get_indexed_data(self, store):
-        """ Get all indexed data from vectorstore """
+        """ Get all indexed data from vectorstore for non-code content """
 
         # get already indexed data
         result = {}
         try:
             self._log_data("Retrieving already indexed data from vectorstore",
                            tool_name="index_documents")
-            data = store.get(include=['documents', 'metadatas'])
+            data = store.get(include=['metadatas'])
             # re-structure data to be more usable
-            for doc_str, meta, db_id in zip(data['documents'], data['metadatas'], data['ids']):
-                doc = json.loads(doc_str)
+            for meta, db_id in zip(data['metadatas'], data['ids']):
+                # get document id from metadata
                 doc_id = str(meta['id'])
                 dependent_docs = meta.get(IndexerKeywords.DEPENDENT_DOCS.value, [])
+                if dependent_docs:
+                    dependent_docs = [d.strip() for d in dependent_docs.split(';') if d.strip()]
                 parent_id = meta.get(IndexerKeywords.PARENT.value, -1)
-                result[doc_id] = {
-                    'metadata': meta,
-                    'document': doc,
-                    'id': db_id,
-                    IndexerKeywords.DEPENDENT_DOCS.value: dependent_docs,
-                    IndexerKeywords.PARENT.value: parent_id
-                }
+                #
+                chunk_id = meta.get('chunk_id')
+                if doc_id in result and chunk_id:
+                    # if document with the same id already saved, add db_id fof current one as chunk
+                    result[doc_id]['all_chunks'].append(db_id)
+                else:
+                    result[doc_id] = {
+                        'metadata': meta,
+                        'id': db_id,
+                        'all_chunks': [db_id],
+                        IndexerKeywords.DEPENDENT_DOCS.value: dependent_docs,
+                        IndexerKeywords.PARENT.value: parent_id
+                    }
         except Exception as e:
             logger.error(f"Failed to get indexed data from vectorstore: {str(e)}. Continuing with empty index.")
         return result
 
-    def _reduce_duplicates(self, documents: Generator[Document, None, None], store) -> List[Any]:
-        """Remove documents already indexed in the vectorstore based on metadata 'id' and 'updated_on' fields."""
+    def _get_code_indexed_data(self, store) -> Dict[str, Dict[str, Any]]:
+        """ Get all indexed data from vectorstore for code content """
 
-        self._log_data("Verification of documents to index started", tool_name="index_documents")
+        # get already indexed data
+        result = {}
+        try:
+            self._log_data("Retrieving already indexed code data from vectorstore",
+                           tool_name="index_documents")
+            data = store.get(include=['metadatas'])
+            # re-structure data to be more usable
+            for meta, db_id in zip(data['metadatas'], data['ids']):
+                filename = meta['filename']
+                commit_hash = meta.get('commit_hash')
+                if filename not in result:
+                    result[filename] = {
+                        'commit_hashes': [],
+                        'ids': []
+                    }
+                if commit_hash is not None:
+                    result[filename]['commit_hashes'].append(commit_hash)
+                result[filename]['ids'].append(db_id)
+        except Exception as e:
+            logger.error(f"Failed to get indexed code data from vectorstore: {str(e)}. Continuing with empty index.")
+        return result
 
-        indexed_data = self._get_indexed_data(store)
-        indexed_ids = set(indexed_data.keys())
-        if not indexed_ids:
+    def _reduce_duplicates(
+            self,
+            documents: Generator[Any, None, None],
+            store,
+            get_indexed_data: Callable,
+            key_fn: Callable,
+            compare_fn: Callable,
+            remove_ids_fn: Callable,
+            log_msg: str = "Verification of documents to index started"
+    ) -> List[Any]:
+        """Generic duplicate reduction logic for documents."""
+        self._log_data(log_msg, tool_name="index_documents")
+        indexed_data = get_indexed_data(store)
+        indexed_keys = set(indexed_data.keys())
+        if not indexed_keys:
             self._log_data("Vectorstore is empty, indexing all incoming documents", tool_name="index_documents")
             return list(documents)
 
         final_docs = []
-        docs_to_remove = []
+        docs_to_remove = set()
 
         for document in documents:
-            doc_id = document.metadata.get('id')
-            # get document's metadata and id and check if already indexed
-            if doc_id in indexed_ids:
-                # document has been indexed already, then verify `updated_on`
-                to_index_updated_on = document.metadata.get('updated_on')
-                indexed_meta = indexed_data[doc_id]['metadata']
-                indexed_updated_on = indexed_meta.get('updated_on')
-                if to_index_updated_on and indexed_updated_on and to_index_updated_on == indexed_updated_on:
-                    # same updated_on, skip indexing
+            key = key_fn(document)
+            if key in indexed_keys:
+                if compare_fn(document, indexed_data[key]):
                     continue
-                # if updated_on is missing or different, we will re-index the document and remove old one
-                # parent doc removal
-                docs_to_remove.append(indexed_data[doc_id]['id'])
-                # mark dependent docs for removal
-                for dependent_doc_id in indexed_data[doc_id][IndexerKeywords.DEPENDENT_DOCS.value]:
-                    docs_to_remove.append(indexed_data[dependent_doc_id]['id'])
+                final_docs.append(document)
+                docs_to_remove.update(remove_ids_fn(indexed_data, key))
             else:
                 final_docs.append(document)
 
@@ -253,16 +298,50 @@ class VectorStoreWrapper(BaseToolApiWrapper):
                 f"Removing {len(docs_to_remove)} documents from vectorstore that are already indexed with different updated_on.",
                 tool_name="index_documents"
             )
-            store.delete(ids=docs_to_remove)
+            store.delete(ids=list(docs_to_remove))
 
         return final_docs
 
-    def index_documents(self, documents: Generator[Document, None, None], progress_step: int = 20, clean_index: bool = True):
+    def _reduce_non_code_duplicates(self, documents: Generator[Any, None, None], store) -> List[Any]:
+        return self._reduce_duplicates(
+            documents,
+            store,
+            self._get_indexed_data,
+            lambda doc: doc.metadata.get('id'),
+            lambda doc, idx: (
+                    doc.metadata.get('updated_on') and
+                    idx['metadata'].get('updated_on') and
+                    doc.metadata.get('updated_on') == idx['metadata'].get('updated_on')
+            ),
+            lambda idx_data, key: (
+                    idx_data[key]['all_chunks'] +
+                    [idx_data[dep_id]['id'] for dep_id in idx_data[key][IndexerKeywords.DEPENDENT_DOCS.value]] +
+                    [chunk_db_id for dep_id in idx_data[key][IndexerKeywords.DEPENDENT_DOCS.value]
+                     for chunk_db_id in idx_data[dep_id]['all_chunks']]
+            ),
+            log_msg="Verification of documents to index started"
+        )
+
+    def _reduce_code_duplicates(self, documents: Generator[Any, None, None], store) -> List[Any]:
+        return self._reduce_duplicates(
+            documents,
+            store,
+            self._get_code_indexed_data,
+            lambda doc: doc.metadata.get('filename'),
+            lambda doc, idx: (
+                    doc.metadata.get('commit_hash') and
+                    idx.get('commit_hashes') and
+                    doc.metadata.get('commit_hash') in idx.get('commit_hashes')
+            ),
+            lambda idx_data, key: idx_data[key]['ids'],
+            log_msg="Verification of code documents to index started"
+        )
+
+    def index_documents(self, documents: Generator[Document, None, None], progress_step: int = 20, clean_index: bool = True, is_code: bool = False):
         """ Index documents in the vectorstore.
 
         Args:
             documents (Any): Generator or list of documents to index.
-            document_processing_func (Optional[Callable]): Function to process documents after duplicates removal and before indexing.
             progress_step (int): Step for progress reporting, default is 20.
             clean_index (bool): If True, clean the index before re-indexing all documents.
         """
@@ -274,7 +353,7 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             logger.info("Cleaning index before re-indexing all documents.")
             self._log_data("Cleaning index before re-indexing all documents. Previous index will be removed", tool_name="index_documents")
             try:
-                self.vectoradapter.delete_dataset(self.dataset)
+                self._clean_collection()
                 self.vectoradapter.persist()
                 self.vectoradapter.vacuum()
                 self._log_data("Previous index has been removed",
@@ -284,9 +363,9 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             if isinstance(documents, types.GeneratorType):
                 documents = list(documents)
         else:
-            # remove duplicates based on metadata 'id' and 'updated_on' fields
-            documents = self._reduce_duplicates(documents, self.vectoradapter.vectorstore)
-
+            # remove duplicates based on metadata 'id' and 'updated_on' or 'commit_hash' fields
+            documents = self._reduce_code_duplicates(documents, self.vectoradapter.vectorstore) if is_code \
+                else self._reduce_non_code_duplicates(documents, self.vectoradapter.vectorstore)
 
         if not documents or len(documents) == 0:
             logger.info("No new documents to index after duplicate check.")
@@ -456,8 +535,12 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             )
             
         # Initialize document map for tracking by ID
-        doc_map = {doc.metadata.get('id', f"idx_{i}"): (doc, score) 
-                  for i, (doc, score) in enumerate(vector_items)}
+        doc_map = {
+            f"{doc.metadata.get('id', f'idx_{i}')}_{doc.metadata['chunk_id']}"
+            if 'chunk_id' in doc.metadata
+            else doc.metadata.get('id', f"idx_{i}"): (doc, score)
+            for i, (doc, score) in enumerate(vector_items)
+        }
         
         # Process full-text search if configured
         if full_text_search and full_text_search.get('enabled') and full_text_search.get('fields'):

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/runtime/utils/utils.py

@@ -4,8 +4,11 @@ from enum import Enum
 TOOLKIT_SPLITTER = "___"
 
 class IndexerKeywords(Enum):
+    # TODO: remove these fields when the indexer is updated
     DEPENDENT_DOCS = 'dependent_docs'
     PARENT = 'parent_id'
+    # DEPENDENCY_ID = 'dependency_id'
+    UPDATED_ON = 'updated_on'
 
 # This pattern matches characters that are NOT alphanumeric, underscores, or hyphens
 clean_string_pattern = re.compile(r'[^a-zA-Z0-9_.-]')

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/tools/ado/__init__.py

@@ -14,6 +14,14 @@ def get_tools(tool_type, tool):
         "token": tool['settings'].get('token', None),
         "limit": tool['settings'].get('limit', 5),
         "toolkit_name": tool.get('toolkit_name', ''),
+        # indexer settings
+        "llm":tool['settings'].get('llm', None),
+        "connection_string":tool['settings'].get('connection_string', None),
+        "collection_name":str(tool['id']),
+        "doctype":'doc',
+        "embedding_model":"HuggingFaceEmbeddings",
+        "embedding_model_params":{"model_name": "sentence-transformers/all-MiniLM-L6-v2"},
+        "vectorstore_type":"PGVector"
     }
     if tool_type == 'ado_plans':
         return AzureDevOpsPlansToolkit().get_toolkit(**config_dict).get_tools()

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/tools/ado/repos/repos_wrapper.py

@@ -302,6 +302,43 @@ class ReposApiWrapper(BaseCodeToolApiWrapper):
 
         return values
 
+    def _get_commits(self, file_path: str, branch: str, top: int = None) -> List[GitCommitRef]:
+        """
+        Get commits for a specific file in a specific branch.
+
+        Args:
+            file_path (str): Path to the file in the repository.
+            branch (str): Branch name to get commits from.
+            top (int, optional): Maximum number of commits to return. Defaults to None.
+
+        Returns:
+            List[GitCommitRef]: List of commit references.
+        """
+        try:
+            version_descriptor = GitVersionDescriptor(
+                version=branch, version_type="branch"
+            )
+            commits = self._client.get_commits(
+                repository_id=self.repository_id,
+                project=self.project,
+                search_criteria=GitQueryCommitsCriteria(item_path=file_path,
+                                                       item_version=version_descriptor, top=top if top else 100),
+            )
+            return commits
+        except Exception as e:
+            msg = f"Failed to get commits for file '{file_path}' on branch '{branch}': {str(e)}"
+            logger.error(msg)
+            return ToolException(msg)
+
+    def _file_commit_hash(self, file_path: str, branch: str) -> str:
+        """Get the commit hash of the last commit that modified a file in a specific branch."""
+
+        commits = self._get_commits(file_path, branch, top=1)
+        if commits:
+            return commits[0].commit_id
+        else:
+            return None
+
     def _get_files(
             self,
             path: str = "",

{alita_sdk-0.3.208 → alita_sdk-0.3.210}/alita_sdk/tools/ado/test_plan/test_plan_wrapper.py

@@ -184,13 +184,6 @@ class TestPlanApiWrapper(BaseVectorStoreToolApiWrapper):
     limit: Optional[int] = 5
     _client: Optional[TestPlanClient] = PrivateAttr()
 
-    llm: Any = None
-    connection_string: Optional[SecretStr] = None
-    collection_name: Optional[str] = None
-    embedding_model: Optional[str] = "HuggingFaceEmbeddings"
-    embedding_model_params: Optional[Dict[str, Any]] = {"model_name": "sentence-transformers/all-MiniLM-L6-v2"}
-    vectorstore_type: Optional[str] = "PGVector"
-
     class Config:
         arbitrary_types_allowed = True