alita-sdk 0.3.254__tar.gz → 0.3.255__tar.gz

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.254
+Version: 0.3.255
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk/configurations/slack.py

@@ -27,6 +27,6 @@ class SlackConfiguration(BaseModel):
             }
         }
     )
-    name: Optional[SecretStr] = Field(description="Slack Bot Token")
+    name: Optional[str] = Field(description="Slack name")
     slack_token: Optional[SecretStr] = Field(description="Slack Token like XOXB-*****-*****-*****-*****")
     channel_id:Optional[str] = Field(description="Channel ID")

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk/runtime/langchain/document_loaders/AlitaDocxMammothLoader.py

@@ -9,6 +9,7 @@ from langchain_core.documents import Document
 from mammoth import convert_to_html
 from markdownify import markdownify
 
+from alita_sdk.tools.chunkers.sematic.markdown_chunker import markdown_by_headers_chunker
 from .utils import perform_llm_prediction_for_image_bytes
 
 
@@ -35,6 +36,7 @@ class AlitaDocxMammothLoader(BaseLoader):
         self.extract_images = kwargs.get('extract_images')
         self.llm = kwargs.get("llm")
         self.prompt = kwargs.get("prompt")
+        self.max_tokens = kwargs.get('max_tokens', 512)
 
     def __handle_image(self, image) -> dict:
         """
@@ -100,11 +102,11 @@ class AlitaDocxMammothLoader(BaseLoader):
         Loads and converts the Docx file to markdown format.
 
         Returns:
-            List[Document]: A list containing a single Document with the markdown content
+            List[Document]: A list containing a Documents with the markdown content
                           and metadata including the source file path.
         """
         result_content = self.get_content()
-        return [Document(page_content=result_content, metadata={'source': str(self.path)})]
+        return list(markdown_by_headers_chunker(iter([Document(page_content=result_content, metadata={'source': str(self.path)})]), config={'max_tokens':self.max_tokens}))
 
     def get_content(self):
         """

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk/runtime/langchain/document_loaders/constants.py

@@ -123,7 +123,9 @@ loaders_map = {
     '.docx': {
         'class': AlitaDocxMammothLoader,
         'is_multimodal_processing': True,
-        'kwargs': {}
+        'kwargs': {
+            'extract_images': True
+        }
     },
     '.doc': {
         'class': AlitaTextLoader,

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk/runtime/tools/vectorstore.py

@@ -336,6 +336,7 @@ class VectorStoreWrapper(BaseToolApiWrapper):
 
         from ..langchain.interfaces.llm_processor import add_documents
 
+        self._log_tool_event(message=f"Starting the indexing... Parameters: {collection_suffix=}, {clean_index=}, {is_code}", tool_name="index_documents")
         # pre-process documents if needed (find duplicates, etc.)
         if clean_index:
             logger.info("Cleaning index before re-indexing all documents.")
@@ -351,9 +352,15 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             if isinstance(documents, types.GeneratorType):
                 documents = list(documents)
         else:
+            self._log_tool_event(
+                message="Filter for duplicates",
+                tool_name="index_documents")
             # remove duplicates based on metadata 'id' and 'updated_on' or 'commit_hash' fields
             documents = self._reduce_code_duplicates(documents, collection_suffix) if is_code \
                 else self._reduce_non_code_duplicates(documents, collection_suffix)
+            self._log_tool_event(
+                message="All the duplicates were filtered out. Proceeding with indexing.",
+                tool_name="index_documents")
 
         if not documents or len(documents) == 0:
             logger.info("No new documents to index after duplicate check.")
@@ -362,8 +369,8 @@ class VectorStoreWrapper(BaseToolApiWrapper):
         # if func is provided, apply it to documents
         # used for processing of documents before indexing,
         # e.g. to avoid time-consuming operations for documents that are already indexed
+        self._log_tool_event(message=f"Processing the dependent documents (attachments, etc.)", tool_name="index_documents")
         dependent_docs_generator = self.process_document_func(documents) if self.process_document_func else []
-
         # notify user about missed required metadata fields: id, updated_on
         # it is not required to have them, but it is recommended to have them for proper re-indexing and duplicate detection
         for doc in documents:
@@ -375,6 +382,9 @@ class VectorStoreWrapper(BaseToolApiWrapper):
 
         documents = documents + list(dependent_docs_generator)
 
+        self._log_tool_event(message=f"Documents for indexing were processed. Total documents: {len(documents)}",
+                             tool_name="index_documents")
+
         # if collection_suffix is provided, add it to metadata of each document
         if collection_suffix:
             for doc in documents:
@@ -386,7 +396,8 @@ class VectorStoreWrapper(BaseToolApiWrapper):
         total_docs = len(documents)
         documents_count = 0
         _documents = []
-
+        self._log_tool_event(message=f"Starting the indexing of processed documents. Total documents: {len(documents)}",
+                             tool_name="index_documents")
         # set default progress step to 20 if out of 0...100 or None
         progress_step = 20 if progress_step not in range(0, 100) else progress_step
         next_progress_point = progress_step

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk/tools/ado/work_item/__init__.py

@@ -26,7 +26,7 @@ class AzureDevOpsWorkItemsToolkit(BaseToolkit):
                                  'toolkit_name': True,
                                  'max_toolkit_length': AzureDevOpsWorkItemsToolkit.toolkit_max_length})
                   ),
-            ado_configuration=(AdoConfiguration, Field(description="Ado Work Item configuration", json_schema_extra={'configuration_types': ['ado_work_item']})),
+            ado_configuration=(AdoConfiguration, Field(description="Ado Work Item configuration", json_schema_extra={'configuration_types': ['ado']})),
             limit=(Optional[int], Field(description="ADO plans limit used for limitation of the list with results", default=5)),
             selected_tools=(List[Literal[tuple(selected_tools)]], Field(default=[], json_schema_extra={'args_schemas': selected_tools})),
             # indexer settings

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk/tools/chunkers/sematic/markdown_chunker.py

@@ -1,7 +1,6 @@
 from typing import Generator
-from langchain.schema import Document
 from langchain_core.documents import Document
-from langchain_text_splitters import MarkdownHeaderTextSplitter
+from langchain_text_splitters import MarkdownHeaderTextSplitter, ExperimentalMarkdownSyntaxTextSplitter
 from langchain.text_splitter import TokenTextSplitter
 from ..utils import tiktoken_length
 from copy import deepcopy as copy
@@ -50,4 +49,32 @@ def markdown_chunker(file_content_generator: Generator[Document, None, None], co
                 yield Document(
                     page_content=chunk.page_content,
                     metadata=docmeta
-                )
+                )
+
+
+def markdown_by_headers_chunker(file_content_generator: Generator[Document, None, None], config: dict, *args, **kwargs) -> Generator[Document, None, None]:
+    strip_header = config.get("strip_header", False)
+    return_each_line = config.get("return_each_line", False)
+    headers_to_split_on = config.get("headers_to_split_on", [])
+    headers_to_split_on = [header.split(' ', 1) for header in headers_to_split_on]
+    for doc in file_content_generator:
+        doc_metadata = doc.metadata
+        doc_content = doc.page_content
+        chunk_id = 0
+        markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(
+            headers_to_split_on=headers_to_split_on, 
+            strip_headers=strip_header,
+            return_each_line=return_each_line
+        )
+        md_header_splits = markdown_splitter.split_text(doc_content)
+        for chunk in md_header_splits:
+            chunk_id += 1
+            headers_meta = list(chunk.metadata.values())
+            docmeta = copy(doc_metadata)
+            docmeta.update({"headers": "; ".join(headers_meta)})
+            docmeta['chunk_id'] = chunk_id
+            docmeta['chunk_type'] = "document"
+            yield Document(
+                page_content=chunk.page_content,
+                metadata=docmeta
+            )

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk/tools/elitea_base.py

@@ -110,7 +110,7 @@ BaseStepbackSearchParams = create_model(
 BaseIndexDataParams = create_model(
     "indexData",
     __base__=BaseIndexParams,
-    progress_step=(Optional[int], Field(default=10, ge=0, le=100,
+    progress_step=(Optional[int], Field(default=5, ge=0, le=100,
                          description="Optional step size for progress reporting during indexing")),
     clean_index=(Optional[bool], Field(default=False,
                        description="Optional flag to enforce clean existing index before indexing new data")),
@@ -132,6 +132,8 @@ class BaseToolApiWrapper(BaseModel):
 
             if tool_name is None:
                 tool_name = 'tool_progress'
+
+            logger.info(message)
             dispatch_custom_event(
                 name="tool_execution_step",
                 data={
@@ -334,7 +336,13 @@ class BaseVectorStoreToolApiWrapper(BaseToolApiWrapper):
         Returns:
             Generator[Document, None, None]: A generator yielding processed documents with metadata.
         """
-        for doc in documents:
+        total_docs = len(documents)
+        self._log_tool_event(
+            message=f"Preparing a base documents for indexing. Total documents: {total_docs}",
+            tool_name="_process_documents"
+        )
+        processed_count = 0
+        for idx, doc in enumerate(documents, 1):
             # Filter documents to process only those that either:
             # - do not have a 'chunk_id' in their metadata, or
             # - have 'chunk_id' explicitly set to 1.
@@ -346,10 +354,19 @@ class BaseVectorStoreToolApiWrapper(BaseToolApiWrapper):
                     for processed_doc in processed_docs:
                         # map processed document (child) to the original document (parent)
                         processed_doc.metadata[IndexerKeywords.PARENT.value] = doc.metadata.get('id', None)
-                        if chunker:=self._get_dependencies_chunker(processed_doc):
-                            yield from chunker(file_content_generator=iter([processed_doc]), config=self._get_dependencies_chunker_config())
+                        if chunker := self._get_dependencies_chunker(processed_doc):
+                            yield from chunker(
+                                file_content_generator=iter([processed_doc]),
+                                config=self._get_dependencies_chunker_config()
+                            )
                         else:
                             yield processed_doc
+                processed_count += 1
+                if processed_count % 5 == 0 or processed_count == total_docs:
+                    self._log_tool_event(
+                        message=f"Prepared {processed_count} out of {total_docs} documents for indexing.",
+                        tool_name="_process_documents"
+                    )
 
 
     # TODO: init store once and re-use the instance
@@ -583,7 +600,7 @@ class BaseCodeToolApiWrapper(BaseVectorStoreToolApiWrapper):
         from .chunkers.code.codeparser import parse_code_files_for_db
 
         _files = self.__handle_get_files("", branch or self.active_branch or self._active_branch)
-
+        self._log_tool_event(message="Listing files in branch", tool_name="loader")
         logger.info(f"Files in branch: {_files}")
 
         def is_whitelisted(file_path: str) -> bool:
@@ -599,11 +616,22 @@ class BaseCodeToolApiWrapper(BaseVectorStoreToolApiWrapper):
             return False
 
         def file_content_generator():
-            for file in _files:
+            self._log_tool_event(message="Reading the files", tool_name="loader")
+            # log the progress of file reading
+            total_files = len(_files)
+            for idx, file in enumerate(_files, 1):
                 if is_whitelisted(file) and not is_blacklisted(file):
+                    # read file ONLY if it matches whitelist and does not match blacklist
+                    file_content = self._read_file(file, branch=branch or self.active_branch or self._active_branch)
+                    # hash the file content to ensure uniqueness
+                    import hashlib
+                    file_hash = hashlib.sha256(file_content.encode("utf-8")).hexdigest()
                     yield {"file_name": file,
-                           "file_content": self._read_file(file, branch=branch or self.active_branch or self._active_branch),
-                           "commit_hash": self._file_commit_hash(file, branch=branch or self.active_branch or self._active_branch)}
+                           "file_content": file_content,
+                           "commit_hash": file_hash}
+                if idx % 10 == 0 or idx == total_files:
+                    self._log_tool_event(message=f"{idx} out of {total_files} files have been read", tool_name="loader")
+            self._log_tool_event(message=f"{len(_files)} have been read", tool_name="loader")
 
         return parse_code_files_for_db(file_content_generator())
     
@@ -621,7 +649,9 @@ class BaseCodeToolApiWrapper(BaseVectorStoreToolApiWrapper):
             blacklist=blacklist
         )
         vectorstore = self._init_vector_store()
-        return vectorstore.index_documents(documents, collection_suffix=collection_suffix, clean_index=False, is_code=True)
+        clean_index = kwargs.get('clean_index', False)
+        return vectorstore.index_documents(documents, collection_suffix=collection_suffix,
+                                           clean_index=clean_index, is_code=True)
 
     def _get_vector_search_tools(self):
         """

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk/tools/utils/content_parser.py

@@ -1,16 +1,13 @@
 import os
 import tempfile
-from copy import deepcopy as copy
 from logging import getLogger
 from pathlib import Path
 from typing import Generator
 
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
-from langchain_text_splitters import TokenTextSplitter
 
 from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map
-from alita_sdk.tools.chunkers.utils import tiktoken_length
 
 logger = getLogger(__name__)
 
@@ -193,37 +190,11 @@ def process_content_by_type(document: Document, content, extension_source: str,
             loader_kwargs = loader_config['kwargs']
 
             loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
-            docs_iterator = loader.load()
-            max_tokens = chunking_config.get('max_tokens', 512)
-            tokens_overlapping = chunking_config.get('tokens_overlapping', 10)
-            chunk_id = 0
-            for chunk in docs_iterator:
-                if tiktoken_length(chunk.page_content) > max_tokens:
-                    for subchunk in TokenTextSplitter(encoding_name="cl100k_base", 
-                                                      chunk_size=max_tokens, 
-                                                      chunk_overlap=tokens_overlapping
-                                                      ).split_text(chunk.page_content):
-                        chunk_id += 1
-                        headers_meta = list(chunk.metadata.values())
-                        docmeta = copy(document.metadata)
-                        docmeta.update({"headers": "; ".join(str(headers_meta))})
-                        docmeta['chunk_id'] = chunk_id
-                        docmeta['chunk_type'] = "document"
-                        yield Document(
-                            page_content=sanitize_for_postgres(subchunk),
-                            metadata=docmeta
-                        )
-                else:
-                    chunk_id += 1
-                    headers_meta = list(chunk.metadata.values())
-                    docmeta = copy(document.metadata)
-                    docmeta.update({"headers": "; ".join(str(headers_meta))})
-                    docmeta['chunk_id'] = chunk_id
-                    docmeta['chunk_type'] = "document"
-                    yield Document(
-                        page_content=sanitize_for_postgres(chunk.page_content),
-                        metadata=docmeta
-                    )
+            for chunk in loader.load():
+                yield Document(
+                    page_content=sanitize_for_postgres(chunk.page_content),
+                    metadata={**document.metadata, **chunk.metadata}
+                )
     finally:
         if temp_file_path and os.path.exists(temp_file_path):
             os.remove(temp_file_path)

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/alita_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.254
+Version: 0.3.255
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.254 → alita_sdk-0.3.255}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "alita_sdk"
-version = "0.3.254"
+version = "0.3.255"
 description = "SDK for building langchain agents using resources from Alita"
 readme = "README.md"
 requires-python = ">=3.10"