alita-sdk 0.3.457__py3-none-any.whl → 0.3.486__py3-none-any.whl

alita_sdk/tools/base_indexer_toolkit.py

@@ -2,8 +2,10 @@ import copy
 import json
 import logging
 import time
+from enum import Enum
 from typing import Any, Optional, List, Dict, Generator
 
+from langchain_core.callbacks import dispatch_custom_event
 from langchain_core.documents import Document
 from pydantic import create_model, Field, SecretStr
 
@@ -15,7 +17,17 @@ from ..runtime.utils.utils import IndexerKeywords
 
 logger = logging.getLogger(__name__)
 
-DEFAULT_CUT_OFF = 0.2
+DEFAULT_CUT_OFF = 0.1
+INDEX_META_UPDATE_INTERVAL = 600.0
+
+class IndexTools(str, Enum):
+    """Enum for index-related tool names."""
+    INDEX_DATA = "index_data"
+    SEARCH_INDEX = "search_index"
+    STEPBACK_SEARCH_INDEX = "stepback_search_index"
+    STEPBACK_SUMMARY_INDEX = "stepback_summary_index"
+    REMOVE_INDEX = "remove_index"
+    LIST_COLLECTIONS = "list_collections"
 
 # Base Vector Store Schema Models
 BaseIndexParams = create_model(
@@ -156,6 +168,16 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         clean_index = kwargs.get("clean_index")
         chunking_tool = kwargs.get("chunking_tool")
         chunking_config = kwargs.get("chunking_config")
+
+        # Store the interval in a private dict to avoid Pydantic field errors
+        if not hasattr(self, "_index_meta_config"):
+            self._index_meta_config: Dict[str, Any] = {}
+
+        self._index_meta_config["update_interval"] = kwargs.get(
+            "meta_update_interval",
+            INDEX_META_UPDATE_INTERVAL,
+        )
+
         result = {"count": 0}
         #
         try:
@@ -163,6 +185,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
                 self._clean_index(index_name)
             #
             self.index_meta_init(index_name, kwargs)
+            self._emit_index_event(index_name)
             #
             self._log_tool_event(f"Indexing data into collection with suffix '{index_name}'. It can take some time...")
             self._log_tool_event(f"Loading the documents to index...{kwargs}")
@@ -178,16 +201,26 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             self._save_index_generator(documents, documents_count, chunking_tool, chunking_config, index_name=index_name, result=result)
             #
             results_count = result["count"]
-            self.index_meta_update(index_name, IndexerKeywords.INDEX_META_COMPLETED.value, results_count)
+            # Final update should always be forced
+            self.index_meta_update(index_name, IndexerKeywords.INDEX_META_COMPLETED.value, results_count, update_force=True)
+            self._emit_index_event(index_name)
             #
             return {"status": "ok", "message": f"successfully indexed {results_count} documents" if results_count > 0
             else "no new documents to index"}
         except Exception as e:
-            self.index_meta_update(index_name, IndexerKeywords.INDEX_META_FAILED.value, result["count"])
+            # Do maximum effort at least send custom event for supposed changed status
+            msg = str(e)
+            try:
+                # Error update should also be forced
+                self.index_meta_update(index_name, IndexerKeywords.INDEX_META_FAILED.value, result["count"], update_force=True)
+            except Exception as ie:
+                logger.error(f"Failed to update index meta status to FAILED for index '{index_name}': {ie}")
+                msg = f"{msg}; additionally failed to update index meta status to FAILED: {ie}"
+            self._emit_index_event(index_name, error=msg)
             raise e
-            
 
     def _save_index_generator(self, base_documents: Generator[Document, None, None], base_total: int, chunking_tool, chunking_config, result, index_name: Optional[str] = None):
+        self._ensure_vectorstore_initialized()
         self._log_tool_event(f"Base documents are ready for indexing. {base_total} base documents in total to index.")
         from ..runtime.langchain.interfaces.llm_processor import add_documents
         #
@@ -240,6 +273,11 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             logger.debug(msg)
             self._log_tool_event(msg)
             result["count"] += dependent_docs_counter
+            # After each base document, try a non-forced meta update; throttling handled inside index_meta_update
+            try:
+                self.index_meta_update(index_name, IndexerKeywords.INDEX_META_IN_PROGRESS.value, result["count"], update_force=False)
+            except Exception as exc:  # best-effort, do not break indexing
+                logger.warning(f"Failed to update index meta during indexing process for index '{index_name}': {exc}")
         if pg_vector_add_docs_chunk:
             add_documents(vectorstore=self.vectorstore, documents=pg_vector_add_docs_chunk)
 
@@ -305,6 +343,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             log_msg: str = "Verification of documents to index started"
     ) -> Generator[Document, None, None]:
         """Generic duplicate reduction logic for documents."""
+        self._ensure_vectorstore_initialized()
         self._log_tool_event(log_msg, tool_name="index_documents")
         indexed_data = self._get_indexed_data(index_name)
         indexed_keys = set(indexed_data.keys())
@@ -460,6 +499,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         )
     
     def index_meta_init(self, index_name: str, index_configuration: dict[str, Any]):
+        self._ensure_vectorstore_initialized()
         index_meta = super().get_index_meta(index_name)
         if not index_meta:
             self._log_tool_event(
@@ -479,12 +519,53 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
                 "updated_on": created_on,
                 "task_id": None,
                 "conversation_id": None,
+                "toolkit_id": self.toolkit_id,
             }
             metadata["history"] = json.dumps([metadata])
             index_meta_doc = Document(page_content=f"{IndexerKeywords.INDEX_META_TYPE.value}_{index_name}", metadata=metadata)
             add_documents(vectorstore=self.vectorstore, documents=[index_meta_doc])
 
-    def index_meta_update(self, index_name: str, state: str, result: int):
+    def index_meta_update(self, index_name: str, state: str, result: int, update_force: bool = True, interval: Optional[float] = None):
+        """Update `index_meta` document with optional time-based throttling.
+
+        Args:
+            index_name: Index name to update meta for.
+            state: New state value for the `index_meta` record.
+            result: Number of processed documents to store in the `updated` field.
+            update_force: If `True`, perform the update unconditionally, ignoring throttling.
+                          If `False`, perform the update only when the effective time interval has passed.
+            interval: Optional custom interval (in seconds) for this call when `update_force` is `False`.
+                      If `None`, falls back to the value stored in `self._index_meta_config["update_interval"]`
+                      if present, otherwise uses `INDEX_META_UPDATE_INTERVAL`.
+        """
+        self._ensure_vectorstore_initialized()
+        if not hasattr(self, "_index_meta_last_update_time"):
+            self._index_meta_last_update_time: Dict[str, float] = {}
+
+        if not update_force:
+            # Resolve effective interval:
+            # 1\) explicit arg
+            # 2\) value from `_index_meta_config`
+            # 3\) default constant
+            cfg_interval = None
+            if hasattr(self, "_index_meta_config"):
+                cfg_interval = self._index_meta_config.get("update_interval")
+
+            eff_interval = (
+                interval
+                if interval is not None
+                else (cfg_interval if cfg_interval is not None else INDEX_META_UPDATE_INTERVAL)
+            )
+
+            last_time = self._index_meta_last_update_time.get(index_name)
+            now = time.time()
+            if last_time is not None and (now - last_time) < eff_interval:
+                return
+            self._index_meta_last_update_time[index_name] = now
+        else:
+            # For forced updates, always refresh last update time
+            self._index_meta_last_update_time[index_name] = time.time()
+
         index_meta_raw = super().get_index_meta(index_name)
         from ..runtime.langchain.interfaces.llm_processor import add_documents
         #
@@ -511,6 +592,55 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             index_meta_doc = Document(page_content=index_meta_raw.get("content", ""), metadata=metadata)
             add_documents(vectorstore=self.vectorstore, documents=[index_meta_doc], ids=[index_meta_raw.get("id")])
 
+    def _emit_index_event(self, index_name: str, error: Optional[str] = None):
+        """
+        Emit custom event for index data operation.
+        
+        Args:
+            index_name: The name of the index
+            error: Error message if the operation failed, None otherwise
+        """
+        index_meta = super().get_index_meta(index_name)
+        
+        if not index_meta:
+            logger.warning(
+                f"No index_meta found for index '{index_name}'. "
+                "Cannot emit index event."
+            )
+            return
+        
+        metadata = index_meta.get("metadata", {})
+        
+        # Determine if this is a reindex operation
+        history_raw = metadata.get("history", "[]")
+        try:
+            history = json.loads(history_raw) if history_raw.strip() else []
+            is_reindex = len(history) > 1
+        except (json.JSONDecodeError, TypeError):
+            is_reindex = False
+        
+        # Build event message
+        event_data = {
+            "id": index_meta.get("id"),
+            "index_name": index_name,
+            "state": "failed" if error is not None else metadata.get("state"),
+            "error": error,
+            "reindex": is_reindex,
+            "indexed": metadata.get("indexed", 0),
+            "updated": metadata.get("updated", 0),
+            "toolkit_id": metadata.get("toolkit_id"),
+        }
+        
+        # Emit the event
+        try:
+            dispatch_custom_event("index_data_status", event_data)
+            logger.debug(
+                f"Emitted index_data_status event for index "
+                f"'{index_name}': {event_data}"
+            )
+        except Exception as e:
+            logger.warning(f"Failed to emit index_data_status event: {e}")
+
     def get_available_tools(self):
         """
         Returns the standardized vector search tools (search operations only).
@@ -521,8 +651,8 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         """
         return [
             {
-                "name": "index_data",
-                "mode": "index_data",
+                "name": IndexTools.INDEX_DATA.value,
+                "mode": IndexTools.INDEX_DATA.value,
                 "ref": self.index_data,
                 "description": "Loads data to index.",
                 "args_schema": create_model(
@@ -532,38 +662,39 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
                 )
             },
             {
-                "name": "search_index",
-                "mode": "search_index",
+                "name": IndexTools.SEARCH_INDEX.value,
+                "mode": IndexTools.SEARCH_INDEX.value,
                 "ref": self.search_index,
                 "description": self.search_index.__doc__,
                 "args_schema": BaseSearchParams
             },
             {
-                "name": "stepback_search_index",
-                "mode": "stepback_search_index",
+                "name": IndexTools.STEPBACK_SEARCH_INDEX.value,
+                "mode": IndexTools.STEPBACK_SEARCH_INDEX.value,
                 "ref": self.stepback_search_index,
                 "description": self.stepback_search_index.__doc__,
                 "args_schema": BaseStepbackSearchParams
             },
             {
-                "name": "stepback_summary_index",
-                "mode": "stepback_summary_index",
+                "name": IndexTools.STEPBACK_SUMMARY_INDEX.value,
+                "mode": IndexTools.STEPBACK_SUMMARY_INDEX.value,
                 "ref": self.stepback_summary_index,
                 "description": self.stepback_summary_index.__doc__,
                 "args_schema": BaseStepbackSearchParams
             },
             {
-                "name": "remove_index",
-                "mode": "remove_index",
+                "name": IndexTools.REMOVE_INDEX.value,
+                "mode": IndexTools.REMOVE_INDEX.value,
                 "ref": self.remove_index,
                 "description": self.remove_index.__doc__,
                 "args_schema": RemoveIndexParams
             },
             {
-                "name": "list_collections",
-                "mode": "list_collections",
+                "name": IndexTools.LIST_COLLECTIONS.value,
+                "mode": IndexTools.LIST_COLLECTIONS.value,
                 "ref": self.list_collections,
                 "description": self.list_collections.__doc__,
-                "args_schema": create_model("ListCollectionsParams")  # No parameters
+                # No parameters
+                "args_schema": create_model("ListCollectionsParams")
             },
-        ]
+        ]

alita_sdk/tools/bitbucket/__init__.py

@@ -47,8 +47,8 @@ class AlitaBitbucketToolkit(BaseToolkit):
         AlitaBitbucketToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         m = create_model(
             name,
-            project=(str, Field(description="Project/Workspace", json_schema_extra={'configuration': True})),
-            repository=(str, Field(description="Repository", json_schema_extra={'max_toolkit_length': AlitaBitbucketToolkit.toolkit_max_length, 'configuration': True})),
+            project=(str, Field(description="Project/Workspace")),
+            repository=(str, Field(description="Repository")),
             branch=(str, Field(description="Main branch", default="main")),
             cloud=(Optional[bool], Field(description="Hosting Option", default=None)),
             bitbucket_configuration=(BitbucketConfiguration, Field(description="Bitbucket Configuration", json_schema_extra={'configuration_types': ['bitbucket']})),

alita_sdk/tools/chunkers/__init__.py

@@ -3,6 +3,7 @@ from .sematic.statistical_chunker import statistical_chunker
 from .sematic.markdown_chunker import markdown_chunker
 from .sematic.proposal_chunker import proposal_chunker
 from .sematic.json_chunker import json_chunker
+from .universal_chunker import universal_chunker, chunk_single_document, get_file_type
 from .models import StatisticalChunkerConfig, MarkdownChunkerConfig, ProposalChunkerConfig
 
 __all__ = {
@@ -10,7 +11,8 @@ __all__ = {
     'statistical': statistical_chunker,
     'markdown': markdown_chunker,
     'proposal': proposal_chunker,
-    'json': json_chunker
+    'json': json_chunker,
+    'universal': universal_chunker,
 }
 
 __confluence_chunkers__ = {

alita_sdk/tools/chunkers/sematic/markdown_chunker.py

@@ -1,4 +1,4 @@
-from typing import Generator
+from typing import Generator, List
 from langchain_core.documents import Document
 from langchain_text_splitters import MarkdownHeaderTextSplitter, ExperimentalMarkdownSyntaxTextSplitter
 from langchain.text_splitter import TokenTextSplitter
@@ -7,28 +7,53 @@ from copy import deepcopy as copy
 
 
 def markdown_chunker(file_content_generator: Generator[Document, None, None], config: dict, *args, **kwargs) -> Generator[Document, None, None]:
+    """
+    Chunks markdown documents by headers, with support for:
+    - Minimum chunk size to avoid tiny fragments
+    - Maximum token limit with overflow splitting
+    - Header metadata preservation
+    
+    Config options:
+        strip_header (bool): Remove headers from content. Default: False
+        return_each_line (bool): Split on every line. Default: False
+        headers_to_split_on (list): Headers to split on, e.g. [('#', 'H1'), ('##', 'H2')]
+        max_tokens (int): Maximum tokens per chunk. Default: 512
+        token_overlap (int): Token overlap for large chunk splitting. Default: 10
+        min_chunk_chars (int): Minimum characters per chunk. Default: 100
+            Chunks smaller than this will be merged with the next chunk.
+    """
     strip_header = config.get("strip_header", False)
     return_each_line = config.get("return_each_line", False)
     headers_to_split_on = config.get("headers_to_split_on", [])
     max_tokens = config.get("max_tokens", 512)
     tokens_overlapping = config.get("token_overlap", 10)
+    min_chunk_chars = config.get("min_chunk_chars", 100)  # Minimum characters per chunk
+    
     headers_to_split_on = [tuple(header) for header in headers_to_split_on]
+    
     for doc in file_content_generator:
         doc_metadata = doc.metadata
         doc_content = doc.page_content
         chunk_id = 0
+        
         markdown_splitter = MarkdownHeaderTextSplitter(
             headers_to_split_on=headers_to_split_on, 
             strip_headers=strip_header,
             return_each_line=return_each_line
         )
         md_header_splits = markdown_splitter.split_text(doc_content)
-        for chunk in md_header_splits:
+        
+        # Merge small chunks with the next one
+        merged_chunks = _merge_small_chunks(md_header_splits, min_chunk_chars)
+        
+        for chunk in merged_chunks:
             if tiktoken_length(chunk.page_content) > max_tokens:
-                for subchunk in TokenTextSplitter(encoding_name="cl100k_base", 
-                                                  chunk_size=max_tokens, 
-                                                  chunk_overlap=tokens_overlapping
-                                                  ).split_text(chunk.page_content):
+                # Split large chunks into smaller ones
+                for subchunk in TokenTextSplitter(
+                    encoding_name="cl100k_base", 
+                    chunk_size=max_tokens, 
+                    chunk_overlap=tokens_overlapping
+                ).split_text(chunk.page_content):
                     chunk_id += 1
                     headers_meta = list(chunk.metadata.values())
                     docmeta = copy(doc_metadata)
@@ -52,6 +77,70 @@ def markdown_chunker(file_content_generator: Generator[Document, None, None], co
                 )
 
 
+def _merge_small_chunks(chunks: List[Document], min_chars: int) -> List[Document]:
+    """
+    Merge chunks that are smaller than min_chars with the next chunk.
+    
+    This prevents tiny fragments (like standalone headers or short notes)
+    from becoming separate chunks.
+    
+    Args:
+        chunks: List of Document chunks from markdown splitter
+        min_chars: Minimum character count for a chunk
+        
+    Returns:
+        List of merged Document chunks
+    """
+    if not chunks:
+        return chunks
+    
+    merged = []
+    pending_content = ""
+    pending_metadata = {}
+    
+    for i, chunk in enumerate(chunks):
+        content = chunk.page_content.strip()
+        
+        if pending_content:
+            # Merge pending content with current chunk
+            combined_content = pending_content + "\n\n" + content
+            # Use the pending metadata (from the header) but can be extended
+            combined_metadata = {**pending_metadata}
+            # Add any new header info from current chunk
+            for key, value in chunk.metadata.items():
+                if key not in combined_metadata or not combined_metadata[key]:
+                    combined_metadata[key] = value
+            
+            if len(combined_content) >= min_chars:
+                # Combined is big enough, emit it
+                merged.append(Document(
+                    page_content=combined_content,
+                    metadata=combined_metadata
+                ))
+                pending_content = ""
+                pending_metadata = {}
+            else:
+                # Still too small, keep accumulating
+                pending_content = combined_content
+                pending_metadata = combined_metadata
+        elif len(content) < min_chars:
+            # Current chunk is too small, start pending
+            pending_content = content
+            pending_metadata = dict(chunk.metadata)
+        else:
+            # Current chunk is big enough
+            merged.append(chunk)
+    
+    # Don't forget any remaining pending content
+    if pending_content:
+        merged.append(Document(
+            page_content=pending_content,
+            metadata=pending_metadata
+        ))
+    
+    return merged
+
+
 def markdown_by_headers_chunker(file_content_generator: Generator[Document, None, None], config: dict, *args, **kwargs) -> Generator[Document, None, None]:
     strip_header = config.get("strip_header", False)
     return_each_line = config.get("return_each_line", False)