alita-sdk 0.3.465__py3-none-any.whl → 0.3.497__py3-none-any.whl

alita_sdk/runtime/utils/streamlit.py

@@ -287,7 +287,6 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                     model_config={
                         "temperature": 0.1,
                         "max_tokens": 1000,
-                        "top_p": 1.0
                     }
                 )
             except Exception as e:
@@ -1256,7 +1255,6 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                     model_config={
                         "temperature": 0.1,
                         "max_tokens": 1000,
-                        "top_p": 1.0
                     }
                 )
             except Exception as e:
@@ -1387,20 +1385,18 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                                 help="Maximum number of tokens in the AI response"
                             )
                             
-                            top_p = st.slider(
-                                "Top-p:",
-                                min_value=0.1,
-                                max_value=1.0,
-                                value=1.0,
-                                step=0.1,
-                                help="Controls diversity via nucleus sampling"
+                            reasoning_effort = st.selectbox(
+                                "Reasoning effort:",
+                                options=['null', 'low', 'medium', 'high'],
+                                index=0,
+                                help="Higher effort better reasoning, slower response"
                             )
                         
                         # Create LLM config
                         llm_config = {
                             'max_tokens': max_tokens,
                             'temperature': temperature,
-                            'top_p': top_p
+                            'reasoning_effort': reasoning_effort
                         }
                         
                         col1, col2 = st.columns([3, 1])

alita_sdk/runtime/utils/toolkit_utils.py

@@ -12,7 +12,9 @@ logger = logging.getLogger(__name__)
 
 def instantiate_toolkit_with_client(toolkit_config: Dict[str, Any], 
                                    llm_client: Any, 
-                                   alita_client: Optional[Any] = None) -> List[Any]:
+                                   alita_client: Optional[Any] = None,
+                                   mcp_tokens: Optional[Dict[str, Any]] = None,
+                                   use_prefix: bool = False) -> List[Any]:
     """
     Instantiate a toolkit with LLM client support.
     
@@ -22,7 +24,11 @@ def instantiate_toolkit_with_client(toolkit_config: Dict[str, Any],
     Args:
         toolkit_config: Configuration dictionary for the toolkit
         llm_client: LLM client instance for tools that need LLM capabilities
-        client: Optional additional client instance
+        alita_client: Optional additional client instance
+        mcp_tokens: Optional dictionary of MCP OAuth tokens by server URL
+        use_prefix: If True, tools get prefixed with toolkit_name to prevent collisions
+                   (for agent use). If False, tools use base names only (for testing interface).
+                   Default False for backward compatibility with testing.
     
     Returns:
         List of instantiated tools from the toolkit
@@ -52,16 +58,17 @@ def instantiate_toolkit_with_client(toolkit_config: Dict[str, Any],
         toolkit_type = toolkit_config.get('type', toolkit_name.lower())
 
         # Create a tool configuration dict with required fields
+        # Note: MCP toolkit always requires toolkit_name, other toolkits respect use_prefix flag
         tool_config = {
             'id': toolkit_config.get('id', random.randint(1, 1000000)),
             'type': toolkit_config.get('type', toolkit_type),
             'settings': settings,
-            'toolkit_name': toolkit_name
+            'toolkit_name': toolkit_name if (use_prefix or toolkit_type == 'mcp') else None
         }
         
         # Get tools using the toolkit configuration with clients
-        # Parameter order: get_tools(tools_list, alita_client, llm, memory_store)
-        tools = get_tools([tool_config], alita_client, llm_client)
+        # Parameter order: get_tools(tools_list, alita_client, llm, memory_store, debug_mode, mcp_tokens)
+        tools = get_tools([tool_config], alita_client, llm_client, mcp_tokens=mcp_tokens)
         
         if not tools:
             logger.warning(f"No tools returned for toolkit {toolkit_name}")
@@ -73,9 +80,11 @@ def instantiate_toolkit_with_client(toolkit_config: Dict[str, Any],
     except Exception as e:
         # Re-raise McpAuthorizationRequired without logging as error
         from ..utils.mcp_oauth import McpAuthorizationRequired
+        
         if isinstance(e, McpAuthorizationRequired):
             logger.info(f"Toolkit {toolkit_name} requires MCP OAuth authorization")
             raise
+        
         # Log and re-raise other errors
         logger.error(f"Error instantiating toolkit {toolkit_name} with client: {str(e)}")
         raise

alita_sdk/tools/__init__.py

@@ -13,6 +13,30 @@ AVAILABLE_TOOLS = {}
 AVAILABLE_TOOLKITS = {}
 FAILED_IMPORTS = {}
 
+
+def _inject_toolkit_id(tool_conf: dict, toolkit_tools) -> None:
+    """Inject `toolkit_id` into tools that expose `api_wrapper.toolkit_id`.
+
+    This reads 'id' from the tool configuration and, if it is an integer,
+    assigns it to the 'toolkit_id' attribute of the 'api_wrapper' for each
+    tool in 'toolkit_tools' that supports it.
+
+    Args:
+        tool_conf: Raw tool configuration item from 'tools_list'.
+        toolkit_tools: List of instantiated tools produced by a toolkit.
+    """
+    toolkit_id = tool_conf.get('id')
+    if isinstance(toolkit_id, int):
+        for t in toolkit_tools:
+            if hasattr(t, 'api_wrapper') and hasattr(t.api_wrapper, 'toolkit_id'):
+                t.api_wrapper.toolkit_id = toolkit_id
+    else:
+        logger.error(
+            f"Toolkit ID is missing or not an integer for tool "
+            f"`{tool_conf.get('type', '')}` with name `{tool_conf.get('name', '')}`"
+        )
+
+
 def _safe_import_tool(tool_name, module_path, get_tools_name=None, toolkit_class_name=None):
     """Safely import a tool module and register available functions/classes."""
     try:
@@ -34,6 +58,7 @@ def _safe_import_tool(tool_name, module_path, get_tools_name=None, toolkit_class
         FAILED_IMPORTS[tool_name] = str(e)
         logger.debug(f"Failed to import {tool_name}: {e}")
 
+
 # Safe imports for all tools
 _safe_import_tool('github', 'github', 'get_tools', 'AlitaGitHubToolkit')
 _safe_import_tool('openapi', 'openapi', 'get_tools')
@@ -90,11 +115,19 @@ available_count = len(AVAILABLE_TOOLS)
 total_attempted = len(AVAILABLE_TOOLS) + len(FAILED_IMPORTS)
 logger.info(f"Tool imports completed: {available_count}/{total_attempted} successful")
 
+# Import community module to trigger community toolkit registration
+try:
+    from alita_sdk import community  # noqa: F401
+    logger.debug("Community toolkits registered successfully")
+except ImportError as e:
+    logger.debug(f"Community module not available: {e}")
+
 
 def get_tools(tools_list, alita, llm, store: Optional[BaseStore] = None, *args, **kwargs):
     tools = []
 
     for tool in tools_list:
+        toolkit_tools = []
         settings = tool.get('settings')
 
         # Skip tools without settings early
@@ -116,53 +149,47 @@ def get_tools(tools_list, alita, llm, store: Optional[BaseStore] = None, *args,
 
         # Set pgvector collection schema if present
         if settings.get('pgvector_configuration'):
-            settings['pgvector_configuration']['collection_schema'] = str(tool['id'])
+            # Use tool id if available, otherwise use toolkit_name or type as fallback
+            collection_id = tool.get('id') or tool.get('toolkit_name') or tool_type
+            settings['pgvector_configuration']['collection_schema'] = str(collection_id)
 
         # Handle ADO special cases
         if tool_type in ['ado_boards', 'ado_wiki', 'ado_plans']:
-            tools.extend(AVAILABLE_TOOLS['ado']['get_tools'](tool_type, tool))
-            continue
-
-        # Handle ADO repos aliases
-        if tool_type in ['ado_repos', 'azure_devops_repos'] and 'ado_repos' in AVAILABLE_TOOLS:
+            toolkit_tools.extend(AVAILABLE_TOOLS['ado']['get_tools'](tool_type, tool))
+        elif tool_type in ['ado_repos', 'azure_devops_repos'] and 'ado_repos' in AVAILABLE_TOOLS:
             try:
-                tools.extend(AVAILABLE_TOOLS['ado_repos']['get_tools'](tool))
+                toolkit_tools.extend(AVAILABLE_TOOLS['ado_repos']['get_tools'](tool))
             except Exception as e:
                 logger.error(f"Error getting ADO repos tools: {e}")
-            continue
-
-        # Skip MCP toolkit - it's handled by runtime/toolkits/tools.py to avoid duplicate loading
-        if tool_type == 'mcp':
+        elif tool_type == 'mcp':
             logger.debug(f"Skipping MCP toolkit '{tool.get('toolkit_name')}' - handled by runtime toolkit system")
-            continue
-
-        # Handle standard tools
-        if tool_type in AVAILABLE_TOOLS and 'get_tools' in AVAILABLE_TOOLS[tool_type]:
+        elif tool_type == 'planning':
+            logger.debug(f"Skipping planning toolkit '{tool.get('toolkit_name')}' - handled by runtime toolkit system")
+        elif tool_type in AVAILABLE_TOOLS and 'get_tools' in AVAILABLE_TOOLS[tool_type]:
             try:
-                tools.extend(AVAILABLE_TOOLS[tool_type]['get_tools'](tool))
+                toolkit_tools.extend(AVAILABLE_TOOLS[tool_type]['get_tools'](tool))
             except Exception as e:
                 logger.error(f"Error getting tools for {tool_type}: {e}")
                 raise ToolException(f"Error getting tools for {tool_type}: {e}")
-            continue
-
-        # Handle custom modules
-        if settings.get("module"):
+        elif settings.get("module"):
             try:
                 mod = import_module(settings.pop("module"))
                 tkitclass = getattr(mod, settings.pop("class"))
                 get_toolkit_params = settings.copy()
                 get_toolkit_params["name"] = tool.get("name")
                 toolkit = tkitclass.get_toolkit(**get_toolkit_params)
-                tools.extend(toolkit.get_tools())
+                toolkit_tools.extend(toolkit.get_tools())
             except Exception as e:
                 logger.error(f"Error in getting custom toolkit: {e}")
-            continue
-
-        # Tool not available
-        if tool_type in FAILED_IMPORTS:
-            logger.warning(f"Tool '{tool_type}' is not available: {FAILED_IMPORTS[tool_type]}")
         else:
-            logger.warning(f"Unknown tool type: {tool_type}")
+            if tool_type in FAILED_IMPORTS:
+                logger.warning(f"Tool '{tool_type}' is not available: {FAILED_IMPORTS[tool_type]}")
+            else:
+                logger.warning(f"Unknown tool type: {tool_type}")
+        #
+        # Always inject toolkit_id to each tool 
+        _inject_toolkit_id(tool, toolkit_tools)
+        tools.extend(toolkit_tools)
 
     return tools
 

alita_sdk/tools/ado/repos/repos_wrapper.py

@@ -111,8 +111,7 @@ class ArgsSchema(Enum):
             Field(
                 description=(
                     "Branch to be used for read file operation."
-                ),
-                default=None
+                )
             ),
         )
     )

alita_sdk/tools/base_indexer_toolkit.py

@@ -2,6 +2,7 @@ import copy
 import json
 import logging
 import time
+from enum import Enum
 from typing import Any, Optional, List, Dict, Generator
 
 from langchain_core.callbacks import dispatch_custom_event
@@ -16,7 +17,17 @@ from ..runtime.utils.utils import IndexerKeywords
 
 logger = logging.getLogger(__name__)
 
-DEFAULT_CUT_OFF = 0.2
+DEFAULT_CUT_OFF = 0.1
+INDEX_META_UPDATE_INTERVAL = 600.0
+
+class IndexTools(str, Enum):
+    """Enum for index-related tool names."""
+    INDEX_DATA = "index_data"
+    SEARCH_INDEX = "search_index"
+    STEPBACK_SEARCH_INDEX = "stepback_search_index"
+    STEPBACK_SUMMARY_INDEX = "stepback_summary_index"
+    REMOVE_INDEX = "remove_index"
+    LIST_COLLECTIONS = "list_collections"
 
 # Base Vector Store Schema Models
 BaseIndexParams = create_model(
@@ -157,6 +168,16 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         clean_index = kwargs.get("clean_index")
         chunking_tool = kwargs.get("chunking_tool")
         chunking_config = kwargs.get("chunking_config")
+
+        # Store the interval in a private dict to avoid Pydantic field errors
+        if not hasattr(self, "_index_meta_config"):
+            self._index_meta_config: Dict[str, Any] = {}
+
+        self._index_meta_config["update_interval"] = kwargs.get(
+            "meta_update_interval",
+            INDEX_META_UPDATE_INTERVAL,
+        )
+
         result = {"count": 0}
         #
         try:
@@ -164,6 +185,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
                 self._clean_index(index_name)
             #
             self.index_meta_init(index_name, kwargs)
+            self._emit_index_event(index_name)
             #
             self._log_tool_event(f"Indexing data into collection with suffix '{index_name}'. It can take some time...")
             self._log_tool_event(f"Loading the documents to index...{kwargs}")
@@ -179,18 +201,26 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             self._save_index_generator(documents, documents_count, chunking_tool, chunking_config, index_name=index_name, result=result)
             #
             results_count = result["count"]
-            self.index_meta_update(index_name, IndexerKeywords.INDEX_META_COMPLETED.value, results_count)
+            # Final update should always be forced
+            self.index_meta_update(index_name, IndexerKeywords.INDEX_META_COMPLETED.value, results_count, update_force=True)
             self._emit_index_event(index_name)
             #
             return {"status": "ok", "message": f"successfully indexed {results_count} documents" if results_count > 0
             else "no new documents to index"}
         except Exception as e:
-            self.index_meta_update(index_name, IndexerKeywords.INDEX_META_FAILED.value, result["count"])
-            self._emit_index_event(index_name, error=str(e))
+            # Do maximum effort at least send custom event for supposed changed status
+            msg = str(e)
+            try:
+                # Error update should also be forced
+                self.index_meta_update(index_name, IndexerKeywords.INDEX_META_FAILED.value, result["count"], update_force=True)
+            except Exception as ie:
+                logger.error(f"Failed to update index meta status to FAILED for index '{index_name}': {ie}")
+                msg = f"{msg}; additionally failed to update index meta status to FAILED: {ie}"
+            self._emit_index_event(index_name, error=msg)
             raise e
-            
 
     def _save_index_generator(self, base_documents: Generator[Document, None, None], base_total: int, chunking_tool, chunking_config, result, index_name: Optional[str] = None):
+        self._ensure_vectorstore_initialized()
         self._log_tool_event(f"Base documents are ready for indexing. {base_total} base documents in total to index.")
         from ..runtime.langchain.interfaces.llm_processor import add_documents
         #
@@ -243,6 +273,11 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             logger.debug(msg)
             self._log_tool_event(msg)
             result["count"] += dependent_docs_counter
+            # After each base document, try a non-forced meta update; throttling handled inside index_meta_update
+            try:
+                self.index_meta_update(index_name, IndexerKeywords.INDEX_META_IN_PROGRESS.value, result["count"], update_force=False)
+            except Exception as exc:  # best-effort, do not break indexing
+                logger.warning(f"Failed to update index meta during indexing process for index '{index_name}': {exc}")
         if pg_vector_add_docs_chunk:
             add_documents(vectorstore=self.vectorstore, documents=pg_vector_add_docs_chunk)
 
@@ -308,6 +343,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             log_msg: str = "Verification of documents to index started"
     ) -> Generator[Document, None, None]:
         """Generic duplicate reduction logic for documents."""
+        self._ensure_vectorstore_initialized()
         self._log_tool_event(log_msg, tool_name="index_documents")
         indexed_data = self._get_indexed_data(index_name)
         indexed_keys = set(indexed_data.keys())
@@ -350,7 +386,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
 
     def remove_index(self, index_name: str = ""):
         """Cleans the indexed data in the collection."""
-        super()._clean_collection(index_name=index_name)
+        super()._clean_collection(index_name=index_name, including_index_meta=True)
         return (f"Collection '{index_name}' has been removed from the vector store.\n"
                 f"Available collections: {self.list_collections()}") if index_name \
             else "All collections have been removed from the vector store." 
@@ -463,6 +499,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         )
     
     def index_meta_init(self, index_name: str, index_configuration: dict[str, Any]):
+        self._ensure_vectorstore_initialized()
         index_meta = super().get_index_meta(index_name)
         if not index_meta:
             self._log_tool_event(
@@ -482,12 +519,53 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
                 "updated_on": created_on,
                 "task_id": None,
                 "conversation_id": None,
+                "toolkit_id": self.toolkit_id,
             }
             metadata["history"] = json.dumps([metadata])
             index_meta_doc = Document(page_content=f"{IndexerKeywords.INDEX_META_TYPE.value}_{index_name}", metadata=metadata)
             add_documents(vectorstore=self.vectorstore, documents=[index_meta_doc])
 
-    def index_meta_update(self, index_name: str, state: str, result: int):
+    def index_meta_update(self, index_name: str, state: str, result: int, update_force: bool = True, interval: Optional[float] = None):
+        """Update `index_meta` document with optional time-based throttling.
+
+        Args:
+            index_name: Index name to update meta for.
+            state: New state value for the `index_meta` record.
+            result: Number of processed documents to store in the `updated` field.
+            update_force: If `True`, perform the update unconditionally, ignoring throttling.
+                          If `False`, perform the update only when the effective time interval has passed.
+            interval: Optional custom interval (in seconds) for this call when `update_force` is `False`.
+                      If `None`, falls back to the value stored in `self._index_meta_config["update_interval"]`
+                      if present, otherwise uses `INDEX_META_UPDATE_INTERVAL`.
+        """
+        self._ensure_vectorstore_initialized()
+        if not hasattr(self, "_index_meta_last_update_time"):
+            self._index_meta_last_update_time: Dict[str, float] = {}
+
+        if not update_force:
+            # Resolve effective interval:
+            # 1\) explicit arg
+            # 2\) value from `_index_meta_config`
+            # 3\) default constant
+            cfg_interval = None
+            if hasattr(self, "_index_meta_config"):
+                cfg_interval = self._index_meta_config.get("update_interval")
+
+            eff_interval = (
+                interval
+                if interval is not None
+                else (cfg_interval if cfg_interval is not None else INDEX_META_UPDATE_INTERVAL)
+            )
+
+            last_time = self._index_meta_last_update_time.get(index_name)
+            now = time.time()
+            if last_time is not None and (now - last_time) < eff_interval:
+                return
+            self._index_meta_last_update_time[index_name] = now
+        else:
+            # For forced updates, always refresh last update time
+            self._index_meta_last_update_time[index_name] = time.time()
+
         index_meta_raw = super().get_index_meta(index_name)
         from ..runtime.langchain.interfaces.llm_processor import add_documents
         #
@@ -545,11 +623,12 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         event_data = {
             "id": index_meta.get("id"),
             "index_name": index_name,
-            "state": metadata.get("state"),
+            "state": "failed" if error is not None else metadata.get("state"),
             "error": error,
             "reindex": is_reindex,
             "indexed": metadata.get("indexed", 0),
             "updated": metadata.get("updated", 0),
+            "toolkit_id": metadata.get("toolkit_id"),
         }
         
         # Emit the event
@@ -572,8 +651,8 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         """
         return [
             {
-                "name": "index_data",
-                "mode": "index_data",
+                "name": IndexTools.INDEX_DATA.value,
+                "mode": IndexTools.INDEX_DATA.value,
                 "ref": self.index_data,
                 "description": "Loads data to index.",
                 "args_schema": create_model(
@@ -583,36 +662,36 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
                 )
             },
             {
-                "name": "search_index",
-                "mode": "search_index",
+                "name": IndexTools.SEARCH_INDEX.value,
+                "mode": IndexTools.SEARCH_INDEX.value,
                 "ref": self.search_index,
                 "description": self.search_index.__doc__,
                 "args_schema": BaseSearchParams
             },
             {
-                "name": "stepback_search_index",
-                "mode": "stepback_search_index",
+                "name": IndexTools.STEPBACK_SEARCH_INDEX.value,
+                "mode": IndexTools.STEPBACK_SEARCH_INDEX.value,
                 "ref": self.stepback_search_index,
                 "description": self.stepback_search_index.__doc__,
                 "args_schema": BaseStepbackSearchParams
             },
             {
-                "name": "stepback_summary_index",
-                "mode": "stepback_summary_index",
+                "name": IndexTools.STEPBACK_SUMMARY_INDEX.value,
+                "mode": IndexTools.STEPBACK_SUMMARY_INDEX.value,
                 "ref": self.stepback_summary_index,
                 "description": self.stepback_summary_index.__doc__,
                 "args_schema": BaseStepbackSearchParams
             },
             {
-                "name": "remove_index",
-                "mode": "remove_index",
+                "name": IndexTools.REMOVE_INDEX.value,
+                "mode": IndexTools.REMOVE_INDEX.value,
                 "ref": self.remove_index,
                 "description": self.remove_index.__doc__,
                 "args_schema": RemoveIndexParams
             },
             {
-                "name": "list_collections",
-                "mode": "list_collections",
+                "name": IndexTools.LIST_COLLECTIONS.value,
+                "mode": IndexTools.LIST_COLLECTIONS.value,
                 "ref": self.list_collections,
                 "description": self.list_collections.__doc__,
                 # No parameters

alita_sdk/tools/bitbucket/__init__.py

@@ -47,8 +47,8 @@ class AlitaBitbucketToolkit(BaseToolkit):
         AlitaBitbucketToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         m = create_model(
             name,
-            project=(str, Field(description="Project/Workspace", json_schema_extra={'configuration': True})),
-            repository=(str, Field(description="Repository", json_schema_extra={'max_toolkit_length': AlitaBitbucketToolkit.toolkit_max_length, 'configuration': True})),
+            project=(str, Field(description="Project/Workspace")),
+            repository=(str, Field(description="Repository")),
             branch=(str, Field(description="Main branch", default="main")),
             cloud=(Optional[bool], Field(description="Hosting Option", default=None)),
             bitbucket_configuration=(BitbucketConfiguration, Field(description="Bitbucket Configuration", json_schema_extra={'configuration_types': ['bitbucket']})),

alita_sdk/tools/chunkers/__init__.py

@@ -3,6 +3,7 @@ from .sematic.statistical_chunker import statistical_chunker
 from .sematic.markdown_chunker import markdown_chunker
 from .sematic.proposal_chunker import proposal_chunker
 from .sematic.json_chunker import json_chunker
+from .universal_chunker import universal_chunker, chunk_single_document, get_file_type
 from .models import StatisticalChunkerConfig, MarkdownChunkerConfig, ProposalChunkerConfig
 
 __all__ = {
@@ -10,7 +11,8 @@ __all__ = {
     'statistical': statistical_chunker,
     'markdown': markdown_chunker,
     'proposal': proposal_chunker,
-    'json': json_chunker
+    'json': json_chunker,
+    'universal': universal_chunker,
 }
 
 __confluence_chunkers__ = {

alita_sdk/tools/chunkers/sematic/json_chunker.py

@@ -17,6 +17,7 @@ def json_chunker(file_content_generator: Generator[Document, None, None], config
             for chunk in chunks:
                 metadata = doc.metadata.copy()
                 metadata['chunk_id'] = chunk_id
+                metadata['method_name'] = 'json'
                 chunk_id += 1
                 yield Document(page_content=json.dumps(chunk), metadata=metadata)
         except Exception as e: