alita-sdk 0.3.204__py3-none-any.whl → 0.3.206__py3-none-any.whl

alita_sdk/runtime/clients/client.py

@@ -10,6 +10,7 @@ from langchain_core.messages import (
 )
 from langchain_core.tools import ToolException
 from langgraph.store.base import BaseStore
+from langchain_openai import OpenAIEmbeddings, ChatOpenAI
 
 from ..langchain.assistant import Assistant as LangChainAssistant
 # from ..llamaindex.assistant import Assistant as LLamaAssistant
@@ -37,6 +38,7 @@ class AlitaClient:
 
         self.base_url = base_url.rstrip('/')
         self.api_path = '/api/v1'
+        self.llm_path = '/llm/v1'
         self.project_id = project_id
         self.auth_token = auth_token
         self.headers = {
@@ -152,6 +154,35 @@ class AlitaClient:
             return resp.json()
         return []
 
+    def get_llm(self, model_name: str, model_config: dict) -> ChatOpenAI:
+        """
+        Get a ChatOpenAI model instance based on the model name and configuration.
+        
+        Args:
+            model_name: Name of the model to retrieve
+            model_config: Configuration parameters for the model
+            
+        Returns:
+            An instance of ChatOpenAI configured with the provided parameters.
+        """
+        if not model_name:
+            raise ValueError("Model name must be provided")
+        
+        logger.info(f"Creating ChatOpenAI model: {model_name} with config: {model_config}")
+        
+        return ChatOpenAI(
+            base_url=f"{self.base_url}{self.llm_path}",
+            model=model_name,
+            api_key=self.auth_token,
+            stream_usage=model_config.get("stream_usage", True),
+            max_tokens=model_config.get("max_tokens", None),
+            top_p=model_config.get("top_p"),
+            temperature=model_config.get("temperature"),
+            max_retries=model_config.get("max_retries", 3),
+            seed=model_config.get("seed", None),
+        )
+         
+    
     def get_app_version_details(self, application_id: int, application_version_id: int) -> dict:
         url = f"{self.application_versions}/{application_id}/{application_version_id}"
         if self.configurations:
@@ -177,11 +208,12 @@ class AlitaClient:
         logger.info(f"Unsecret response: {data}")
         return data.get('value', None)
 
-    def application(self, client: Any, application_id: int, application_version_id: int,
+    def application(self, application_id: int, application_version_id: int,
                     tools: Optional[list] = None, chat_history: Optional[List[Any]] = None,
                     app_type=None, memory=None, runtime='langchain',
                     application_variables: Optional[dict] = None,
-                    version_details: Optional[dict] = None, store: Optional[BaseStore] = None):
+                    version_details: Optional[dict] = None, store: Optional[BaseStore] = None,
+                    llm: Optional[ChatOpenAI] = None):
         if tools is None:
             tools = []
         if chat_history is None:
@@ -200,7 +232,15 @@ class AlitaClient:
             for var in data.get('variables', {}):
                 if var['name'] in application_variables:
                     var.update(application_variables[var['name']])
-
+        if llm is None:
+            llm = self.get_llm(
+                model_name=data['llm_settings']['model_name'],
+                model_config = {
+                    "max_tokens": data['llm_settings']['max_tokens'],
+                    "top_p": data['llm_settings']['top_p'],
+                    "temperature": data['llm_settings']['temperature']
+                }
+            )
         if not app_type:
             app_type = data.get("agent_type", "react")
         if app_type == "alita":
@@ -212,10 +252,10 @@ class AlitaClient:
         elif app_type == 'autogen':
             app_type = "openai"
         if runtime == 'nonrunnable':
-            return LangChainAssistant(self, data, client, chat_history, app_type,
+            return LangChainAssistant(self, data, llm, chat_history, app_type,
                                       tools=tools, memory=memory, store=store)
         if runtime == 'langchain':
-            return LangChainAssistant(self, data, client,
+            return LangChainAssistant(self, data, llm,
                                       chat_history, app_type,
                                       tools=tools, memory=memory, store=store).runnable()
         elif runtime == 'llama':

alita_sdk/runtime/langchain/assistant.py

@@ -36,29 +36,30 @@ class Assistant:
 
         logger.debug("Data for agent creation: %s", data)
         logger.info("App type: %s", app_type)
-
+        
+        self.client = client
         # For predict agents, use the client as-is since it's already configured
-        if app_type == "predict":
-            self.client = client
-        else:
-            # For other agent types, configure client from llm_settings
-            self.client = copy(client)
-            self.client.max_tokens = data['llm_settings']['max_tokens']
-            self.client.temperature = data['llm_settings']['temperature']
-            self.client.top_p = data['llm_settings']['top_p']
-            self.client.top_k = data['llm_settings']['top_k']
-            self.client.model_name = data['llm_settings']['model_name']
-            self.client.integration_uid = data['llm_settings']['integration_uid']
+        # if app_type == "predict":
+        #     self.client = client
+        # else:
+        #     # For other agent types, configure client from llm_settings
+        #     self.client = copy(client)
+        #     self.client.max_tokens = data['llm_settings']['max_tokens']
+        #     self.client.temperature = data['llm_settings']['temperature']
+        #     self.client.top_p = data['llm_settings']['top_p']
+        #     self.client.top_k = data['llm_settings']['top_k']
+        #     self.client.model_name = data['llm_settings']['model_name']
+        #     self.client.integration_uid = data['llm_settings']['integration_uid']
 
-            model_type = data["llm_settings"]["indexer_config"]["ai_model"]
-            model_params = data["llm_settings"]["indexer_config"]["ai_model_params"]
-            #
-            target_pkg, target_name = model_type.rsplit(".", 1)
-            target_cls = getattr(
-                importlib.import_module(target_pkg),
-                target_name
-            )
-            self.client = target_cls(**model_params)
+        #     model_type = data["llm_settings"]["indexer_config"]["ai_model"]
+        #     model_params = data["llm_settings"]["indexer_config"]["ai_model_params"]
+        #     #
+        #     target_pkg, target_name = model_type.rsplit(".", 1)
+        #     target_cls = getattr(
+        #         importlib.import_module(target_pkg),
+        #         target_name
+        #     )
+        #     self.client = target_cls(**model_params)
         # validate agents compatibility: non-pipeline agents cannot have pipelines as toolkits
         if app_type not in ["pipeline", "predict"]:
             tools_to_check = data.get('tools', [])

alita_sdk/runtime/langchain/interfaces/llm_processor.py

@@ -50,9 +50,6 @@ def get_model(model_type: str, model_params: dict):
         return get_llm(model_type)(**model_params)
     if model_type == "PreloadedChatModel":
         return PreloadedChatModel(**model_params)
-    if model_type == "Alita":
-        from ...llms.alita import AlitaClient
-        return AlitaClient(**model_params)
     if model_type in chat_models:
         model = getattr(
             __import__("langchain_community.chat_models", fromlist=[model_type]),
@@ -185,7 +182,7 @@ def add_documents(vectorstore, documents):
         texts.append(document.page_content)
         for key in document.metadata:
             if isinstance(document.metadata[key], list):
-                document.metadata[key] = "; ".join(document.metadata[key])
+                document.metadata[key] = "; ".join([str(val) for val in document.metadata[key]])
             if isinstance(document.metadata[key], dict):
                 document.metadata[key] = dumps(document.metadata[key])
         metadata.append(document.metadata)

alita_sdk/runtime/toolkits/application.py

@@ -21,25 +21,20 @@ class ApplicationToolkit(BaseToolkit):
         )
     
     @classmethod
-    def get_toolkit(cls, client: Any, application_id: int, application_version_id: int, app_api_key: str,
+    def get_toolkit(cls, client: 'AlitaClient', application_id: int, application_version_id: int,
                     selected_tools: list[str] = [], store: Optional[BaseStore] = None):
-        from ..llms.alita import AlitaChatModel
         
         app_details = client.get_app_details(application_id)
         version_details = client.get_app_version_details(application_id, application_version_id)
-        settings = {
-            "deployment": client.base_url,
-            "model": version_details['llm_settings']['model_name'],
-            "api_key": app_api_key,
-            "project_id": client.project_id,
-            "integration_uid": version_details['llm_settings']['integration_uid'],
+        model_settings = {
             "max_tokens": version_details['llm_settings']['max_tokens'],
             "top_p": version_details['llm_settings']['top_p'],
-            "top_k": version_details['llm_settings']['top_k'],
             "temperature": version_details['llm_settings']['temperature'],
         }
 
-        app = client.application(AlitaChatModel(**settings), application_id, application_version_id, store=store)
+        app = client.application(application_id, application_version_id, store=store, 
+                                 llm=client.get_llm(version_details['llm_settings']['model_name'], 
+                                                    model_settings))
         return cls(tools=[Application(name=app_details.get("name"), 
                                       description=app_details.get("description"), 
                                       application=app, 

alita_sdk/runtime/toolkits/tools.py

@@ -54,7 +54,6 @@ def get_tools(tools_list: list, alita_client, llm, memory_store: BaseStore = Non
                 alita_client,
                 application_id=int(tool['settings']['application_id']),
                 application_version_id=int(tool['settings']['application_version_id']),
-                app_api_key=alita_client.auth_token,
                 selected_tools=[]
             ).get_tools())
         elif tool['type'] == 'application' and tool.get('agent_type', '') == 'pipeline':

alita_sdk/runtime/tools/vectorstore.py

@@ -1,13 +1,18 @@
 import json
-from json import dumps
-from typing import Any, Optional, List, Dict
+import math
+import types
+from typing import Any, Optional, List, Dict, Callable, Generator
+
+from langchain_core.documents import Document
 from pydantic import BaseModel, model_validator, Field
-from langchain_core.tools import ToolException
 from ..langchain.tools.vector import VectorAdapter
 from langchain_core.messages import HumanMessage
 from alita_sdk.tools.elitea_base import BaseToolApiWrapper
 from logging import getLogger
 
+from ..utils.logging import dispatch_custom_event
+from ..utils.utils import IndexerKeywords
+
 logger = getLogger(__name__)
 
 class IndexDocumentsModel(BaseModel):
@@ -139,6 +144,7 @@ class VectorStoreWrapper(BaseToolApiWrapper):
     vectoradapter: Any = None
     pg_helper: Any = None
     embeddings: Any = None
+    process_document_func: Optional[Callable] = None
     
     @model_validator(mode='before')
     @classmethod
@@ -182,18 +188,132 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             except Exception as e:
                 logger.error(f"Failed to initialize PGVectorSearch: {str(e)}")
 
-    def index_documents(self, documents):
+    def _get_indexed_data(self, store):
+        """ Get all indexed data from vectorstore """
+
+        # get already indexed data
+        result = {}
+        try:
+            self._log_data("Retrieving already indexed data from vectorstore",
+                           tool_name="index_documents")
+            data = store.get(include=['documents', 'metadatas'])
+            # re-structure data to be more usable
+            for doc_str, meta, db_id in zip(data['documents'], data['metadatas'], data['ids']):
+                doc = json.loads(doc_str)
+                doc_id = str(meta['id'])
+                dependent_docs = meta.get(IndexerKeywords.DEPENDENT_DOCS.value, [])
+                parent_id = meta.get(IndexerKeywords.PARENT.value, -1)
+                result[doc_id] = {
+                    'metadata': meta,
+                    'document': doc,
+                    'id': db_id,
+                    IndexerKeywords.DEPENDENT_DOCS.value: dependent_docs,
+                    IndexerKeywords.PARENT.value: parent_id
+                }
+        except Exception as e:
+            logger.error(f"Failed to get indexed data from vectorstore: {str(e)}. Continuing with empty index.")
+        return result
+
+    def _reduce_duplicates(self, documents: Generator[Document, None, None], store) -> List[Any]:
+        """Remove documents already indexed in the vectorstore based on metadata 'id' and 'updated_on' fields."""
+
+        self._log_data("Verification of documents to index started", tool_name="index_documents")
+
+        indexed_data = self._get_indexed_data(store)
+        indexed_ids = set(indexed_data.keys())
+        if not indexed_ids:
+            self._log_data("Vectorstore is empty, indexing all incoming documents", tool_name="index_documents")
+            return list(documents)
+
+        final_docs = []
+        docs_to_remove = []
+
+        for document in documents:
+            doc_id = document.metadata.get('id')
+            # get document's metadata and id and check if already indexed
+            if doc_id in indexed_ids:
+                # document has been indexed already, then verify `updated_on`
+                to_index_updated_on = document.metadata.get('updated_on')
+                indexed_meta = indexed_data[doc_id]['metadata']
+                indexed_updated_on = indexed_meta.get('updated_on')
+                if to_index_updated_on and indexed_updated_on and to_index_updated_on == indexed_updated_on:
+                    # same updated_on, skip indexing
+                    continue
+                # if updated_on is missing or different, we will re-index the document and remove old one
+                # parent doc removal
+                docs_to_remove.append(indexed_data[doc_id]['id'])
+                # mark dependent docs for removal
+                for dependent_doc_id in indexed_data[doc_id][IndexerKeywords.DEPENDENT_DOCS.value]:
+                    docs_to_remove.append(indexed_data[dependent_doc_id]['id'])
+            else:
+                final_docs.append(document)
+
+        if docs_to_remove:
+            self._log_data(
+                f"Removing {len(docs_to_remove)} documents from vectorstore that are already indexed with different updated_on.",
+                tool_name="index_documents"
+            )
+            store.delete(ids=docs_to_remove)
+
+        return final_docs
+
+    def index_documents(self, documents: Generator[Document, None, None], progress_step: int = 20, clean_index: bool = True):
+        """ Index documents in the vectorstore.
+
+        Args:
+            documents (Any): Generator or list of documents to index.
+            document_processing_func (Optional[Callable]): Function to process documents after duplicates removal and before indexing.
+            progress_step (int): Step for progress reporting, default is 20.
+            clean_index (bool): If True, clean the index before re-indexing all documents.
+        """
+
         from ..langchain.interfaces.llm_processor import add_documents
+
+        # pre-process documents if needed (find duplicates, etc.)
+        if clean_index:
+            logger.info("Cleaning index before re-indexing all documents.")
+            self._log_data("Cleaning index before re-indexing all documents. Previous index will be removed", tool_name="index_documents")
+            try:
+                self.vectoradapter.delete_dataset(self.dataset)
+                self.vectoradapter.persist()
+                self.vectoradapter.vacuum()
+                self._log_data("Previous index has been removed",
+                               tool_name="index_documents")
+            except Exception as e:
+                logger.warning(f"Failed to clean index: {str(e)}. Continuing with re-indexing.")
+            if isinstance(documents, types.GeneratorType):
+                documents = list(documents)
+        else:
+            # remove duplicates based on metadata 'id' and 'updated_on' fields
+            documents = self._reduce_duplicates(documents, self.vectoradapter.vectorstore)
+
+
+        if not documents or len(documents) == 0:
+            logger.info("No new documents to index after duplicate check.")
+            return {"status": "ok", "message": "No new documents to index."}
+
+        # if func is provided, apply it to documents
+        # used for processing of documents before indexing,
+        # e.g. to avoid time-consuming operations for documents that are already indexed
+        dependent_docs_generator = self.process_document_func(documents) if self.process_document_func else []
+
+        # notify user about missed required metadata fields: id, updated_on
+        # it is not required to have them, but it is recommended to have them for proper re-indexing and duplicate detection
+        for doc in documents:
+            if 'id' not in doc.metadata or 'updated_on' not in doc.metadata:
+                logger.warning(f"Document is missing required metadata field 'id' or 'updated_on': {doc.metadata}")
+
         logger.debug(f"Indexing documents: {documents}")
         logger.debug(self.vectoradapter)
-        self.vectoradapter.delete_dataset(self.dataset)
-        self.vectoradapter.persist()
-        logger.debug(f"Deleted Dataset")
-        #
-        self.vectoradapter.vacuum()
-        #
+
+        documents = documents + list(dependent_docs_generator)
+        total_docs = len(documents)
         documents_count = 0
         _documents = []
+
+        # set default progress step to 20 if out of 0...100 or None
+        progress_step = 20 if progress_step not in range(0, 100) else progress_step
+        next_progress_point = progress_step
         for document in documents:
             documents_count += 1
             # logger.debug(f"Indexing document: {document}")
@@ -203,7 +323,14 @@ class VectorStoreWrapper(BaseToolApiWrapper):
                     add_documents(vectorstore=self.vectoradapter.vectorstore, documents=_documents)
                     self.vectoradapter.persist()
                     _documents = []
-            except Exception as e:
+
+                percent = math.floor((documents_count / total_docs) * 100)
+                if percent >= next_progress_point:
+                    msg = f"Indexing progress: {percent}%. Processed {documents_count} of {total_docs} documents."
+                    logger.debug(msg)
+                    self._log_data(msg)
+                    next_progress_point += progress_step
+            except Exception:
                 from traceback import format_exc
                 logger.error(f"Error: {format_exc()}")
                 return {"status": "error", "message": f"Error: {format_exc()}"}
@@ -383,9 +510,11 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             combined_items = [item for item in combined_items if abs(item[1]) >= cut_off]
         
         # Sort by score and limit results
-        combined_items.sort(key=lambda x: x[1], reverse=True)
+
+        # for chroma we want ascending order (lower score is better), for others descending
+        combined_items.sort(key=lambda x: x[1], reverse= self.vectorstore_type.lower() != 'chroma')
         combined_items = combined_items[:search_top]
-        
+
         # Format output based on doctype
         if doctype == 'code':
             return code_format(combined_items)
@@ -498,6 +627,21 @@ class VectorStoreWrapper(BaseToolApiWrapper):
         ])
         return result.content
 
+    def _log_data(self, message: str, tool_name: str = "index_data"):
+        """Log data and dispatch custom event for indexing progress"""
+
+        try:
+            dispatch_custom_event(
+                name="thinking_step",
+                data={
+                    "message": message,
+                    "tool_name": tool_name,
+                    "toolkit": "vectorstore",
+                },
+            )
+        except Exception as e:
+            logger.warning(f"Failed to dispatch progress event: {str(e)}")
+
     def get_available_tools(self):
         return [
             {

alita_sdk/runtime/utils/streamlit.py

@@ -65,7 +65,8 @@ def pil_to_base64_string(pil_image):
         return None
 
 
-from alita_sdk.runtime.llms.alita import AlitaChatModel
+# from alita_sdk.runtime.llms.alita import AlitaChatModel
+from alita_sdk.runtime.clients.client import AlitaClient
 from alita_sdk.runtime.utils.AlitaCallback import AlitaStreamlitCallback
 from alita_sdk.runtime.toolkits.tools import get_toolkits, get_tools
 from alita_sdk.community.utils import check_schema
@@ -369,7 +370,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                 st.rerun()
         
         # Determine login form title and expansion state
-        if st.session_state.llm:
+        if st.session_state.client:
             login_title = "✅ Elitea Login (Connected)"
             # Collapse after successful login, but allow expansion
             if st.session_state.login_form_expanded is True:
@@ -385,13 +386,13 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
             deployment_secret = environ.get('XSECRET', 'secret')
             api_key_value = environ.get('API_KEY', None)
             project_id_value = int(environ.get('PROJECT_ID', 0))
-            if st.session_state.llm:
-                deployment_value = st.session_state.llm.deployment
-                api_key_value = st.session_state.llm.api_token
-                project_id_value = st.session_state.llm.project_id
-                
+            if st.session_state.client:
+                deployment_value = st.session_state.client.base_url
+                api_key_value = st.session_state.client.auth_token
+                project_id_value = st.session_state.client.project_id
+
             # Show current connection status
-            if st.session_state.llm:
+            if st.session_state.client:
                 st.success(f"Connected to: {deployment_value}")
                 st.info(f"Project ID: {project_id_value}")
                 
@@ -403,22 +404,24 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                 deployment_secret = st.text_input("Deployment Secret", placeholder="Enter Deployment Secret", value=deployment_secret)
                 
                 # Change button text based on login status
-                button_text = "Re-Login" if st.session_state.llm else "Login"
+                button_text = "Re-Login" if st.session_state.client else "Login"
                 submitted = st.form_submit_button(button_text)
                 
                 if submitted:
                     with st.spinner("Logging to Alita..."):
                         try:
-                            st.session_state.llm = AlitaChatModel(**{
-                                    "deployment": deployment,
-                                    "api_token": api_key,
-                                    "project_id": project_id,
-                                })
-                            client = st.session_state.llm.client
+
+                            st.session_state.client = AlitaClient(
+                                base_url=deployment,
+                                project_id=project_id,
+                                auth_token=api_key,
+                                api_extra_headers={"X-SECRET": deployment_secret}
+                            )
+                            
                             
                             # Fetch specific project secret for pgvector connection
                             try:
-                                pgvector_connstr = client.unsecret('pgvector_project_connstr')
+                                pgvector_connstr = st.session_state.client.unsecret('pgvector_project_connstr')
                                 if pgvector_connstr:
                                     st.session_state.project_secrets = {'pgvector_project_connstr': pgvector_connstr}
                                     logger.info("Successfully retrieved pgvector connection string from project secrets")
@@ -429,7 +432,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                                 logger.warning(f"Could not retrieve pgvector connection string: {str(e)}")
                                 st.session_state.project_secrets = {}
                             
-                            integrations = client.all_models_and_integrations()
+                            integrations = st.session_state.client.all_models_and_integrations()
                             unique_models = set()
                             models_list = []
                             for entry in integrations:
@@ -438,7 +441,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                                     if model.get('capabilities', {}).get('chat_completion') and model['name'] not in unique_models:
                                         unique_models.add(model['name'])
                                         models_list.append({'name': model['name'], 'integration_id': entry['uid']})
-                            st.session_state.agents = client.get_list_of_apps()
+                            st.session_state.agents = st.session_state.client.get_list_of_apps()
                             st.session_state.models = models_list
                             clear_chat_history()
                             
@@ -454,6 +457,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                         except Exception as e:
                             logger.error(f"Error loggin to ELITEA: {format_exc()}")
                             st.session_state.agents = None
+                            st.session_state.client = None
                             st.session_state.models = None
                             st.session_state.llm = None
                             st.session_state.project_secrets = None
@@ -463,7 +467,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
         llmconfig, toolkit_config = st.tabs(["Alita Agents", "Toolkit Testing"])
         
         with llmconfig:
-            if st.session_state.llm:
+            if st.session_state.client:
                 st.title("Available Agents")
                 st.write("This one will load latest version of agent")
                 with st.form("agents_form", clear_on_submit=False):
@@ -477,7 +481,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                             agent = next((a for a in st.session_state.agents if a['name'] == options), None)
                             if agent:
                                 agent_id = agent['id']
-                                agent_details = st.session_state.llm.client.get_app_details(agent_id)
+                                agent_details = st.session_state.client.get_app_details(agent_id)
                                 latest_version = next((v for v in agent_details['versions'] if v['name'] == agent_version_name), None)
                                 if latest_version:
                                     agent_version_id = latest_version['id']
@@ -504,11 +508,11 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                                         
                                         # Try to get the complete agent configuration
                                         try:
-                                            agent_version_details = st.session_state.llm.client.get_app_version_details(agent_id, agent_version_id)
+                                            agent_version_details = st.session_state.client.get_app_version_details(agent_id, agent_version_id)
                                             agent_full_config = agent_version_details
                                         except AttributeError:
                                             try:
-                                                agent_version_details = st.session_state.llm.client.get_application_version_details(agent_id, agent_version_id)
+                                                agent_version_details = st.session_state.client.get_application_version_details(agent_id, agent_version_id)
                                                 agent_full_config = agent_version_details
                                             except AttributeError:
                                                 # Use the version details we already have
@@ -630,8 +634,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                                         st.session_state.agent_toolkit_configs = {}
                                         st.session_state.agent_raw_config = None
                                     
-                                    st.session_state.agent_executor = st.session_state.llm.client.application(
-                                        client=st.session_state.llm,
+                                    st.session_state.agent_executor = st.session_state.client.application(
                                         application_id=agent_id,
                                         application_version_id=agent_version_id,
                                         app_type=agent_type if agent_type else None,
@@ -658,7 +661,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
             """)
             
             # Check if user is logged in
-            if not st.session_state.llm:
+            if not st.session_state.client:
                 st.warning("⚠️ **Please log in first!**")
                 st.info("""
                 📋 **To use Toolkit Testing:**
@@ -671,7 +674,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                 st.stop()
             
             # User is logged in, proceed with toolkit testing
-            if st.session_state.llm:
+            if st.session_state.client:
                 # Show project secrets status with detailed debugging
                 secrets_status = st.session_state.project_secrets
                 
@@ -685,7 +688,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                 # Debug info (can be removed later)
                 with st.expander("🔍 Debug Info", expanded=False):
                     st.write(f"**Project Secrets Status:** {type(secrets_status)} - {secrets_status}")
-                    st.write(f"**LLM Status:** {'Connected' if st.session_state.llm else 'Not Connected'}")
+                    # st.write(f"**LLM Status:** {'Connected' if st.session_state.llm else 'Not Connected'}")
                 
                 # Toolkit selection and configuration
                 st.markdown("---")
@@ -1021,7 +1024,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                 st.markdown("👈 Please use the **Alita Login Form** in the sidebar to authenticate.")
                         
     # Main content area
-    if st.session_state.llm and st.session_state.agent_executor and st.session_state.agent_chat:
+    if st.session_state.client and st.session_state.agent_executor and st.session_state.agent_chat:
         try:
             st.title(st.session_state.agent_name)
         except:
@@ -1043,7 +1046,7 @@ def run_streamlit(st, ai_icon=None, user_icon=None):
                 st.session_state.thread_id = response.get("thread_id", None)
                 st.session_state.messages.append({"role": "assistant", "content": response["output"]})
     
-    elif st.session_state.llm and st.session_state.show_toolkit_testing and st.session_state.configured_toolkit:
+    elif st.session_state.client and st.session_state.show_toolkit_testing and st.session_state.configured_toolkit:
         # Toolkit Testing Main View
         st.title("🚀 Toolkit Testing Interface")
         
@@ -1281,7 +1284,7 @@ Please explain how you would use these tools to help the user, even though I can
                 st.rerun()
     
     else:
-        if st.session_state.llm:
+        if st.session_state.client:
             st.title("🎯 Alita SDK Toolkit Interface")
             st.markdown("""
             ### Welcome to the Alita SDK!

alita_sdk/runtime/utils/utils.py

@@ -1,7 +1,12 @@
 import re
+from enum import Enum
 
 TOOLKIT_SPLITTER = "___"
 
+class IndexerKeywords(Enum):
+    DEPENDENT_DOCS = 'dependent_docs'
+    PARENT = 'parent_id'
+
 # This pattern matches characters that are NOT alphanumeric, underscores, or hyphens
 clean_string_pattern = re.compile(r'[^a-zA-Z0-9_.-]')
 

alita_sdk/tools/__init__.py

@@ -2,6 +2,7 @@ import logging
 from importlib import import_module
 from typing import Optional
 
+from langchain_core.tools import ToolException
 from langgraph.store.base import BaseStore
 
 logger = logging.getLogger(__name__)
@@ -80,6 +81,8 @@ _safe_import_tool('postman', 'postman', 'get_tools', 'PostmanToolkit')
 _safe_import_tool('memory', 'memory', 'get_tools', 'MemoryToolkit')
 _safe_import_tool('zephyr_squad', 'zephyr_squad', 'get_tools', 'ZephyrSquadToolkit')
 _safe_import_tool('slack', 'slack', 'get_tools', 'SlackToolkit')
+_safe_import_tool('bigquery', 'google.bigquery', 'get_tools', 'BigQueryToolkit')
+_safe_import_tool('delta_lake', 'aws.delta_lake', 'get_tools', 'DeltaLakeToolkit')
 
 # Log import summary
 available_count = len(AVAILABLE_TOOLS)
@@ -111,6 +114,7 @@ def get_tools(tools_list, alita, llm, store: Optional[BaseStore] = None, *args,
 
             except Exception as e:
                 logger.error(f"Error getting tools for {tool_type}: {e}")
+                raise ToolException(f"Error getting tools for {tool_type}: {e}")
 
         # Handle ADO repos special case (it might be requested as azure_devops_repos)
         elif tool_type in ['ado_repos', 'azure_devops_repos'] and 'ado_repos' in AVAILABLE_TOOLS: