MindsDB 25.5.3.0__py3-none-any.whl → 25.5.4.0__py3-none-any.whl

mindsdb/integrations/utilities/handler_utils.py

@@ -43,6 +43,11 @@ def get_api_key(
     if f"{api_name.lower()}_api_key" in create_args:
         return create_args[f"{api_name.lower()}_api_key"]
 
+    # 2.5 - Check in params dictionary if it exists (for agents)
+    if "params" in create_args and create_args["params"] is not None:
+        if f"{api_name.lower()}_api_key" in create_args["params"]:
+            return create_args["params"][f"{api_name.lower()}_api_key"]
+
     # 3
     if engine_storage is not None:
         connection_args = engine_storage.get_connection_args()
@@ -68,7 +73,14 @@ def get_api_key(
         return create_args['api_keys'][api_name]
 
     if strict:
-        raise Exception(
-            f"Missing API key '{api_name.lower()}_api_key'. Either re-create this ML_ENGINE specifying the '{api_name.lower()}_api_key' parameter, or re-create this model and pass the API key with `USING` syntax."
-        )  # noqa
+        provider_upper = api_name.upper()
+        api_key_env_var = f"{provider_upper}_API_KEY"
+        api_key_arg = f"{api_name.lower()}_api_key"
+        error_message = (
+            f"API key for {api_name} not found. Please provide it using one of the following methods:\n"
+            f"1. Set the {api_key_env_var} environment variable\n"
+            f"2. Provide it as '{api_key_arg}' parameter when creating an agent using the CREATE AGENT syntax\n"
+            f"   Example: CREATE AGENT my_agent USING model='gpt-4', provider='{api_name}', {api_key_arg}='your-api-key';\n"
+        )
+        raise Exception(error_message)
     return None

mindsdb/integrations/utilities/handlers/api_utilities/__init__.py

@@ -1 +0,0 @@
-from .microsoft import MSGraphAPIBaseClient

mindsdb/integrations/utilities/handlers/auth_utilities/__init__.py

@@ -1,2 +0,0 @@
-from .google import GoogleUserOAuth2Manager, GoogleServiceAccountOAuth2Manager
-from .microsoft import MSGraphAPIApplicationPermissionsManager, MSGraphAPIDelegatedPermissionsManager

mindsdb/integrations/utilities/utils.py

@@ -8,7 +8,7 @@ def format_exception_error(exception):
         exception_type, _exception_object, exception_traceback = sys.exc_info()
         filename = exception_traceback.tb_frame.f_code.co_filename
         line_number = exception_traceback.tb_lineno
-        error_message = f'{exception_type.__name__}: {exception}, raised at: {filename}#{line_number}'
+        error_message = f"{exception_type.__name__}: {exception}, raised at: {filename}#{line_number}"
     except Exception:
         error_message = str(exception)
     return error_message
@@ -26,7 +26,7 @@ def dict_to_yaml(d, indent=0):
 
 
 # Mocks won't always have 'name' attribute.
-def get_class_name(instance: Any, default: str = 'unknown'):
-    if hasattr(instance.__class__, 'name'):
+def get_class_name(instance: Any, default: str = "unknown"):
+    if hasattr(instance.__class__, "name"):
         return instance.__class__.name
     return default

mindsdb/interfaces/agents/agents_controller.py

@@ -19,9 +19,10 @@ from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
 from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS
 from .langchain_agent import get_llm_provider
 
-
 default_project = config.get('default_project')
 
+DEFAULT_TEXT2SQL_DATABASE = 'mindsdb'
+
 
 class AgentsController:
     '''Handles CRUD operations at the database level for Agents'''
@@ -158,6 +159,13 @@ class AgentsController:
                  with one of keys is "name", and other is additional parameters for relationship agent<>skill
             provider (str): The provider of the model
             params (Dict[str, str]): Parameters to use when running the agent
+                database: The database to use for text2sql skills (default is 'mindsdb')
+                knowledge_base_database: The database to use for knowledge base queries (default is 'mindsdb')
+                include_tables: List of tables to include for text2sql skills
+                ignore_tables: List of tables to ignore for text2sql skills
+                include_knowledge_bases: List of knowledge bases to include for text2sql skills
+                ignore_knowledge_bases: List of knowledge bases to ignore for text2sql skills
+                <provider>_api_key: API key for the provider (e.g., openai_api_key)
 
         Returns:
             agent (db.Agents): The created agent
@@ -176,6 +184,128 @@ class AgentsController:
 
         _, provider = self.check_model_provider(model_name, provider)
 
+        # Extract API key if provided in the format <provider>_api_key
+        provider_api_key_param = f"{provider.lower()}_api_key"
+        if provider_api_key_param in params:
+            # Keep the API key in params for the agent to use
+            # It will be picked up by get_api_key() in handler_utils.py
+            pass
+
+        # Extract table and knowledge base parameters from params
+        database = params.pop('database', None)
+        knowledge_base_database = params.pop('knowledge_base_database', DEFAULT_TEXT2SQL_DATABASE)
+        include_tables = params.pop('include_tables', None)
+        ignore_tables = params.pop('ignore_tables', None)
+        include_knowledge_bases = params.pop('include_knowledge_bases', None)
+        ignore_knowledge_bases = params.pop('ignore_knowledge_bases', None)
+
+        # Save the extracted parameters back to params for API responses only if they were explicitly provided
+        # or if they're needed for skills
+        need_params = include_tables or ignore_tables or include_knowledge_bases or ignore_knowledge_bases
+
+        if 'database' in params or need_params:
+            params['database'] = database
+
+        if 'knowledge_base_database' in params or include_knowledge_bases or ignore_knowledge_bases:
+            params['knowledge_base_database'] = knowledge_base_database
+
+        if include_tables is not None:
+            params['include_tables'] = include_tables
+        if ignore_tables is not None:
+            params['ignore_tables'] = ignore_tables
+        if include_knowledge_bases is not None:
+            params['include_knowledge_bases'] = include_knowledge_bases
+        if ignore_knowledge_bases is not None:
+            params['ignore_knowledge_bases'] = ignore_knowledge_bases
+
+        # Convert string parameters to lists if needed
+        if isinstance(include_tables, str):
+            include_tables = [t.strip() for t in include_tables.split(',')]
+        if isinstance(ignore_tables, str):
+            ignore_tables = [t.strip() for t in ignore_tables.split(',')]
+        if isinstance(include_knowledge_bases, str):
+            include_knowledge_bases = [kb.strip() for kb in include_knowledge_bases.split(',')]
+        if isinstance(ignore_knowledge_bases, str):
+            ignore_knowledge_bases = [kb.strip() for kb in ignore_knowledge_bases.split(',')]
+
+        # Auto-create SQL skill if no skills are provided but include_tables or include_knowledge_bases params are provided
+        if not skills and (include_tables or include_knowledge_bases):
+            # Determine database to use (default to 'mindsdb')
+            db_name = database
+            kb_db_name = knowledge_base_database
+
+            # If database is not explicitly provided but tables are, try to extract the database name from the first table
+            if not database and include_tables and len(include_tables) > 0:
+                parts = include_tables[0].split('.')
+                if len(parts) >= 2:
+                    db_name = parts[0]
+            elif not database and ignore_tables and len(ignore_tables) > 0:
+                parts = ignore_tables[0].split('.')
+                if len(parts) >= 2:
+                    db_name = parts[0]
+
+            # Create a default SQL skill
+            skill_name = f"{name}_sql_skill"
+            skill_params = {
+                'type': 'sql',
+                'database': db_name,
+                'description': f'Auto-generated SQL skill for agent {name}'
+            }
+
+            # Add knowledge base database if provided
+            if knowledge_base_database:
+                skill_params['knowledge_base_database'] = knowledge_base_database
+
+            # Add knowledge base parameters if provided
+            if include_knowledge_bases:
+                skill_params['include_knowledge_bases'] = include_knowledge_bases
+            if ignore_knowledge_bases:
+                skill_params['ignore_knowledge_bases'] = ignore_knowledge_bases
+
+            try:
+                # Check if skill already exists
+                existing_skill = self.skills_controller.get_skill(skill_name, project_name)
+                if existing_skill is None:
+                    # Create the skill
+                    skill_type = skill_params.pop('type')
+                    self.skills_controller.add_skill(
+                        name=skill_name,
+                        project_name=project_name,
+                        type=skill_type,
+                        params=skill_params
+                    )
+                else:
+                    # Update the skill if parameters have changed
+                    params_changed = False
+
+                    # Check if database has changed
+                    if existing_skill.params.get('database') != db_name:
+                        existing_skill.params['database'] = db_name
+                        params_changed = True
+
+                    # Check if knowledge base database has changed
+                    if knowledge_base_database and existing_skill.params.get('knowledge_base_database') != kb_db_name:
+                        existing_skill.params['knowledge_base_database'] = kb_db_name
+                        params_changed = True
+
+                    # Check if knowledge base parameters have changed
+                    if include_knowledge_bases and existing_skill.params.get('include_knowledge_bases') != include_knowledge_bases:
+                        existing_skill.params['include_knowledge_bases'] = include_knowledge_bases
+                        params_changed = True
+
+                    if ignore_knowledge_bases and existing_skill.params.get('ignore_knowledge_bases') != ignore_knowledge_bases:
+                        existing_skill.params['ignore_knowledge_bases'] = ignore_knowledge_bases
+                        params_changed = True
+
+                    # Update the skill if needed
+                    if params_changed:
+                        flag_modified(existing_skill, 'params')
+                        db.session.commit()
+
+                skills = [skill_name]
+            except Exception as e:
+                raise ValueError(f"Failed to auto-create or update SQL skill: {str(e)}")
+
         agent = db.Agents(
             name=name,
             project_id=project.id,
@@ -193,10 +323,43 @@ class AgentsController:
             else:
                 parameters = skill.copy()
                 skill_name = parameters.pop('name')
+
             existing_skill = self.skills_controller.get_skill(skill_name, project_name)
             if existing_skill is None:
                 db.session.rollback()
                 raise ValueError(f'Skill with name does not exist: {skill_name}')
+
+            # Add table restrictions if this is a text2sql skill
+            if existing_skill.type == 'sql' and (include_tables or ignore_tables):
+                parameters['tables'] = include_tables or ignore_tables
+
+            # Add knowledge base restrictions if this is a text2sql skill
+            if existing_skill.type == 'sql':
+                # Pass database parameter if provided
+                if database and 'database' not in parameters:
+                    parameters['database'] = database
+
+                # Pass knowledge base database parameter if provided
+                if knowledge_base_database and 'knowledge_base_database' not in parameters:
+                    parameters['knowledge_base_database'] = knowledge_base_database
+
+                # Add knowledge base parameters to both the skill and the association parameters
+                if include_knowledge_bases:
+                    parameters['include_knowledge_bases'] = include_knowledge_bases
+                    if 'include_knowledge_bases' not in existing_skill.params:
+                        existing_skill.params['include_knowledge_bases'] = include_knowledge_bases
+                        flag_modified(existing_skill, 'params')
+                elif ignore_knowledge_bases:
+                    parameters['ignore_knowledge_bases'] = ignore_knowledge_bases
+                    if 'ignore_knowledge_bases' not in existing_skill.params:
+                        existing_skill.params['ignore_knowledge_bases'] = ignore_knowledge_bases
+                        flag_modified(existing_skill, 'params')
+
+                # Ensure knowledge_base_database is set in the skill's params
+                if knowledge_base_database and 'knowledge_base_database' not in existing_skill.params:
+                    existing_skill.params['knowledge_base_database'] = knowledge_base_database
+                    flag_modified(existing_skill, 'params')
+
             association = db.AgentSkillsAssociation(
                 parameters=parameters,
                 agent=agent,

mindsdb/interfaces/agents/constants.py

@@ -202,3 +202,18 @@ DEFAULT_EMBEDDINGS_MODEL_PROVIDER = "openai"
 DEFAULT_EMBEDDINGS_MODEL_CLASS = OpenAIEmbeddings
 DEFAULT_TIKTOKEN_MODEL_NAME = os.getenv('DEFAULT_TIKTOKEN_MODEL_NAME', 'gpt-4')
 AGENT_CHUNK_POLLING_INTERVAL_SECONDS = os.getenv('AGENT_CHUNK_POLLING_INTERVAL_SECONDS', 1.0)
+DEFAULT_TEXT2SQL_DATABASE = "mindsdb"
+DEFAULT_AGENT_SYSTEM_PROMPT = """
+You are an AI assistant powered by MindsDB. When answering questions, follow these guidelines:
+
+1. For factual questions about specific topics, use the knowledge base tools in this sequence:
+   - First use kb_list_tool to see available knowledge bases
+   - Then use kb_info_tool to understand the structure of relevant knowledge bases
+   - Finally use kb_query_tool to query the knowledge base for specific information
+
+2. For questions about database tables and their contents:
+   - Use the sql_tool to query the tables directly
+   - You can join tables if needed to get comprehensive information
+
+For factual questions, ALWAYS use the available tools to look up information rather than relying on your internal knowledge.
+"""

mindsdb/interfaces/agents/langchain_agent.py

@@ -54,7 +54,8 @@ from mindsdb.interfaces.agents.constants import (
     NVIDIA_NIM_CHAT_MODELS,
     USER_COLUMN,
     ASSISTANT_COLUMN,
-    CONTEXT_COLUMN, TRACE_ID_COLUMN
+    CONTEXT_COLUMN, TRACE_ID_COLUMN,
+    DEFAULT_AGENT_SYSTEM_PROMPT
 )
 from mindsdb.interfaces.skills.skill_tool import skill_tool, SkillData
 from langchain_anthropic import ChatAnthropic
@@ -173,6 +174,10 @@ def create_chat_model(args: Dict):
 
 def prepare_prompts(df, base_template, input_variables, user_column=USER_COLUMN):
     empty_prompt_ids = np.where(df[input_variables].isna().all(axis=1).values)[0]
+
+    # Combine system prompt with user-provided template
+    base_template = f"{DEFAULT_AGENT_SYSTEM_PROMPT}\n\n{base_template}"
+
     base_template = base_template.replace('{{', '{').replace('}}', '}')
     prompts = []
 
@@ -194,8 +199,9 @@ def prepare_callbacks(self, args):
     return callbacks, context_callback
 
 
-def handle_agent_error(e):
-    error_message = f"An error occurred during agent execution: {str(e)}"
+def handle_agent_error(e, error_message=None):
+    if error_message is None:
+        error_message = f"An error occurred during agent execution: {str(e)}"
     logger.error(error_message, exc_info=True)
     return error_message
 
@@ -524,7 +530,13 @@ AI: {response}"""
                 output = result['output'] if isinstance(result, dict) and 'output' in result else str(result)
                 return {CONTEXT_COLUMN: captured_context, ASSISTANT_COLUMN: output}
             except Exception as e:
-                return {CONTEXT_COLUMN: [], ASSISTANT_COLUMN: handle_agent_error(e)}
+                error_message = str(e)
+                # Special handling for API key errors
+                if "API key" in error_message and ("not found" in error_message or "missing" in error_message):
+                    # Format API key error more clearly
+                    logger.error(f"API Key Error: {error_message}")
+                    error_message = f"API Key Error: {error_message}"
+                return {CONTEXT_COLUMN: [], ASSISTANT_COLUMN: handle_agent_error(e, error_message)}
 
         completions = []
         contexts = []

mindsdb/interfaces/agents/mindsdb_database_agent.py

@@ -2,6 +2,7 @@
     Wrapper around MindsDB's executor and integration controller following the implementation of the original
     langchain.sql_database.SQLDatabase class to partly replicate its behavior.
 """
+import traceback
 from typing import Any, Iterable, List, Optional
 
 from mindsdb.utilities import log
@@ -57,8 +58,20 @@ class MindsDBSQL(SQLDatabase):
         """Information about all tables in the database."""
         return self._sql_agent.get_table_info()
 
+    @property
+    def kb_info(self) -> str:
+        """Information about all knowledge bases in the database."""
+        return self._sql_agent.get_knowledge_base_info()
+
     def get_usable_table_names(self) -> Iterable[str]:
-        return self._sql_agent.get_usable_table_names()
+        """Information about all tables in the database."""
+        try:
+            return self._sql_agent.get_usable_table_names()
+        except Exception as e:
+            # If there's an error accessing tables through the integration, handle it gracefully
+            logger.warning(f"Error getting usable table names: {str(e)}")
+            # Return an empty list instead of raising an exception
+            return []
 
     def get_table_info_no_throw(self, table_names: Optional[List[str]] = None) -> str:
         for i in range(len(table_names)):
@@ -66,5 +79,91 @@ class MindsDBSQL(SQLDatabase):
         return self._sql_agent.get_table_info_safe(table_names)
 
     def run_no_throw(self, command: str, fetch: str = "all") -> str:
+        """Execute a SQL command and return the result as a string.
+
+        This method catches any exceptions and returns an error message instead of raising an exception.
+
+        Args:
+            command: The SQL command to execute
+            fetch: Whether to fetch 'all' results or just 'one'
+
+        Returns:
+            A string representation of the result or an error message
+        """
         command = extract_essential(command)
-        return self._sql_agent.query_safe(command)
+
+        try:
+
+            # Log the query for debugging
+            logger.info(f"Executing SQL query: {command}")
+
+            # remove backticks
+            command = command.replace('`', '')
+
+            # Parse the SQL string to an AST object first
+            from mindsdb_sql_parser import parse_sql
+            ast_query = parse_sql(command)
+
+            # Now execute the parsed query
+            result = self._sql_agent.skill_tool.get_command_executor().execute_command(ast_query, database_name="mindsdb")
+
+            # Convert ExecuteAnswer to a DataFrame for easier manipulation
+            df = None
+            if hasattr(result, 'data') and hasattr(result.data, 'data_frame'):
+                df = result.data.data_frame
+            else:
+                # Fallback to to_df when data_frame attr not available
+                try:
+                    df = result.data.to_df()
+                except Exception:
+                    df = None
+
+            # Default behaviour (string)
+            if df is not None:
+                if not df.empty:
+                    return df.to_string(index=False)
+                else:
+                    return "Query executed successfully, but returned no data."
+
+            return str(result)
+
+        except Exception as e:
+            logger.error(f"Error executing SQL command: {str(e)}\n{traceback.format_exc()}")
+            # If this is a knowledge base query, provide a more helpful error message
+            if "knowledge_base" in command.lower() or any(kb in command for kb in self._sql_agent.get_usable_knowledge_base_names()):
+                return f"Error executing knowledge base query: {str(e)}. Please check that the knowledge base exists and your query syntax is correct."
+            return f"Error: {str(e)}"
+
+    # def run_no_throw(self, command: str, fetch: str = "all") -> str:
+    #     """Execute a SQL command and return the result as a string.
+    #
+    #     This method catches any exceptions and returns an error message instead of raising an exception.
+    #
+    #     Args:
+    #         command: The SQL command to execute
+    #         fetch: Whether to fetch 'all' results or just 'one'
+    #
+    #     Returns:
+    #         A string representation of the result or an error message
+    #     """
+    #     command = extract_essential(command)
+    #     try:
+    #         return self._sql_agent.query_safe(command)
+    #     except Exception as e:
+    #         logger.error(f"Error executing SQL command: {str(e)}")
+    #         # If this is a knowledge base query, provide a more helpful error message
+    #         if "knowledge_base" in command.lower() or any(kb in command for kb in self._sql_agent.get_usable_knowledge_base_names()):
+    #             return f"Error executing knowledge base query: {str(e)}. Please check that the knowledge base exists and your query syntax is correct."
+    #         return f"Error: {str(e)}"
+
+    def get_usable_knowledge_base_names(self) -> List[str]:
+        """Get a list of usable knowledge base names.
+
+        Returns:
+            A list of knowledge base names that can be used in queries
+        """
+        try:
+            return self._sql_agent.get_usable_knowledge_base_names()
+        except Exception as e:
+            logger.error(f"Error getting usable knowledge base names: {str(e)}")
+            return []

mindsdb/interfaces/knowledge_base/controller.py

@@ -32,6 +32,7 @@ from mindsdb.integrations.handlers.langchain_embedding_handler.langchain_embeddi
 from mindsdb.interfaces.agents.constants import DEFAULT_EMBEDDINGS_MODEL_CLASS
 from mindsdb.interfaces.agents.langchain_agent import create_chat_model, get_llm_provider
 from mindsdb.interfaces.database.projects import ProjectController
+from mindsdb.interfaces.variables.variables_controller import variables_controller
 from mindsdb.interfaces.knowledge_base.preprocessing.models import PreprocessingConfig, Document
 from mindsdb.interfaces.knowledge_base.preprocessing.document_preprocessor import PreprocessorFactory
 from mindsdb.interfaces.model.functions import PredictorRecordNotFound
@@ -125,6 +126,11 @@ class KnowledgeBaseTable:
         logger.debug(f"Configuring preprocessing with config: {config}")
         self.document_preprocessor = None  # Reset existing preprocessor
         if config is not None:
+            # Ensure content_column is set for JSON chunking if not already specified
+            if config.get('type') == 'json_chunking' and config.get('json_chunking_config'):
+                if 'content_column' not in config['json_chunking_config']:
+                    config['json_chunking_config']['content_column'] = 'content'
+
             preprocessing_config = PreprocessingConfig(**config)
             self.document_preprocessor = PreprocessorFactory.create_preprocessor(preprocessing_config)
             logger.debug(f"Created preprocessor of type: {type(self.document_preprocessor)}")
@@ -820,6 +826,15 @@ class KnowledgeBaseTable:
         # Convert everything else to string
         return str(value)
 
+    def create_index(self):
+        """
+        Create an index on the knowledge base table
+        :param index_name: name of the index
+        :param params: parameters for the index
+        """
+        db_handler = self.get_vector_db()
+        db_handler.create_index(self._kb.vector_database_table)
+
 
 class KnowledgeBaseController:
     """
@@ -846,6 +861,9 @@ class KnowledgeBaseController:
         :param is_sparse: Whether to use sparse vectors for embeddings
         :param vector_size: Optional size specification for vectors, required when is_sparse=True
         """
+        # fill variables
+        params = variables_controller.fill_parameters(params)
+
         # Validate preprocessing config first if provided
         if preprocessing_config is not None:
             PreprocessingConfig(**preprocessing_config)  # Validate before storing
@@ -1016,6 +1034,8 @@ class KnowledgeBaseController:
                 params['question_column'] = 'content'
             if api_key:
                 params[f"{engine}_api_key"] = api_key
+                if 'api_key' in params:
+                    params.pop('api_key')
             if 'base_url' in params:
                 params['api_base'] = params.pop('base_url')
 
@@ -1153,6 +1173,11 @@ class KnowledgeBaseController:
 
         return data
 
+    def create_index(self, table_name, project_name):
+        project_id = self.session.database_controller.get_project(project_name).id
+        kb_table = self.get_table(table_name, project_id)
+        kb_table.create_index()
+
     def update(self, name: str, project_id: int, **kwargs) -> db.KnowledgeBase:
         """
         Update a knowledge base record

mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py

@@ -377,18 +377,21 @@ class PreprocessorFactory:
     ) -> DocumentPreprocessor:
         """
         Create appropriate preprocessor based on configuration
-        : param config: Preprocessing configuration
-        : return: Configured preprocessor instance
-        : raises ValueError: If unknown preprocessor type specified
+        :param config: Preprocessing configuration
+        :return: Configured preprocessor instance
+        :raises ValueError: If unknown preprocessor type specified
         """
         if config is None:
+            # Default to text chunking if no config provided
             return TextChunkingPreprocessor()
 
-        if config.type == PreprocessorType.CONTEXTUAL:
-            return ContextualPreprocessor(
-                config.contextual_config or ContextualConfig()
-            )
-        elif config.type == PreprocessorType.TEXT_CHUNKING:
+        if config.type == PreprocessorType.TEXT_CHUNKING:
             return TextChunkingPreprocessor(config.text_chunking_config)
-
-        raise ValueError(f"Unknown preprocessor type: {config.type}")
+        elif config.type == PreprocessorType.CONTEXTUAL:
+            return ContextualPreprocessor(config.contextual_config)
+        elif config.type == PreprocessorType.JSON_CHUNKING:
+            # Import here to avoid circular imports
+            from mindsdb.interfaces.knowledge_base.preprocessing.json_chunker import JSONChunkingPreprocessor
+            return JSONChunkingPreprocessor(config.json_chunking_config)
+        else:
+            raise ValueError(f"Unknown preprocessor type: {config.type}")