MindsDB 25.7.1.0__py3-none-any.whl → 25.7.2.0__py3-none-any.whl

mindsdb/interfaces/knowledge_base/controller.py

@@ -1,17 +1,19 @@
 import os
 import copy
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Any, Text
 import json
 import decimal
 
 import pandas as pd
 import numpy as np
+from pydantic import BaseModel, ValidationError
 from sqlalchemy.orm.attributes import flag_modified
 
 from mindsdb_sql_parser.ast import BinaryOperation, Constant, Identifier, Select, Update, Delete, Star
 from mindsdb_sql_parser.ast.mindsdb import CreatePredictor
 from mindsdb_sql_parser import parse_sql
 
+from mindsdb.integrations.libs.keyword_search_base import KeywordSearchBase
 from mindsdb.integrations.utilities.query_traversal import query_traversal
 
 import mindsdb.interfaces.storage.db as db
@@ -37,7 +39,7 @@ from mindsdb.interfaces.knowledge_base.evaluate import EvaluateBase
 from mindsdb.interfaces.knowledge_base.executor import KnowledgeBaseQueryExecutor
 from mindsdb.interfaces.model.functions import PredictorRecordNotFound
 from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
-from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator
+from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator, KeywordSearchArgs
 from mindsdb.utilities.config import config
 from mindsdb.utilities.context import context as ctx
 
@@ -49,6 +51,20 @@ from mindsdb.integrations.utilities.rag.rerankers.base_reranker import BaseLLMRe
 logger = log.getLogger(__name__)
 
 
+class KnowledgeBaseInputParams(BaseModel):
+    metadata_columns: List[str] | None = None
+    content_columns: List[str] | None = None
+    id_column: str | None = None
+    kb_no_upsert: bool = False
+    embedding_model: Dict[Text, Any] | None = None
+    is_sparse: bool = False
+    vector_size: int | None = None
+    reranking_model: Dict[Text, Any] | None = None
+
+    class Config:
+        extra = "forbid"
+
+
 def get_model_params(model_params: dict, default_config_key: str):
     """
     Get model parameters by combining default config with user provided parameters.
@@ -101,7 +117,10 @@ def get_reranking_model_from_params(reranking_model_params: dict):
 
     if "api_key" not in params_copy:
         params_copy["api_key"] = get_api_key(provider, params_copy, strict=False)
-    params_copy["model"] = params_copy.pop("model_name", None)
+
+    if "model_name" not in params_copy:
+        raise ValueError("'model_name' must be provided for reranking model")
+    params_copy["model"] = params_copy.pop("model_name")
 
     return BaseLLMReranker(**params_copy)
 
@@ -179,17 +198,20 @@ class KnowledgeBaseTable:
         df = executor.run(query)
 
         if (
-            query.group_by is not None
-            or query.order_by is not None
-            or query.having is not None
-            or query.distinct is True
-            or len(query.targets) != 1
-            or not isinstance(query.targets[0], Star)
+            query_copy.group_by is not None
+            or query_copy.order_by is not None
+            or query_copy.having is not None
+            or query_copy.distinct is True
+            or len(query_copy.targets) != 1
+            or not isinstance(query_copy.targets[0], Star)
         ):
             query_copy.where = None
             if "metadata" in df.columns:
                 df["metadata"] = df["metadata"].apply(to_json)
 
+            if query_copy.from_table is None:
+                query_copy.from_table = Identifier(parts=[self._kb.name])
+
             df = query_df(df, query_copy, session=self.session)
 
         return df
@@ -218,8 +240,12 @@ class KnowledgeBaseTable:
 
         # extract values from conditions and prepare for vectordb
         conditions = []
+        keyword_search_conditions = []
+        keyword_search_cols_and_values = []
         query_text = None
         relevance_threshold = None
+        reranking_enabled_flag = True
+        hybrid_search_enabled_flag = False
         query_conditions = db_handler.extract_conditions(query.where)
         if query_conditions is not None:
             for item in query_conditions:
@@ -235,9 +261,17 @@ class KnowledgeBaseTable:
                         logger.error(error_msg)
                         raise ValueError(error_msg)
                 elif item.column == "reranking":
+                    reranking_enabled_flag = item.value
+                    # cast to boolean
+                    if isinstance(reranking_enabled_flag, str):
+                        reranking_enabled_flag = reranking_enabled_flag.lower() not in ("false")
+                elif item.column == "hybrid_search":
+                    hybrid_search_enabled_flag = item.value
+                    # cast to boolean
+                    if isinstance(hybrid_search_enabled_flag, str):
+                        hybrid_search_enabled_flag = hybrid_search_enabled_flag.lower() not in ("false")
                     if item.value is False or (isinstance(item.value, str) and item.value.lower() == "false"):
                         disable_reranking = True
-
                 elif item.column == "relevance" and item.op.value != FilterOperator.GREATER_THAN_OR_EQUAL.value:
                     raise ValueError(
                         f"Invalid operator for relevance: {item.op.value}. Only GREATER_THAN_OR_EQUAL is allowed."
@@ -253,8 +287,16 @@ class KnowledgeBaseTable:
                             op=FilterOperator.EQUAL,
                         )
                     )
+                    keyword_search_cols_and_values.append((TableField.CONTENT.value, item.value))
                 else:
                     conditions.append(item)
+                    keyword_search_conditions.append(item)  # keyword search conditions do not use embeddings
+
+        if len(keyword_search_cols_and_values) > 1:
+            raise ValueError(
+                "Multiple content columns found in query conditions. "
+                "Only one content column is allowed for keyword search."
+            )
 
         logger.debug(f"Extracted query text: {query_text}")
 
@@ -272,9 +314,42 @@ class KnowledgeBaseTable:
         allowed_metadata_columns = self._get_allowed_metadata_columns()
         df = db_handler.dispatch_select(query, conditions, allowed_metadata_columns=allowed_metadata_columns)
         df = self.addapt_result_columns(df)
-
         logger.debug(f"Query returned {len(df)} rows")
         logger.debug(f"Columns in response: {df.columns.tolist()}")
+
+        if hybrid_search_enabled_flag and not isinstance(db_handler, KeywordSearchBase):
+            raise ValueError(f"Hybrid search is enabled but the db_handler {type(db_handler)} does not support it. ")
+        # check if db_handler inherits from KeywordSearchBase
+        if hybrid_search_enabled_flag and isinstance(db_handler, KeywordSearchBase):
+            # If query_text is present, use it for keyword search
+            logger.debug(f"Performing keyword search with query text: {query_text}")
+            keyword_search_args = KeywordSearchArgs(query=query_text, column=TableField.CONTENT.value)
+            keyword_query_obj = copy.deepcopy(query)
+
+            keyword_query_obj.targets = [
+                Identifier(TableField.ID.value),
+                Identifier(TableField.CONTENT.value),
+                Identifier(TableField.METADATA.value),
+            ]
+
+            df_keyword_select = db_handler.dispatch_select(
+                keyword_query_obj, keyword_search_conditions, keyword_search_args=keyword_search_args
+            )
+            df_keyword_select = self.addapt_result_columns(df_keyword_select)
+            logger.debug(f"Keyword search returned {len(df_keyword_select)} rows")
+            logger.debug(f"Columns in keyword search response: {df_keyword_select.columns.tolist()}")
+            # ensure df and df_keyword_select have exactly the same columns
+            if not df_keyword_select.empty:
+                if set(df.columns) != set(df_keyword_select.columns):
+                    raise ValueError(
+                        f"Keyword search returned different columns: {df_keyword_select.columns} "
+                        f"than expected: {df.columns}"
+                    )
+                df = pd.concat([df, df_keyword_select], ignore_index=True)
+                # if chunk_id column exists remove duplicates based on chunk_id
+                if "chunk_id" in df.columns:
+                    df = df.drop_duplicates(subset=["chunk_id"])
+
         # Check if we have a rerank_model configured in KB params
         df = self.add_relevance(df, query_text, relevance_threshold, disable_reranking)
 
@@ -736,8 +811,7 @@ class KnowledgeBaseTable:
         if model_id is None:
             # call litellm handler
             messages = list(df[TableField.CONTENT.value])
-            embedding_params = copy.deepcopy(config.get("default_embedding_model", {}))
-            embedding_params.update(self._kb.params["embedding_model"])
+            embedding_params = get_model_params(self._kb.params.get("embedding_model", {}), "default_embedding_model")
             results = self.call_litellm_embedding(self.session, embedding_params, messages)
             results = [[val] for val in results]
             return pd.DataFrame(results, columns=[TableField.EMBEDDINGS.value])
@@ -783,6 +857,9 @@ class KnowledgeBaseTable:
     def call_litellm_embedding(session, model_params, messages):
         args = copy.deepcopy(model_params)
 
+        if "model_name" not in args:
+            raise ValueError("'model_name' must be provided for embedding model")
+
         llm_model = args.pop("model_name")
         engine = args.pop("provider")
 
@@ -936,6 +1013,24 @@ class KnowledgeBaseController:
         # fill variables
         params = variables_controller.fill_parameters(params)
 
+        try:
+            KnowledgeBaseInputParams.model_validate(params)
+        except ValidationError as e:
+            problems = []
+            for error in e.errors():
+                parameter = ".".join([str(i) for i in error["loc"]])
+                param_type = error["type"]
+                if param_type == "extra_forbidden":
+                    msg = f"Parameter '{parameter}' is not allowed"
+                else:
+                    msg = f"Error in '{parameter}' (type: {param_type}): {error['msg']}. Input: {repr(error['input'])}"
+                problems.append(msg)
+
+            msg = "\n".join(problems)
+            if len(problems) > 1:
+                msg = "\n" + msg
+            raise ValueError(f"Problem with knowledge base parameters: {msg}")
+
         # Validate preprocessing config first if provided
         if preprocessing_config is not None:
             PreprocessingConfig(**preprocessing_config)  # Validate before storing
@@ -961,24 +1056,6 @@ class KnowledgeBaseController:
                 return kb
             raise EntityExistsError("Knowledge base already exists", name)
 
-        embedding_params = copy.deepcopy(config.get("default_embedding_model", {}))
-
-        # Legacy
-        # model_name = None
-        # model_project = project
-        # if embedding_model:
-        #     model_name = embedding_model.parts[-1]
-        #     if len(embedding_model.parts) > 1:
-        #         model_project = self.session.database_controller.get_project(embedding_model.parts[-2])
-
-        # elif "embedding_model" in params:
-        #     if isinstance(params["embedding_model"], str):
-        #         # it is model name
-        #         model_name = params["embedding_model"]
-        #     else:
-        #         # it is params for model
-        #         embedding_params.update(params["embedding_model"])
-
         embedding_params = get_model_params(params.get("embedding_model", {}), "default_embedding_model")
 
         # if model_name is None:  # Legacy
@@ -1009,7 +1086,11 @@ class KnowledgeBaseController:
         if reranking_model_params:
             # Get reranking model from params.
             # This is called here to check validaity of the parameters.
-            get_reranking_model_from_params(reranking_model_params)
+            try:
+                reranker = get_reranking_model_from_params(reranking_model_params)
+                reranker.get_scores("test", ["test"])
+            except (ValueError, RuntimeError) as e:
+                raise RuntimeError(f"Problem with reranker config: {e}")
 
         # search for the vector database table
         if storage is None:
@@ -1102,15 +1183,26 @@ class KnowledgeBaseController:
         except PredictorRecordNotFound:
             pass
 
-        if params.get("provider", None) not in ("openai", "azure_openai"):
+        if "provider" not in params:
+            raise ValueError("'provider' parameter is required for embedding model")
+
+        if params["provider"] not in ("openai", "azure_openai"):
             # try use litellm
-            KnowledgeBaseTable.call_litellm_embedding(self.session, params, ["test"])
+            try:
+                KnowledgeBaseTable.call_litellm_embedding(self.session, params, ["test"])
+            except Exception as e:
+                raise RuntimeError(f"Problem with embedding model config: {e}")
             return
 
         if "provider" in params:
             engine = params.pop("provider").lower()
 
-        api_key = get_api_key(engine, params, strict=False) or params.pop("api_key")
+        api_key = get_api_key(engine, params, strict=False)
+        if api_key is None:
+            if "api_key" in params:
+                params.pop("api_key")
+            else:
+                raise ValueError("'api_key' parameter is required for embedding model")
 
         if engine == "azure_openai":
             engine = "openai"

mindsdb/interfaces/knowledge_base/evaluate.py

@@ -90,7 +90,7 @@ class EvaluateBase:
             df = response.data_frame
 
             if "content" not in df.columns:
-                raise ValueError("`content` column isn't found in source data")
+                raise ValueError(f"`content` column isn't found in provided sql: {gen_params['from_sql']}")
 
             df.rename(columns={"content": "chunk_content"}, inplace=True)
         else:
@@ -186,7 +186,7 @@ class EvaluateBase:
             to_table = params["save_to"]
             if isinstance(to_table, str):
                 to_table = Identifier(to_table)
-            self.save_to_table(to_table, scores)
+            self.save_to_table(to_table, scores.copy())
 
         return scores
 

mindsdb/utilities/config.py

@@ -28,6 +28,13 @@ def _merge_configs(original_config: dict, override_config: dict) -> dict:
     return original_config
 
 
+def _overwrite_configs(original_config: dict, override_config: dict) -> dict:
+    """Overwrite original config with override config."""
+    for key in list(override_config.keys()):
+        original_config[key] = override_config[key]
+    return original_config
+
+
 def create_data_dir(path: Path) -> None:
     """Create a directory and checks that it is writable.
 
@@ -196,6 +203,15 @@ class Config:
                     "host": "0.0.0.0",  # API server binds to all interfaces by default
                     "port": "8000",
                 },
+                "a2a": {
+                    "host": api_host,
+                    "port": 47338,
+                    "mindsdb_host": "localhost",
+                    "mindsdb_port": 47334,
+                    "agent_name": "my_agent",
+                    "project_name": "mindsdb",
+                    "enabled": False,
+                },
             },
             "cache": {"type": "local"},
             "ml_task_queue": {"type": "local"},
@@ -209,15 +225,6 @@ class Config:
             "default_llm": {},
             "default_embedding_model": {},
             "default_reranking_model": {},
-            "a2a": {
-                "host": "localhost",
-                "port": 47338,
-                "mindsdb_host": "localhost",
-                "mindsdb_port": 47334,
-                "agent_name": "my_agent",
-                "project_name": "mindsdb",
-                "enabled": False,
-            },
             "data_catalog": {
                 "enabled": False,
             },
@@ -243,12 +250,11 @@ class Config:
         """Collect config values from env vars to self._env_config"""
         self._env_config = {
             "logging": {"handlers": {"console": {}, "file": {}}},
-            "api": {"http": {"server": {}}},
+            "api": {"http": {"server": {}}, "a2a": {}},
             "auth": {},
             "paths": {},
             "permanent_storage": {},
             "ml_task_queue": {},
-            "a2a": {},
         }
 
         # region storage root path
@@ -390,7 +396,7 @@ class Config:
             )
 
         if a2a_config:
-            self._env_config["a2a"] = a2a_config
+            self._env_config["api"]["a2a"] = a2a_config
         # endregion
 
     def fetch_auto_config(self) -> bool:
@@ -457,47 +463,36 @@ class Config:
         _merge_configs(new_config, self._env_config)
 
         # Apply command-line arguments for A2A
-        cmd_args_config = {}
+        a2a_config = {}
 
         # Check for A2A command-line arguments
         if hasattr(self.cmd_args, "a2a_host") and self.cmd_args.a2a_host is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["host"] = self.cmd_args.a2a_host
+            a2a_config["host"] = self.cmd_args.a2a_host
 
         if hasattr(self.cmd_args, "a2a_port") and self.cmd_args.a2a_port is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["port"] = self.cmd_args.a2a_port
+            a2a_config["port"] = self.cmd_args.a2a_port
 
         if hasattr(self.cmd_args, "mindsdb_host") and self.cmd_args.mindsdb_host is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["mindsdb_host"] = self.cmd_args.mindsdb_host
+            a2a_config["mindsdb_host"] = self.cmd_args.mindsdb_host
 
         if hasattr(self.cmd_args, "mindsdb_port") and self.cmd_args.mindsdb_port is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["mindsdb_port"] = self.cmd_args.mindsdb_port
+            a2a_config["mindsdb_port"] = self.cmd_args.mindsdb_port
 
         if hasattr(self.cmd_args, "agent_name") and self.cmd_args.agent_name is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["agent_name"] = self.cmd_args.agent_name
+            a2a_config["agent_name"] = self.cmd_args.agent_name
 
         if hasattr(self.cmd_args, "project_name") and self.cmd_args.project_name is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["project_name"] = self.cmd_args.project_name
+            a2a_config["project_name"] = self.cmd_args.project_name
 
         # Merge command-line args config with highest priority
-        if cmd_args_config:
-            _merge_configs(new_config, cmd_args_config)
+        if a2a_config:
+            _merge_configs(new_config, {"api": {"a2a": a2a_config}})
 
         # Ensure A2A port is never 0, which would prevent the A2A API from starting
-        if "a2a" in new_config and isinstance(new_config["a2a"], dict):
-            if "port" in new_config["a2a"] and (new_config["a2a"]["port"] == 0 or new_config["a2a"]["port"] is None):
-                new_config["a2a"]["port"] = 47338  # Use the default port value
+        a2a_config = new_config["api"].get("a2a")
+        if a2a_config is not None and isinstance(a2a_config, dict):
+            if "port" in a2a_config and (a2a_config["port"] == 0 or a2a_config["port"] is None):
+                a2a_config["port"] = 47338  # Use the default port value
 
         # region create dirs
         for key, value in new_config["paths"].items():
@@ -522,11 +517,23 @@ class Config:
         self.ensure_auto_config_is_relevant()
         return self._config
 
-    def update(self, data: dict) -> None:
-        """Update calues in `auto` config"""
+    def update(self, data: dict, overwrite: bool = False) -> None:
+        """
+        Update values in `auto` config.
+        Args:
+            data (dict): data to update in `auto` config.
+            overwrite (bool): if True, overwrite existing keys, otherwise merge them.
+                - False (default): Merge recursively. Existing nested dictionaries are preserved
+                and only the specified keys in `data` are updated.
+                - True: Overwrite completely. Existing keys are replaced entirely with values
+                from `data`, discarding any nested structure not present in `data`.
+        """
         self.ensure_auto_config_is_relevant()
 
-        _merge_configs(self._auto_config, data)
+        if overwrite:
+            _overwrite_configs(self._auto_config, data)
+        else:
+            _merge_configs(self._auto_config, data)
 
         self.auto_config_path.write_text(json.dumps(self._auto_config, indent=4))
 

mindsdb/utilities/exception.py

@@ -40,6 +40,7 @@ def format_db_error_message(
     db_type: str | None = None,
     db_error_msg: str | None = None,
     failed_query: str | None = None,
+    is_external: bool = True,
 ) -> str:
     """Format the error message for the database query.
 
@@ -48,11 +49,21 @@ def format_db_error_message(
         db_type (str | None): The type of the database.
         db_error_msg (str | None): The error message.
         failed_query (str | None): The failed query.
+        is_external (bool): True if error appeared in external database, False if in internal duckdb
 
     Returns:
         str: The formatted error message.
     """
     error_message = "Failed to execute external database query during query processing."
+    if is_external:
+        error_message = (
+            "An error occurred while executing a derived query on the external "
+            "database during processing of your original SQL query."
+        )
+    else:
+        error_message = (
+            "An error occurred while processing an internally generated query derived from your original SQL statement."
+        )
     if db_name is not None or db_type is not None:
         error_message += "\n\nDatabase Details:"
         if db_name is not None: