MindsDB 25.6.4.0__py3-none-any.whl → 25.7.2.0__py3-none-any.whl

mindsdb/utilities/config.py

@@ -28,6 +28,13 @@ def _merge_configs(original_config: dict, override_config: dict) -> dict:
     return original_config
 
 
+def _overwrite_configs(original_config: dict, override_config: dict) -> dict:
+    """Overwrite original config with override config."""
+    for key in list(override_config.keys()):
+        original_config[key] = override_config[key]
+    return original_config
+
+
 def create_data_dir(path: Path) -> None:
     """Create a directory and checks that it is writable.
 
@@ -196,10 +203,20 @@ class Config:
                     "host": "0.0.0.0",  # API server binds to all interfaces by default
                     "port": "8000",
                 },
+                "a2a": {
+                    "host": api_host,
+                    "port": 47338,
+                    "mindsdb_host": "localhost",
+                    "mindsdb_port": 47334,
+                    "agent_name": "my_agent",
+                    "project_name": "mindsdb",
+                    "enabled": False,
+                },
             },
             "cache": {"type": "local"},
             "ml_task_queue": {"type": "local"},
-            "file_upload_domains": [],
+            "url_file_upload": {"enabled": True, "allowed_origins": [], "disallowed_origins": []},
+            "file_upload_domains": [],  # deprecated, use config[url_file_upload][allowed_origins] instead
             "web_crawling_allowed_sites": [],
             "cloud": False,
             "jobs": {"disable": False},
@@ -208,15 +225,6 @@ class Config:
             "default_llm": {},
             "default_embedding_model": {},
             "default_reranking_model": {},
-            "a2a": {
-                "host": "localhost",
-                "port": 47338,
-                "mindsdb_host": "localhost",
-                "mindsdb_port": 47334,
-                "agent_name": "my_agent",
-                "project_name": "mindsdb",
-                "enabled": False,
-            },
             "data_catalog": {
                 "enabled": False,
             },
@@ -242,12 +250,11 @@ class Config:
         """Collect config values from env vars to self._env_config"""
         self._env_config = {
             "logging": {"handlers": {"console": {}, "file": {}}},
-            "api": {"http": {"server": {}}},
+            "api": {"http": {"server": {}}, "a2a": {}},
             "auth": {},
             "paths": {},
             "permanent_storage": {},
             "ml_task_queue": {},
-            "a2a": {},
         }
 
         # region storage root path
@@ -389,7 +396,7 @@ class Config:
             )
 
         if a2a_config:
-            self._env_config["a2a"] = a2a_config
+            self._env_config["api"]["a2a"] = a2a_config
         # endregion
 
     def fetch_auto_config(self) -> bool:
@@ -456,47 +463,36 @@ class Config:
         _merge_configs(new_config, self._env_config)
 
         # Apply command-line arguments for A2A
-        cmd_args_config = {}
+        a2a_config = {}
 
         # Check for A2A command-line arguments
         if hasattr(self.cmd_args, "a2a_host") and self.cmd_args.a2a_host is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["host"] = self.cmd_args.a2a_host
+            a2a_config["host"] = self.cmd_args.a2a_host
 
         if hasattr(self.cmd_args, "a2a_port") and self.cmd_args.a2a_port is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["port"] = self.cmd_args.a2a_port
+            a2a_config["port"] = self.cmd_args.a2a_port
 
         if hasattr(self.cmd_args, "mindsdb_host") and self.cmd_args.mindsdb_host is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["mindsdb_host"] = self.cmd_args.mindsdb_host
+            a2a_config["mindsdb_host"] = self.cmd_args.mindsdb_host
 
         if hasattr(self.cmd_args, "mindsdb_port") and self.cmd_args.mindsdb_port is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["mindsdb_port"] = self.cmd_args.mindsdb_port
+            a2a_config["mindsdb_port"] = self.cmd_args.mindsdb_port
 
         if hasattr(self.cmd_args, "agent_name") and self.cmd_args.agent_name is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["agent_name"] = self.cmd_args.agent_name
+            a2a_config["agent_name"] = self.cmd_args.agent_name
 
         if hasattr(self.cmd_args, "project_name") and self.cmd_args.project_name is not None:
-            if "a2a" not in cmd_args_config:
-                cmd_args_config["a2a"] = {}
-            cmd_args_config["a2a"]["project_name"] = self.cmd_args.project_name
+            a2a_config["project_name"] = self.cmd_args.project_name
 
         # Merge command-line args config with highest priority
-        if cmd_args_config:
-            _merge_configs(new_config, cmd_args_config)
+        if a2a_config:
+            _merge_configs(new_config, {"api": {"a2a": a2a_config}})
 
         # Ensure A2A port is never 0, which would prevent the A2A API from starting
-        if "a2a" in new_config and isinstance(new_config["a2a"], dict):
-            if "port" in new_config["a2a"] and (new_config["a2a"]["port"] == 0 or new_config["a2a"]["port"] is None):
-                new_config["a2a"]["port"] = 47338  # Use the default port value
+        a2a_config = new_config["api"].get("a2a")
+        if a2a_config is not None and isinstance(a2a_config, dict):
+            if "port" in a2a_config and (a2a_config["port"] == 0 or a2a_config["port"] is None):
+                a2a_config["port"] = 47338  # Use the default port value
 
         # region create dirs
         for key, value in new_config["paths"].items():
@@ -521,11 +517,23 @@ class Config:
         self.ensure_auto_config_is_relevant()
         return self._config
 
-    def update(self, data: dict) -> None:
-        """Update calues in `auto` config"""
+    def update(self, data: dict, overwrite: bool = False) -> None:
+        """
+        Update values in `auto` config.
+        Args:
+            data (dict): data to update in `auto` config.
+            overwrite (bool): if True, overwrite existing keys, otherwise merge them.
+                - False (default): Merge recursively. Existing nested dictionaries are preserved
+                and only the specified keys in `data` are updated.
+                - True: Overwrite completely. Existing keys are replaced entirely with values
+                from `data`, discarding any nested structure not present in `data`.
+        """
         self.ensure_auto_config_is_relevant()
 
-        _merge_configs(self._auto_config, data)
+        if overwrite:
+            _overwrite_configs(self._auto_config, data)
+        else:
+            _merge_configs(self._auto_config, data)
 
         self.auto_config_path.write_text(json.dumps(self._auto_config, indent=4))
 
@@ -548,6 +556,16 @@ class Config:
                 "Use 'MINDSDB_HTTP_SERVER_TYPE' instead."
             )
 
+        file_upload_domains = self._config.get("file_upload_domains")
+        if isinstance(file_upload_domains, list) and len(file_upload_domains) > 0:
+            allowed_origins = self._config["url_file_upload"]["allowed_origins"]
+            if isinstance(allowed_origins, list) and len(allowed_origins) == 0:
+                self._config["url_file_upload"]["allowed_origins"] = file_upload_domains
+            logger.warning(
+                'Config option "file_upload_domains" is deprecated, '
+                'use config["url_file_upload"]["allowed_origins"] instead.'
+            )
+
         for env_name in ("MINDSDB_HTTP_SERVER_TYPE", "MINDSDB_DEFAULT_SERVER"):
             env_value = os.environ.get(env_name, "")
             if env_value.lower() not in ("waitress", "flask", "gunicorn", ""):

mindsdb/utilities/exception.py

@@ -1,29 +1,80 @@
+from textwrap import indent
+
+
 class BaseEntityException(Exception):
     """Base exception for entitys errors
 
-        Attributes:
-            message (str): error message
-            entity_name (str): entity name
+    Attributes:
+        message (str): error message
+        entity_name (str): entity name
     """
+
     def __init__(self, message: str, entity_name: str = None) -> None:
         self.message = message
-        self.entity_name = entity_name or 'unknown'
+        self.entity_name = entity_name or "unknown"
 
     def __str__(self) -> str:
-        return f'{self.message}: {self.entity_name}'
+        return f"{self.message}: {self.entity_name}"
 
 
 class EntityExistsError(BaseEntityException):
     """Raise when entity exists, but should not"""
+
     def __init__(self, message: str = None, entity_name: str = None) -> None:
         if message is None:
-            message = 'Entity exists error'
+            message = "Entity exists error"
         super().__init__(message, entity_name)
 
 
 class EntityNotExistsError(BaseEntityException):
     """Raise when entity not exists, but should"""
+
     def __init__(self, message: str = None, entity_name: str = None) -> None:
         if message is None:
-            message = 'Entity does not exists error'
+            message = "Entity does not exists error"
         super().__init__(message, entity_name)
+
+
+def format_db_error_message(
+    db_name: str | None = None,
+    db_type: str | None = None,
+    db_error_msg: str | None = None,
+    failed_query: str | None = None,
+    is_external: bool = True,
+) -> str:
+    """Format the error message for the database query.
+
+    Args:
+        db_name (str | None): The name of the database.
+        db_type (str | None): The type of the database.
+        db_error_msg (str | None): The error message.
+        failed_query (str | None): The failed query.
+        is_external (bool): True if error appeared in external database, False if in internal duckdb
+
+    Returns:
+        str: The formatted error message.
+    """
+    error_message = "Failed to execute external database query during query processing."
+    if is_external:
+        error_message = (
+            "An error occurred while executing a derived query on the external "
+            "database during processing of your original SQL query."
+        )
+    else:
+        error_message = (
+            "An error occurred while processing an internally generated query derived from your original SQL statement."
+        )
+    if db_name is not None or db_type is not None:
+        error_message += "\n\nDatabase Details:"
+        if db_name is not None:
+            error_message += f"\n- Name: {db_name}"
+        if db_type is not None:
+            error_message += f"\n- Type: {db_type}"
+
+    if db_error_msg is not None:
+        error_message += f"\n\nError:\n{indent(db_error_msg, '    ')}"
+
+    if failed_query is not None:
+        error_message += f"\n\nFailed Query:\n{indent(failed_query, '    ')}"
+
+    return error_message

mindsdb/utilities/security.py

@@ -27,28 +27,71 @@ def clear_filename(filename: str) -> str:
 
     if not filename:
         return filename
-    badchars = '\\/:*?\"<>|'
+    badchars = '\\/:*?"<>|'
     for c in badchars:
-        filename = filename.replace(c, '')
+        filename = filename.replace(c, "")
     return filename
 
 
-def validate_urls(urls, allowed_urls):
+def _split_url(url: str) -> tuple[str, str]:
+    """
+    Splits the URL into scheme and netloc.
+
+    Args:
+        url (str): The URL to split.
+
+    Returns:
+        tuple[str, str]: The scheme and netloc of the URL.
+
+    Raises:
+        ValueError: If the URL does not include protocol and host name.
+    """
+    parsed_url = urlparse(url)
+    if not (parsed_url.scheme and parsed_url.netloc):
+        raise ValueError(f"URL must include protocol and host name: {url}")
+    return parsed_url.scheme.lower(), parsed_url.netloc.lower()
+
+
+def validate_urls(urls: str | list[str], allowed_urls: list[str], disallowed_urls: list[str] | None = None) -> bool:
     """
     Checks if the provided URL(s) is/are from an allowed host.
 
-    This function parses the URL(s) and checks the network location part (netloc)
+    This function parses the URL(s) and checks the origin (scheme + netloc)
     against a list of allowed hosts.
 
-    :param urls: The URL(s) to check. Can be a single URL (str) or a list of URLs (list).
-    :param allowed_urls: The list of allowed URLs.
-    :return bool:  True if the URL(s) is/are from an allowed host, False otherwise.
+    Examples:
+        validate_urls("http://site.com/file", ["site.com"]) -> Exception
+        validate_urls("https://site.com/file", ["https://site.com"]) -> True
+        validate_urls("http://site.com/file", ["https://site.com"]) -> False
+        validate_urls("https://site.com/file", ["https://example.com"]) -> False
+        validate_urls("site.com/file", ["https://site.com"]) -> Exception
+
+    Args:
+        urls (str | list[str]): The URL(s) to check. Can be a single URL (str) or a list of URLs (list).
+        allowed_urls (list[str]): The list of allowed URLs.
+        disallowed_urls (list[str]): The list of disallowed URLs. If provided, the function
+                                     will return False if the URL is in the disallowed list.
+
+    Returns:
+        bool: True if the URL(s) is/are from an allowed host and not in the disallowed list, False otherwise.
     """
-    allowed_netlocs = [urlparse(allowed_url).netloc for allowed_url in allowed_urls]
+    if disallowed_urls is None:
+        disallowed_urls = []
+
+    allowed_origins = [_split_url(url) for url in allowed_urls]
+    disallowed_origins = [_split_url(url) for url in disallowed_urls]
 
     if isinstance(urls, str):
         urls = [urls]
 
-    # Check if all provided URLs are from the allowed sites
-    valid = all(urlparse(url).netloc in allowed_netlocs for url in urls)
-    return valid
+    if allowed_origins:
+        for url in urls:
+            if _split_url(url) not in allowed_origins:
+                return False
+
+    if disallowed_origins:
+        for url in urls:
+            if _split_url(url) in disallowed_origins:
+                return False
+
+    return True