MindsDB 25.6.4.0__py3-none-any.whl → 25.7.2.0__py3-none-any.whl

mindsdb/api/http/namespaces/file.py

@@ -3,6 +3,7 @@ import shutil
 import tarfile
 import tempfile
 import zipfile
+from urllib.parse import urlparse
 
 import multipart
 import requests
@@ -13,7 +14,7 @@ from flask_restx import Resource
 from mindsdb.api.http.namespaces.configs.files import ns_conf
 from mindsdb.api.http.utils import http_error
 from mindsdb.metrics.metrics import api_endpoint_metrics
-from mindsdb.utilities.config import Config
+from mindsdb.utilities.config import config
 from mindsdb.utilities.context import context as ctx
 from mindsdb.utilities import log
 from mindsdb.utilities.security import is_private_url, clear_filename, validate_urls
@@ -105,31 +106,55 @@ class File(Resource):
 
         if data.get("source_type") == "url":
             url = data["source"]
-            config = Config()
-            allowed_urls = config.get("file_upload_domains", [])
-            if allowed_urls and not validate_urls(url, allowed_urls):
-                return http_error(400, "Invalid File URL source.", f"Allowed hosts are: {', '.join(allowed_urls)}.")
+            try:
+                url = urlparse(url)
+                if not (url.scheme and url.netloc):
+                    raise ValueError()
+                url = url.geturl()
+            except Exception:
+                return http_error(
+                    400,
+                    "Invalid URL",
+                    f"The URL is not valid: {data['source']}",
+                )
+
+            url_file_upload_enabled = config["url_file_upload"]["enabled"]
+            if url_file_upload_enabled is False:
+                return http_error(400, "URL file upload is disabled.", "URL file upload is disabled.")
+
+            allowed_origins = config["url_file_upload"]["allowed_origins"]
+            disallowed_origins = config["url_file_upload"]["disallowed_origins"]
+
+            if validate_urls(url, allowed_origins, disallowed_origins) is False:
+                return http_error(
+                    400,
+                    "Invalid URL",
+                    "URL is not allowed for security reasons. Allowed hosts are: "
+                    f"{', '.join(allowed_origins) if allowed_origins else 'not specified'}.",
+                )
+
             data["file"] = clear_filename(data["name"])
             is_cloud = config.get("cloud", False)
-            if is_cloud and is_private_url(url):
-                return http_error(400, f"URL is private: {url}")
-
-            if is_cloud is True and ctx.user_class != 1:
-                info = requests.head(url)
-                file_size = info.headers.get("Content-Length")
-                try:
-                    file_size = int(file_size)
-                except Exception:
-                    pass
-
-                if file_size is None:
-                    return http_error(
-                        400,
-                        "Error getting file info",
-                        "Сan't determine remote file size",
-                    )
-                if file_size > MAX_FILE_SIZE:
-                    return http_error(400, "File is too big", f"Upload limit for file is {MAX_FILE_SIZE >> 20} MB")
+            if is_cloud:
+                if is_private_url(url):
+                    return http_error(400, f"URL is private: {url}")
+
+                if ctx.user_class != 1:
+                    info = requests.head(url, timeout=30)
+                    file_size = info.headers.get("Content-Length")
+                    try:
+                        file_size = int(file_size)
+                    except Exception:
+                        pass
+
+                    if file_size is None:
+                        return http_error(
+                            400,
+                            "Error getting file info",
+                            "Сan't determine remote file size",
+                        )
+                    if file_size > MAX_FILE_SIZE:
+                        return http_error(400, "File is too big", f"Upload limit for file is {MAX_FILE_SIZE >> 20} MB")
             with requests.get(url, stream=True) as r:
                 if r.status_code != 200:
                     return http_error(400, "Error getting file", f"Got status code: {r.status_code}")

mindsdb/api/mcp/start.py

@@ -1,7 +1,8 @@
 import os
+from typing import Any
+from textwrap import dedent
 from contextlib import asynccontextmanager
 from collections.abc import AsyncIterator
-from typing import Optional, Dict, Any
 from dataclasses import dataclass
 
 import uvicorn
@@ -41,16 +42,32 @@ async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]:
 mcp = FastMCP(
     "MindsDB",
     lifespan=app_lifespan,
-    dependencies=["mindsdb"]  # Add any additional dependencies
+    dependencies=["mindsdb"],  # Add any additional dependencies
 )
 # MCP Queries
 LISTING_QUERY = "SHOW DATABASES"
 
 
-@mcp.tool()
-def query(query: str, context: Optional[Dict] = None) -> Dict[str, Any]:
-    """
-    Execute a SQL query against MindsDB
+query_tool_description = dedent("""\
+    Executes a SQL query against MindsDB.
+
+    A database must be specified either in the `context` parameter or directly in the query string (e.g., `SELECT * FROM my_database.my_table`). Queries like `SELECT * FROM my_table` will fail without a `context`.
+
+    Args:
+        query (str): The SQL query to execute.
+        context (dict, optional): The default database context. For example, `{"db": "my_postgres"}`.
+
+    Returns:
+        A dictionary describing the result.
+        - For a successful query with no data to return (e.g., an `UPDATE` statement), the response is `{"type": "ok"}`.
+        - If the query returns tabular data, the response is a dictionary containing `data` (a list of rows) and `column_names` (a list of column names). For example: `{"type": "table", "data": [[1, "a"], [2, "b"]], "column_names": ["column_a", "column_b"]}`.
+        - In case of an error, a response is `{"type": "error", "error_message": "the error message"}`.
+""")
+
+
+@mcp.tool(name="query", description=query_tool_description)
+def query(query: str, context: dict | None = None) -> dict[str, Any]:
+    """Execute a SQL query against MindsDB
 
     Args:
         query: The SQL query to execute
@@ -63,7 +80,7 @@ def query(query: str, context: Optional[Dict] = None) -> Dict[str, Any]:
     if context is None:
         context = {}
 
-    logger.debug(f'Incoming MCP query: {query}')
+    logger.debug(f"Incoming MCP query: {query}")
 
     mysql_proxy = FakeMysqlProxy()
     mysql_proxy.set_context(context)
@@ -78,34 +95,30 @@ def query(query: str, context: Optional[Dict] = None) -> Dict[str, Any]:
             return {
                 "type": SQL_RESPONSE_TYPE.TABLE,
                 "data": result.result_set.to_lists(json_types=True),
-                "column_names": [
-                    column.alias or column.name
-                    for column in result.result_set.columns
-                ],
+                "column_names": [column.alias or column.name for column in result.result_set.columns],
             }
         else:
-            return {
-                "type": SQL_RESPONSE_TYPE.ERROR,
-                "error_code": 0,
-                "error_message": "Unknown response type"
-            }
+            return {"type": SQL_RESPONSE_TYPE.ERROR, "error_code": 0, "error_message": "Unknown response type"}
 
     except Exception as e:
         logger.error(f"Error processing query: {str(e)}")
-        return {
-            "type": SQL_RESPONSE_TYPE.ERROR,
-            "error_code": 0,
-            "error_message": str(e)
-        }
+        return {"type": SQL_RESPONSE_TYPE.ERROR, "error_code": 0, "error_message": str(e)}
 
 
-@mcp.tool()
-def list_databases() -> Dict[str, Any]:
+list_databases_tool_description = (
+    "Returns a list of all database connections currently available in MindsDB. "
+    + "The tool takes no parameters and responds with a list of database names, "
+    + 'for example: ["my_postgres", "my_mysql", "test_db"].'
+)
+
+
+@mcp.tool(name="list_databases", description=list_databases_tool_description)
+def list_databases() -> list[str]:
     """
-    List all databases in MindsDB along with their tables
+    List all databases in MindsDB
 
     Returns:
-        Dict containing the list of databases and their tables
+       list[str]: list of databases
     """
 
     mysql_proxy = FakeMysqlProxy()
@@ -124,6 +137,7 @@ def list_databases() -> Dict[str, Any]:
 
         elif result.type == SQL_RESPONSE_TYPE.TABLE:
             data = result.result_set.to_lists(json_types=True)
+            data = [val[0] for val in data]
             return data
 
     except Exception as e:
@@ -135,12 +149,12 @@ def list_databases() -> Dict[str, Any]:
 
 
 class CustomAuthMiddleware(BaseHTTPMiddleware):
-    """Custom middleware to handle authentication basing on header 'Authorization'
-    """
+    """Custom middleware to handle authentication basing on header 'Authorization'"""
+
     async def dispatch(self, request: Request, call_next):
-        mcp_access_token = os.environ.get('MINDSDB_MCP_ACCESS_TOKEN')
+        mcp_access_token = os.environ.get("MINDSDB_MCP_ACCESS_TOKEN")
         if mcp_access_token is not None:
-            auth_token = request.headers.get('Authorization', '').partition('Bearer ')[-1]
+            auth_token = request.headers.get("Authorization", "").partition("Bearer ")[-1]
             if mcp_access_token != auth_token:
                 return Response(status_code=401, content="Unauthorized", media_type="text/plain")
 
@@ -171,8 +185,8 @@ def start(*args, **kwargs):
         port (int): Port to listen on
     """
     config = Config()
-    port = int(config['api'].get('mcp', {}).get('port', 47337))
-    host = config['api'].get('mcp', {}).get('host', '127.0.0.1')
+    port = int(config["api"].get("mcp", {}).get("port", 47337))
+    host = config["api"].get("mcp", {}).get("host", "127.0.0.1")
 
     logger.info(f"Starting MCP server on {host}:{port}")
     mcp.settings.host = host

mindsdb/integrations/handlers/chromadb_handler/chromadb_handler.py

@@ -91,9 +91,7 @@ class ChromaDBHandler(VectorStoreHandler):
                 self.persist_directory = config.persist_directory
             elif not self.handler_storage.is_temporal:
                 # get full persistence directory from handler storage
-                self.persist_directory = self.handler_storage.folder_get(
-                    config.persist_directory
-                )
+                self.persist_directory = self.handler_storage.folder_get(config.persist_directory)
                 self._use_handler_storage = True
 
         return config
@@ -141,7 +139,7 @@ class ChromaDBHandler(VectorStoreHandler):
     def disconnect(self):
         """Close the database connection."""
         if self.is_connected:
-            if hasattr(self._client, 'close'):
+            if hasattr(self._client, "close"):
                 self._client.close()  # Some ChromaDB clients have a close method
             self._client = None
             self.is_connected = False
@@ -182,9 +180,7 @@ class ChromaDBHandler(VectorStoreHandler):
 
         return mapping[operator]
 
-    def _translate_metadata_condition(
-        self, conditions: List[FilterCondition]
-    ) -> Optional[dict]:
+    def _translate_metadata_condition(self, conditions: List[FilterCondition]) -> Optional[dict]:
         """
         Translate a list of FilterCondition objects a dict that can be used by ChromaDB.
         E.g.,
@@ -212,9 +208,7 @@ class ChromaDBHandler(VectorStoreHandler):
         if conditions is None:
             return None
         metadata_conditions = [
-            condition
-            for condition in conditions
-            if condition.column.startswith(TableField.METADATA.value)
+            condition for condition in conditions if condition.column.startswith(TableField.METADATA.value)
         ]
         if len(metadata_conditions) == 0:
             return None
@@ -224,19 +218,11 @@ class ChromaDBHandler(VectorStoreHandler):
         for condition in metadata_conditions:
             metadata_key = condition.column.split(".")[-1]
 
-            chroma_db_conditions.append(
-                {
-                    metadata_key: {
-                        self._get_chromadb_operator(condition.op): condition.value
-                    }
-                }
-            )
+            chroma_db_conditions.append({metadata_key: {self._get_chromadb_operator(condition.op): condition.value}})
 
         # we combine all metadata conditions into a single dict
         metadata_condition = (
-            {"$and": chroma_db_conditions}
-            if len(chroma_db_conditions) > 1
-            else chroma_db_conditions[0]
+            {"$and": chroma_db_conditions} if len(chroma_db_conditions) > 1 else chroma_db_conditions[0]
         )
         return metadata_condition
 
@@ -248,7 +234,6 @@ class ChromaDBHandler(VectorStoreHandler):
         offset: int = None,
         limit: int = None,
     ) -> pd.DataFrame:
-
         collection = self._client.get_collection(table_name)
         filters = self._translate_metadata_condition(conditions)
 
@@ -258,38 +243,43 @@ class ChromaDBHandler(VectorStoreHandler):
         vector_filter = (
             []
             if conditions is None
-            else [
-                condition
-                for condition in conditions
-                if condition.column == TableField.EMBEDDINGS.value
-            ]
+            else [condition for condition in conditions if condition.column == TableField.EMBEDDINGS.value]
         )
 
         if len(vector_filter) > 0:
             vector_filter = vector_filter[0]
         else:
             vector_filter = None
-        id_filters = []
+        ids_include = []
+        ids_exclude = []
+
         if conditions is not None:
             for condition in conditions:
                 if condition.column != TableField.ID.value:
                     continue
                 if condition.op == FilterOperator.EQUAL:
-                    id_filters.append(condition.value)
+                    ids_include.append(condition.value)
                 elif condition.op == FilterOperator.IN:
-                    id_filters.extend(condition.value)
+                    ids_include.extend(condition.value)
+                elif condition.op == FilterOperator.NOT_EQUAL:
+                    ids_exclude.append(condition.value)
+                elif condition.op == FilterOperator.NOT_IN:
+                    ids_exclude.extend(condition.value)
 
         if vector_filter is not None:
             # similarity search
             query_payload = {
                 "where": filters,
-                "query_embeddings": vector_filter.value
-                if vector_filter is not None
-                else None,
+                "query_embeddings": vector_filter.value if vector_filter is not None else None,
                 "include": include + ["distances"],
             }
+
             if limit is not None:
-                query_payload["n_results"] = limit
+                if len(ids_include) == 0 and len(ids_exclude) == 0:
+                    query_payload["n_results"] = limit
+                else:
+                    # get more results if we have filters by id
+                    query_payload["n_results"] = limit * 10
 
             result = collection.query(**query_payload)
             ids = result["ids"][0]
@@ -301,7 +291,7 @@ class ChromaDBHandler(VectorStoreHandler):
         else:
             # general get query
             result = collection.get(
-                ids=id_filters or None,
+                ids=ids_include or None,
                 where=filters,
                 limit=limit,
                 offset=offset,
@@ -337,13 +327,21 @@ class ChromaDBHandler(VectorStoreHandler):
                         break
 
         df = pd.DataFrame(payload)
+        if ids_exclude or ids_include:
+            if ids_exclude:
+                df = df[~df[TableField.ID.value].isin(ids_exclude)]
+            if ids_include:
+                df = df[df[TableField.ID.value].isin(ids_include)]
+            if limit is not None:
+                df = df[:limit]
+
         if distance_filter is not None:
             op_map = {
-                '<': '__lt__',
-                '<=': '__le__',
-                '>': '__gt__',
-                '>=': '__ge__',
-                '=': '__eq__',
+                "<": "__lt__",
+                "<=": "__le__",
+                ">": "__gt__",
+                ">=": "__ge__",
+                "=": "__eq__",
             }
             op = op_map.get(distance_filter.op.value)
             if op:
@@ -393,7 +391,7 @@ class ChromaDBHandler(VectorStoreHandler):
         else:
             # Convert IDs to strings and remove any duplicates
             df[TableField.ID.value] = df[TableField.ID.value].astype(str)
-            df = df.drop_duplicates(subset=[TableField.ID.value], keep='last')
+            df = df.drop_duplicates(subset=[TableField.ID.value], keep="last")
 
         return df
 
@@ -413,7 +411,7 @@ class ChromaDBHandler(VectorStoreHandler):
             df = df.dropna(subset=[TableField.METADATA.value])
 
         # Convert embeddings from string to list if they are strings
-        if TableField.EMBEDDINGS.value in df.columns and df[TableField.EMBEDDINGS.value].dtype == 'object':
+        if TableField.EMBEDDINGS.value in df.columns and df[TableField.EMBEDDINGS.value].dtype == "object":
             df[TableField.EMBEDDINGS.value] = df[TableField.EMBEDDINGS.value].apply(
                 lambda x: ast.literal_eval(x) if isinstance(x, str) else x
             )
@@ -429,7 +427,7 @@ class ChromaDBHandler(VectorStoreHandler):
                 ids=data_dict[TableField.ID.value],
                 documents=data_dict[TableField.CONTENT.value],
                 embeddings=data_dict.get(TableField.EMBEDDINGS.value, None),
-                metadatas=data_dict.get(TableField.METADATA.value, None)
+                metadatas=data_dict.get(TableField.METADATA.value, None),
             )
             self._sync()
         except Exception as e:
@@ -467,16 +465,10 @@ class ChromaDBHandler(VectorStoreHandler):
         )
         self._sync()
 
-    def delete(
-        self, table_name: str, conditions: List[FilterCondition] = None
-    ):
+    def delete(self, table_name: str, conditions: List[FilterCondition] = None):
         filters = self._translate_metadata_condition(conditions)
         # get id filters
-        id_filters = [
-            condition.value
-            for condition in conditions
-            if condition.column == TableField.ID.value
-        ] or None
+        id_filters = [condition.value for condition in conditions if condition.column == TableField.ID.value] or None
 
         if filters is None and id_filters is None:
             raise Exception("Delete query must have at least one condition!")
@@ -488,8 +480,9 @@ class ChromaDBHandler(VectorStoreHandler):
         """
         Create a collection with the given name in the ChromaDB database.
         """
-        self._client.create_collection(table_name, get_or_create=if_not_exists,
-                                       metadata=self.create_collection_metadata)
+        self._client.create_collection(
+            table_name, get_or_create=if_not_exists, metadata=self.create_collection_metadata
+        )
         self._sync()
 
     def drop_table(self, table_name: str, if_exists=True):

mindsdb/integrations/handlers/huggingface_handler/__init__.py

@@ -1,20 +1,25 @@
 from mindsdb.integrations.libs.const import HANDLER_TYPE
 
 from .__about__ import __version__ as version, __description__ as description
-try:
-    from .huggingface_handler import HuggingFaceHandler as Handler
-    import_error = None
-except Exception as e:
-    Handler = None
-    import_error = e
+# try:
+#     from .huggingface_handler import HuggingFaceHandler as Handler
+#     import_error = None
+# except Exception as e:
+#     Handler = None
+#     import_error = e
 
-title = 'Hugging Face'
-name = 'huggingface'
+# NOTE: security vulnerability is in `pytorch` v2.7.1, revert changes here and in
+# requirements.txt/requirements_cpu.txt when new version is released
+Handler = None
+import_error = """
+    The `huggingface_handler` is temporary disabled in current version of MindsDB due to security vulnerability.
+"""
+
+title = "Hugging Face"
+name = "huggingface"
 type = HANDLER_TYPE.ML
 icon_path = "icon.svg"
 permanent = False
-execution_method = 'subprocess_keep'
+execution_method = "subprocess_keep"
 
-__all__ = [
-    'Handler', 'version', 'name', 'type', 'title', 'description', 'import_error', 'icon_path'
-]
+__all__ = ["Handler", "version", "name", "type", "title", "description", "import_error", "icon_path"]