MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl

mindsdb/api/http/namespaces/sql.py

@@ -1,5 +1,4 @@
 from http import HTTPStatus
-import traceback
 
 from flask import request
 from flask_restx import Resource
@@ -18,6 +17,7 @@ from mindsdb.metrics.metrics import api_endpoint_metrics
 from mindsdb.utilities import log
 from mindsdb.utilities.config import Config
 from mindsdb.utilities.context import context as ctx
+from mindsdb.utilities.exception import QueryError
 
 logger = log.getLogger(__name__)
 
@@ -29,18 +29,14 @@ class Query(Resource):
         super().__init__(*args, **kwargs)
 
     @ns_conf.doc("query")
-    @api_endpoint_metrics('POST', '/sql/query')
+    @api_endpoint_metrics("POST", "/sql/query")
     def post(self):
         query = request.json["query"]
         context = request.json.get("context", {})
 
         if isinstance(query, str) is False or isinstance(context, dict) is False:
-            return http_error(
-                HTTPStatus.BAD_REQUEST,
-                'Wrong arguments',
-                'Please provide "query" with the request.'
-            )
-        logger.debug(f'Incoming query: {query}')
+            return http_error(HTTPStatus.BAD_REQUEST, "Wrong arguments", 'Please provide "query" with the request.')
+        logger.debug(f"Incoming query: {query}")
 
         if context.get("profiling") is True:
             profiler.enable()
@@ -50,9 +46,7 @@ class Query(Resource):
         error_text = None
         error_traceback = None
 
-        profiler.set_meta(
-            query=query, api="http", environment=Config().get("environment")
-        )
+        profiler.set_meta(query=query, api="http", environment=Config().get("environment"))
         with profiler.Context("http_query_processing"):
             mysql_proxy = FakeMysqlProxy()
             mysql_proxy.set_context(context)
@@ -67,8 +61,18 @@ class Query(Resource):
                     "error_code": 0,
                     "error_message": str(e),
                 }
-                logger.error(f"Error query processing: \n{traceback.format_exc()}")
-
+                logger.warning(f"Error query processing: {e}")
+            except QueryError as e:
+                error_type = "expected" if e.is_expected else "unexpected"
+                query_response = {
+                    "type": SQL_RESPONSE_TYPE.ERROR,
+                    "error_code": 0,
+                    "error_message": str(e),
+                }
+                if e.is_expected:
+                    logger.warning(f"Query failed due to expected reason: {e}")
+                else:
+                    logger.exception("Error query processing:")
             except UnknownError as e:
                 # unclassified
                 error_type = "unexpected"
@@ -77,7 +81,7 @@ class Query(Resource):
                     "error_code": 0,
                     "error_message": str(e),
                 }
-                logger.error(f"Error query processing: \n{traceback.format_exc()}")
+                logger.exception("Error query processing:")
 
             except Exception as e:
                 error_type = "unexpected"
@@ -86,7 +90,7 @@ class Query(Resource):
                     "error_code": 0,
                     "error_message": str(e),
                 }
-                logger.error(f"Error query processing: \n{traceback.format_exc()}")
+                logger.exception("Error query processing:")
 
             if query_response.get("type") == SQL_RESPONSE_TYPE.ERROR:
                 error_type = "expected"
@@ -115,7 +119,7 @@ class Query(Resource):
 @ns_conf.param("list_databases", "lists databases of mindsdb")
 class ListDatabases(Resource):
     @ns_conf.doc("list_databases")
-    @api_endpoint_metrics('GET', '/sql/list_databases')
+    @api_endpoint_metrics("GET", "/sql/list_databases")
     def get(self):
         listing_query = "SHOW DATABASES"
         mysql_proxy = FakeMysqlProxy()
@@ -133,17 +137,21 @@ class ListDatabases(Resource):
                 listing_query_response = {"type": "ok"}
             elif result.type == SQL_RESPONSE_TYPE.TABLE:
                 listing_query_response = {
-                    "data": [{
-                        "name": db_row[0],
-                        "tables": [
-                            table_row[0]
-                            for table_row in mysql_proxy.process_query(
-                                "SHOW TABLES FROM `{}`".format(db_row[0])
-                            ).result_set.to_lists()
-                        ]
-                    } for db_row in result.result_set.to_lists()]
+                    "data": [
+                        {
+                            "name": db_row[0],
+                            "tables": [
+                                table_row[0]
+                                for table_row in mysql_proxy.process_query(
+                                    "SHOW TABLES FROM `{}`".format(db_row[0])
+                                ).result_set.to_lists()
+                            ],
+                        }
+                        for db_row in result.result_set.to_lists()
+                    ]
                 }
         except Exception as e:
+            logger.exception("Error while retrieving list of databases")
             listing_query_response = {
                 "type": "error",
                 "error_code": 0,

mindsdb/api/http/namespaces/tab.py

@@ -1,5 +1,4 @@
 import json
-import traceback
 from http import HTTPStatus
 
 from flask import request
@@ -29,20 +28,20 @@ def _is_request_valid() -> bool:
     if (
         isinstance(data, dict) is False
         or len(data.keys()) == 0
-        or len(set(data.keys()) - {'index', 'name', 'content'}) != 0
+        or len(set(data.keys()) - {"index", "name", "content"}) != 0
     ):
         return False
     return True
 
 
-@ns_conf.route('/')
+@ns_conf.route("/")
 class Tabs(Resource):
-    @ns_conf.doc('get_tabs')
-    @api_endpoint_metrics('GET', '/tabs')
+    @ns_conf.doc("get_tabs")
+    @api_endpoint_metrics("GET", "/tabs")
     def get(self):
-        mode = request.args.get('mode')
+        mode = request.args.get("mode")
 
-        if mode == 'new':
+        if mode == "new":
             return tabs_controller.get_all(), 200
         else:
             # deprecated
@@ -51,26 +50,23 @@ class Tabs(Resource):
             try:
                 raw_data = storage.file_get(TABS_FILENAME)
                 tabs = json.loads(raw_data)
-            except Exception as e:
-                logger.warning("unable to get tabs data - %s", e)
+            except Exception:
+                logger.warning("unable to get tabs data - %s", exc_info=True)
                 return {}, 200
             return tabs, 200
 
-    @ns_conf.doc('save_tab')
-    @api_endpoint_metrics('POST', '/tabs')
+    @ns_conf.doc("save_tab")
+    @api_endpoint_metrics("POST", "/tabs")
     def post(self):
-        mode = request.args.get('mode')
+        mode = request.args.get("mode")
 
-        if mode == 'new':
+        if mode == "new":
             if _is_request_valid() is False:
-                return http_error(400, 'Error', 'Invalid parameters')
+                return http_error(400, "Error", "Invalid parameters")
             data = request.json
             tab_meta = tabs_controller.add(**data)
             tabs_meta = tabs_controller._get_tabs_meta()
-            return {
-                'tab_meta': tab_meta,
-                'tabs_meta': tabs_meta
-            }, 200
+            return {"tab_meta": tab_meta, "tabs_meta": tabs_meta}, 200
         else:
             # deprecated
             storage = get_storage()
@@ -78,54 +74,48 @@ class Tabs(Resource):
                 tabs = request.json
                 b_types = json.dumps(tabs).encode("utf-8")
                 storage.file_set(TABS_FILENAME, b_types)
-            except Exception as e:
-                logger.error("unable to store tabs data - %s", e)
-                logger.error(traceback.format_exc())
+            except Exception:
+                logger.exception("Unable to store tabs data:")
                 return http_error(
-                    HTTPStatus.INTERNAL_SERVER_ERROR,
-                    "Can't save tabs",
-                    'something went wrong during tabs saving'
+                    HTTPStatus.INTERNAL_SERVER_ERROR, "Can't save tabs", "something went wrong during tabs saving"
                 )
 
-            return '', 200
+            return "", 200
 
 
 @ns_conf.route("/<tab_id>")
 @ns_conf.param("tab_id", "id of tab")
 class Tab(Resource):
     @ns_conf.doc("get_tab")
-    @api_endpoint_metrics('GET', '/tabs/tab')
+    @api_endpoint_metrics("GET", "/tabs/tab")
     def get(self, tab_id: int):
         try:
             tab_data = tabs_controller.get(int(tab_id))
         except EntityNotExistsError:
-            return http_error(404, 'Error', 'The tab does not exist')
+            return http_error(404, "Error", "The tab does not exist")
 
         return tab_data, 200
 
     @ns_conf.doc("put_tab")
-    @api_endpoint_metrics('PUT', '/tabs/tab')
+    @api_endpoint_metrics("PUT", "/tabs/tab")
     def put(self, tab_id: int):
         if _is_request_valid() is False:
-            return http_error(400, 'Error', 'Invalid parameters')
+            return http_error(400, "Error", "Invalid parameters")
         data = request.json
         try:
             tab_meta = tabs_controller.modify(int(tab_id), **data)
         except EntityNotExistsError:
-            return http_error(404, 'Error', 'The tab does not exist')
+            return http_error(404, "Error", "The tab does not exist")
 
         tabs_meta = tabs_controller._get_tabs_meta()
 
-        return {
-            'tab_meta': tab_meta,
-            'tabs_meta': tabs_meta
-        }, 200
+        return {"tab_meta": tab_meta, "tabs_meta": tabs_meta}, 200
 
     @ns_conf.doc("delete_tab")
-    @api_endpoint_metrics('DELETE', '/tabs/tab')
+    @api_endpoint_metrics("DELETE", "/tabs/tab")
     def delete(self, tab_id: int):
         try:
             tabs_controller.delete(int(tab_id))
         except EntityNotExistsError:
-            return http_error(404, 'Error', 'The tab does not exist')
-        return '', 200
+            return http_error(404, "Error", "The tab does not exist")
+        return "", 200

mindsdb/api/http/namespaces/views.py

@@ -47,7 +47,7 @@ class ViewsList(Resource):
         try:
             project = session.database_controller.get_project(project_name)
         except EntityNotExistsError:
-            return http_error(HTTPStatus.NOT_FOUND, "Not found", f"Project name {project_name} does not exist")
+            return http_error(HTTPStatus.NOT_FOUND, "Project not found", f"Project name {project_name} does not exist")
 
         if project.get_view(name) is not None:
             return http_error(HTTPStatus.CONFLICT, "Name conflict", f"View with name {name} already exists.")

mindsdb/api/http/start.py

@@ -3,6 +3,10 @@ import gc
 gc.disable()
 
 from flask import Flask
+from starlette.applications import Starlette
+from starlette.routing import Mount
+from a2wsgi import WSGIMiddleware
+import uvicorn
 
 from mindsdb.api.http.initialize import initialize_app
 from mindsdb.interfaces.storage import db
@@ -11,13 +15,6 @@ from mindsdb.utilities.config import config
 from mindsdb.utilities.functions import init_lexer_parsers
 from mindsdb.integrations.libs.ml_exec_base import process_cache
 from mindsdb.api.common.middleware import PATAuthMiddleware
-
-
-from starlette.applications import Starlette
-from starlette.routing import Mount
-from starlette.middleware.wsgi import WSGIMiddleware
-import uvicorn
-
 from mindsdb.api.a2a import get_a2a_app
 from mindsdb.api.mcp import get_mcp_app
 
@@ -26,12 +23,12 @@ gc.enable()
 logger = log.getLogger(__name__)
 
 
-def start(verbose, app: Flask = None):
+def start(verbose, app: Flask = None, is_restart: bool = False):
     db.init()
     init_lexer_parsers()
 
     if app is None:
-        app = initialize_app()
+        app = initialize_app(is_restart)
 
     port = config["api"]["http"]["port"]
     host = config["api"]["http"]["host"]
@@ -48,7 +45,16 @@ def start(verbose, app: Flask = None):
     routes.append(Mount("/mcp", app=mcp))
 
     # Root app LAST so it won't shadow the others
-    routes.append(Mount("/", app=WSGIMiddleware(app)))
+    routes.append(
+        Mount(
+            "/",
+            app=WSGIMiddleware(
+                app,
+                workers=config["api"]["http"]["a2wsgi"]["workers"],
+                send_queue_size=config["api"]["http"]["a2wsgi"]["send_queue_size"],
+            ),
+        )
+    )
 
     # Setting logging to None makes uvicorn use the existing logging configuration
     uvicorn.run(Starlette(routes=routes, debug=verbose), host=host, port=int(port), log_level=None, log_config=None)

mindsdb/api/mcp/__init__.py

@@ -98,7 +98,7 @@ def query(query: str, context: dict | None = None) -> dict[str, Any]:
             return {"type": SQL_RESPONSE_TYPE.ERROR, "error_code": 0, "error_message": "Unknown response type"}
 
     except Exception as e:
-        logger.error(f"Error processing query: {str(e)}")
+        logger.exception("Error processing query:")
         return {"type": SQL_RESPONSE_TYPE.ERROR, "error_code": 0, "error_message": str(e)}
 
 
@@ -138,6 +138,7 @@ def list_databases() -> list[str]:
             return data
 
     except Exception as e:
+        logger.exception("Error while retrieving list of databases")
         return {
             "type": "error",
             "error_code": 0,

mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py

@@ -1,12 +1,14 @@
 from mindsdb_sql_parser import parse_sql
-from mindsdb.api.executor.planner import utils as planner_utils
+from mindsdb_sql_parser.exceptions import ParsingException
+from mindsdb_sql_parser.ast.base import ASTNode
 
 import mindsdb.utilities.profiler as profiler
-from mindsdb.api.executor.sql_query.result_set import Column
 from mindsdb.api.executor.sql_query import SQLQuery
+from mindsdb.api.executor.sql_query.result_set import Column
+from mindsdb.api.executor.planner import utils as planner_utils
 from mindsdb.api.executor.data_types.answer import ExecuteAnswer
 from mindsdb.api.executor.command_executor import ExecuteCommands
-from mindsdb.api.mysql.mysql_proxy.utilities import ErSqlSyntaxError
+from mindsdb.api.executor.exceptions import SqlSyntaxError
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
 from mindsdb.utilities import log
 
@@ -18,7 +20,7 @@ class Executor:
         self.session = session
         self.sqlserver = sqlserver
 
-        self.query = None
+        self.query: ASTNode = None
 
         self.columns: list[Column] = []
         self.params: list[Column] = []
@@ -32,14 +34,13 @@ class Executor:
         self.sql = ""
         self.sql_lower = ""
 
-        context = {'connection_id': self.sqlserver.connection_id}
+        context = {"connection_id": self.sqlserver.connection_id}
         self.command_executor = ExecuteCommands(self.session, context)
 
     def change_default_db(self, new_db):
         self.command_executor.change_default_db(new_db)
 
     def stmt_prepare(self, sql):
-
         self.parse(sql)
 
         # if not params
@@ -57,11 +58,7 @@ class Executor:
 
             sqlquery.prepare_query()
 
-            self.params = [Column(
-                name=p.value,
-                alias=p.value,
-                type=MYSQL_DATA_TYPE.TEXT
-            ) for p in params]
+            self.params = [Column(name=p.value, alias=p.value, type=MYSQL_DATA_TYPE.TEXT) for p in params]
 
             # TODO:
             #   select * from mindsdb.models doesn't invoke prepare_steps and columns_list is empty
@@ -90,17 +87,15 @@ class Executor:
 
         try:
             self.query = parse_sql(sql)
-        except Exception as mdb_error:
+        except ParsingException as mdb_error:
             # not all statements are parsed by parse_sql
-            logger.warning('Failed to parse SQL query')
-            logger.debug(f'Query that cannot be parsed: {sql}')
-
-            raise ErSqlSyntaxError(
-                f"The SQL statement cannot be parsed - {sql}: {mdb_error}"
-            ) from mdb_error
+            logger.warning("Failed to parse SQL query")
+            logger.debug(f"Query that cannot be parsed: {sql}")
 
-            # == a place for workarounds ==
-            # or run sql in integration without parsing
+            raise SqlSyntaxError(f"The SQL statement cannot be parsed - {sql}: {mdb_error}") from mdb_error
+        except Exception:
+            logger.exception(f"Unexpected error while parsing SQL query: {sql}")
+            raise
 
     @profiler.profile()
     def do_execute(self):

mindsdb/api/mysql/mysql_proxy/mysql_proxy.py

@@ -20,6 +20,7 @@ import struct
 import sys
 import tempfile
 import traceback
+import logging
 from functools import partial
 from typing import List
 from dataclasses import dataclass
@@ -67,11 +68,7 @@ from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import (
 )
 from mindsdb.api.executor.data_types.answer import ExecuteAnswer
 from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
-from mindsdb.api.mysql.mysql_proxy.utilities import (
-    ErWrongCharset,
-    SqlApiException,
-)
-from mindsdb.api.executor import exceptions as exec_exc
+from mindsdb.api.executor import exceptions as executor_exceptions
 
 from mindsdb.api.common.middleware import check_auth
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
@@ -81,7 +78,9 @@ from mindsdb.utilities.config import config
 from mindsdb.utilities.context import context as ctx
 from mindsdb.utilities.otel import increment_otel_query_request_counter
 from mindsdb.utilities.wizards import make_ssl_cert
+from mindsdb.utilities.exception import QueryError
 from mindsdb.api.mysql.mysql_proxy.utilities.dump import dump_result_set_to_mysql, column_to_mysql_column_dict
+from mindsdb.api.executor.exceptions import WrongCharsetError
 
 logger = log.getLogger(__name__)
 
@@ -128,6 +127,14 @@ class SQLAnswer:
             raise ValueError(f"Unsupported response type for dump HTTP response: {self.resp_type}")
 
 
+class MysqlTCPServer(SocketServer.ThreadingTCPServer):
+    """
+    Custom TCP Server with increased request queue size
+    """
+
+    request_queue_size = 30
+
+
 class MysqlProxy(SocketServer.BaseRequestHandler):
     """
     The Main Server controller class
@@ -428,7 +435,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
         try:
             return text.decode("utf-8")
         except Exception:
-            raise ErWrongCharset(f"SQL contains non utf-8 values: {text}")
+            raise WrongCharsetError(f"SQL contains non utf-8 values: {text}")
 
     def is_cloud_connection(self):
         """Determine source of connection. Must be call before handshake.
@@ -489,6 +496,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
 
     @profiler.profile()
     def process_query(self, sql) -> SQLAnswer:
+        log.log_ram_info(logger)
         executor = Executor(session=self.session, sqlserver=self)
         executor.query_execute(sql)
         executor_answer = executor.executor_answer
@@ -647,8 +655,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
             try:
                 success = p.get()
             except Exception:
-                logger.error("Session closed, on packet read error")
-                logger.error(traceback.format_exc())
+                logger.exception("Session closed, on packet read error:")
                 return
 
             if success is False:
@@ -712,59 +719,28 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
                 else:
                     logger.warning("Command has no specific handler, return OK msg")
                     logger.debug(str(p))
-                    # p.pprintPacket() TODO: Make a version of print packet
-                    # that sends it to debug instead
                     response = SQLAnswer(RESPONSE_TYPE.OK)
 
-            except SqlApiException as e:
-                # classified error
-                error_type = "expected"
-
-                response = SQLAnswer(
-                    resp_type=RESPONSE_TYPE.ERROR,
-                    error_code=e.err_code,
-                    error_message=str(e),
-                )
-
-            except exec_exc.ExecutorException as e:
-                # unclassified
-                error_type = "expected"
-
-                if isinstance(e, exec_exc.NotSupportedYet):
-                    error_code = ERR.ER_NOT_SUPPORTED_YET
-                elif isinstance(e, exec_exc.KeyColumnDoesNotExist):
-                    error_code = ERR.ER_KEY_COLUMN_DOES_NOT_EXIST
-                elif isinstance(e, exec_exc.TableNotExistError):
-                    error_code = ERR.ER_TABLE_EXISTS_ERROR
-                elif isinstance(e, exec_exc.WrongArgumentError):
-                    error_code = ERR.ER_WRONG_ARGUMENTS
-                elif isinstance(e, exec_exc.LogicError):
-                    error_code = ERR.ER_WRONG_USAGE
-                elif isinstance(e, (exec_exc.BadDbError, exec_exc.BadTableError)):
-                    error_code = ERR.ER_BAD_DB_ERROR
+            except (QueryError, executor_exceptions.ExecutorException, executor_exceptions.UnknownError) as e:
+                error_type = "expected" if e.is_expected else "unexpected"
+                error_code = e.mysql_error_code
+                if e.is_expected:
+                    if logger.isEnabledFor(logging.DEBUG):
+                        logger.info("Query execution failed with expected error:", exc_info=True)
+                    else:
+                        logger.info(f"Query execution failed with expected error: {e}")
                 else:
-                    error_code = ERR.ER_SYNTAX_ERROR
-
+                    logger.exception("Query execution failed with error")
                 response = SQLAnswer(
                     resp_type=RESPONSE_TYPE.ERROR,
                     error_code=error_code,
                     error_message=str(e),
                 )
-            except exec_exc.UnknownError as e:
-                # unclassified
-                error_type = "unexpected"
-
-                response = SQLAnswer(
-                    resp_type=RESPONSE_TYPE.ERROR,
-                    error_code=ERR.ER_UNKNOWN_ERROR,
-                    error_message=str(e),
-                )
 
             except Exception as e:
-                # any other exception
                 error_type = "unexpected"
                 error_traceback = traceback.format_exc()
-                logger.error(f"ERROR while executing query\n{error_traceback}\n{e}")
+                logger.exception("ERROR while executing query:")
                 error_code = ERR.ER_SYNTAX_ERROR
                 response = SQLAnswer(
                     resp_type=RESPONSE_TYPE.ERROR,
@@ -856,7 +832,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
         logger.info(f"Starting MindsDB Mysql proxy server on tcp://{host}:{port}")
 
         SocketServer.TCPServer.allow_reuse_address = True
-        server = SocketServer.ThreadingTCPServer((host, port), MysqlProxy)
+        server = MysqlTCPServer((host, port), MysqlProxy)
         server.mindsdb_config = config
         server.check_auth = partial(check_auth, config=config)
         server.cert_path = cert_path

mindsdb/api/mysql/mysql_proxy/utilities/__init__.py

@@ -1 +0,0 @@
-from .exceptions import *

mindsdb/api/mysql/mysql_proxy/utilities/dump.py

@@ -4,6 +4,7 @@ from typing import Any
 from array import array
 
 import numpy as np
+import orjson
 from numpy import dtype as np_dtype
 import pandas as pd
 from pandas.api import types as pd_types
@@ -21,7 +22,8 @@ from mindsdb.utilities.json_encoder import CustomJSONEncoder
 
 logger = log.getLogger(__name__)
 
-json_encoder = CustomJSONEncoder()
+# Pre-bind default encoder for custom types so we can serialize JSON consistently
+_default_json = CustomJSONEncoder().default
 
 
 def column_to_mysql_column_dict(column: Column, database_name: str | None = None) -> dict[str, str | int]:
@@ -115,7 +117,11 @@ def _dump_str(var: Any) -> str | None:
             return str(var)[2:-1]
     if isinstance(var, (dict, list)):
         try:
-            return json_encoder.encode(var)
+            return orjson.dumps(
+                var,
+                default=_default_json,
+                option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_PASSTHROUGH_DATETIME,
+            ).decode("utf-8")
         except Exception:
             return str(var)
     # pd.isna returns array of bools for list