MindsDB 25.9.2.0a1__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +39 -20
- mindsdb/api/a2a/agent.py +7 -9
- mindsdb/api/a2a/common/server/server.py +3 -3
- mindsdb/api/a2a/common/server/task_manager.py +4 -4
- mindsdb/api/a2a/task_manager.py +15 -17
- mindsdb/api/common/middleware.py +9 -11
- mindsdb/api/executor/command_executor.py +2 -4
- mindsdb/api/executor/datahub/datanodes/datanode.py +2 -2
- mindsdb/api/executor/datahub/datanodes/integration_datanode.py +100 -48
- mindsdb/api/executor/datahub/datanodes/project_datanode.py +8 -4
- mindsdb/api/executor/datahub/datanodes/system_tables.py +1 -1
- mindsdb/api/executor/exceptions.py +29 -10
- mindsdb/api/executor/planner/plan_join.py +17 -3
- mindsdb/api/executor/sql_query/sql_query.py +74 -74
- mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +1 -2
- mindsdb/api/executor/sql_query/steps/subselect_step.py +0 -1
- mindsdb/api/executor/utilities/functions.py +6 -6
- mindsdb/api/executor/utilities/sql.py +32 -16
- mindsdb/api/http/gui.py +5 -11
- mindsdb/api/http/initialize.py +8 -10
- mindsdb/api/http/namespaces/agents.py +10 -12
- mindsdb/api/http/namespaces/analysis.py +13 -20
- mindsdb/api/http/namespaces/auth.py +1 -1
- mindsdb/api/http/namespaces/config.py +15 -11
- mindsdb/api/http/namespaces/databases.py +140 -201
- mindsdb/api/http/namespaces/file.py +15 -4
- mindsdb/api/http/namespaces/handlers.py +7 -2
- mindsdb/api/http/namespaces/knowledge_bases.py +8 -7
- mindsdb/api/http/namespaces/models.py +94 -126
- mindsdb/api/http/namespaces/projects.py +13 -22
- mindsdb/api/http/namespaces/sql.py +33 -25
- mindsdb/api/http/namespaces/tab.py +27 -37
- mindsdb/api/http/namespaces/views.py +1 -1
- mindsdb/api/http/start.py +14 -8
- mindsdb/api/mcp/__init__.py +2 -1
- mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +15 -20
- mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +26 -50
- mindsdb/api/mysql/mysql_proxy/utilities/__init__.py +0 -1
- mindsdb/api/postgres/postgres_proxy/executor/executor.py +6 -13
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +40 -28
- mindsdb/integrations/handlers/byom_handler/byom_handler.py +168 -185
- mindsdb/integrations/handlers/file_handler/file_handler.py +7 -0
- mindsdb/integrations/handlers/lightwood_handler/functions.py +45 -79
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +13 -1
- mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +25 -12
- mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +2 -1
- mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +1 -0
- mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +4 -4
- mindsdb/integrations/libs/api_handler.py +10 -10
- mindsdb/integrations/libs/base.py +4 -4
- mindsdb/integrations/libs/llm/utils.py +2 -2
- mindsdb/integrations/libs/ml_handler_process/create_engine_process.py +4 -7
- mindsdb/integrations/libs/ml_handler_process/func_call_process.py +2 -7
- mindsdb/integrations/libs/ml_handler_process/learn_process.py +37 -47
- mindsdb/integrations/libs/ml_handler_process/update_engine_process.py +4 -7
- mindsdb/integrations/libs/ml_handler_process/update_process.py +2 -7
- mindsdb/integrations/libs/process_cache.py +132 -140
- mindsdb/integrations/libs/response.py +18 -12
- mindsdb/integrations/libs/vectordatabase_handler.py +26 -0
- mindsdb/integrations/utilities/files/file_reader.py +6 -7
- mindsdb/integrations/utilities/rag/config_loader.py +37 -26
- mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +59 -9
- mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py +4 -4
- mindsdb/integrations/utilities/rag/retrievers/sql_retriever.py +55 -133
- mindsdb/integrations/utilities/rag/settings.py +58 -133
- mindsdb/integrations/utilities/rag/splitters/file_splitter.py +5 -15
- mindsdb/interfaces/agents/agents_controller.py +2 -1
- mindsdb/interfaces/agents/constants.py +0 -2
- mindsdb/interfaces/agents/litellm_server.py +34 -58
- mindsdb/interfaces/agents/mcp_client_agent.py +10 -10
- mindsdb/interfaces/agents/mindsdb_database_agent.py +5 -5
- mindsdb/interfaces/agents/run_mcp_agent.py +12 -21
- mindsdb/interfaces/chatbot/chatbot_task.py +20 -23
- mindsdb/interfaces/chatbot/polling.py +30 -18
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +10 -10
- mindsdb/interfaces/database/integrations.py +19 -2
- mindsdb/interfaces/file/file_controller.py +6 -6
- mindsdb/interfaces/functions/controller.py +1 -1
- mindsdb/interfaces/functions/to_markdown.py +2 -2
- mindsdb/interfaces/jobs/jobs_controller.py +5 -5
- mindsdb/interfaces/jobs/scheduler.py +3 -8
- mindsdb/interfaces/knowledge_base/controller.py +50 -23
- mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +40 -61
- mindsdb/interfaces/model/model_controller.py +170 -166
- mindsdb/interfaces/query_context/context_controller.py +14 -2
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +6 -4
- mindsdb/interfaces/skills/retrieval_tool.py +43 -50
- mindsdb/interfaces/skills/skill_tool.py +2 -2
- mindsdb/interfaces/skills/sql_agent.py +25 -19
- mindsdb/interfaces/storage/fs.py +114 -169
- mindsdb/interfaces/storage/json.py +19 -18
- mindsdb/interfaces/tabs/tabs_controller.py +49 -72
- mindsdb/interfaces/tasks/task_monitor.py +3 -9
- mindsdb/interfaces/tasks/task_thread.py +7 -9
- mindsdb/interfaces/triggers/trigger_task.py +7 -13
- mindsdb/interfaces/triggers/triggers_controller.py +47 -50
- mindsdb/migrations/migrate.py +16 -16
- mindsdb/utilities/api_status.py +58 -0
- mindsdb/utilities/config.py +49 -0
- mindsdb/utilities/exception.py +40 -1
- mindsdb/utilities/fs.py +0 -1
- mindsdb/utilities/hooks/profiling.py +17 -14
- mindsdb/utilities/langfuse.py +40 -45
- mindsdb/utilities/log.py +272 -0
- mindsdb/utilities/ml_task_queue/consumer.py +52 -58
- mindsdb/utilities/ml_task_queue/producer.py +26 -30
- mindsdb/utilities/render/sqlalchemy_render.py +7 -6
- mindsdb/utilities/utils.py +2 -2
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/METADATA +269 -264
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/RECORD +115 -115
- mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -14
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/WHEEL +0 -0
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import json
|
|
2
|
-
import traceback
|
|
3
2
|
from http import HTTPStatus
|
|
4
3
|
|
|
5
4
|
from flask import request
|
|
@@ -29,20 +28,20 @@ def _is_request_valid() -> bool:
|
|
|
29
28
|
if (
|
|
30
29
|
isinstance(data, dict) is False
|
|
31
30
|
or len(data.keys()) == 0
|
|
32
|
-
or len(set(data.keys()) - {
|
|
31
|
+
or len(set(data.keys()) - {"index", "name", "content"}) != 0
|
|
33
32
|
):
|
|
34
33
|
return False
|
|
35
34
|
return True
|
|
36
35
|
|
|
37
36
|
|
|
38
|
-
@ns_conf.route(
|
|
37
|
+
@ns_conf.route("/")
|
|
39
38
|
class Tabs(Resource):
|
|
40
|
-
@ns_conf.doc(
|
|
41
|
-
@api_endpoint_metrics(
|
|
39
|
+
@ns_conf.doc("get_tabs")
|
|
40
|
+
@api_endpoint_metrics("GET", "/tabs")
|
|
42
41
|
def get(self):
|
|
43
|
-
mode = request.args.get(
|
|
42
|
+
mode = request.args.get("mode")
|
|
44
43
|
|
|
45
|
-
if mode ==
|
|
44
|
+
if mode == "new":
|
|
46
45
|
return tabs_controller.get_all(), 200
|
|
47
46
|
else:
|
|
48
47
|
# deprecated
|
|
@@ -51,26 +50,23 @@ class Tabs(Resource):
|
|
|
51
50
|
try:
|
|
52
51
|
raw_data = storage.file_get(TABS_FILENAME)
|
|
53
52
|
tabs = json.loads(raw_data)
|
|
54
|
-
except Exception
|
|
55
|
-
logger.warning("unable to get tabs data - %s",
|
|
53
|
+
except Exception:
|
|
54
|
+
logger.warning("unable to get tabs data - %s", exc_info=True)
|
|
56
55
|
return {}, 200
|
|
57
56
|
return tabs, 200
|
|
58
57
|
|
|
59
|
-
@ns_conf.doc(
|
|
60
|
-
@api_endpoint_metrics(
|
|
58
|
+
@ns_conf.doc("save_tab")
|
|
59
|
+
@api_endpoint_metrics("POST", "/tabs")
|
|
61
60
|
def post(self):
|
|
62
|
-
mode = request.args.get(
|
|
61
|
+
mode = request.args.get("mode")
|
|
63
62
|
|
|
64
|
-
if mode ==
|
|
63
|
+
if mode == "new":
|
|
65
64
|
if _is_request_valid() is False:
|
|
66
|
-
return http_error(400,
|
|
65
|
+
return http_error(400, "Error", "Invalid parameters")
|
|
67
66
|
data = request.json
|
|
68
67
|
tab_meta = tabs_controller.add(**data)
|
|
69
68
|
tabs_meta = tabs_controller._get_tabs_meta()
|
|
70
|
-
return {
|
|
71
|
-
'tab_meta': tab_meta,
|
|
72
|
-
'tabs_meta': tabs_meta
|
|
73
|
-
}, 200
|
|
69
|
+
return {"tab_meta": tab_meta, "tabs_meta": tabs_meta}, 200
|
|
74
70
|
else:
|
|
75
71
|
# deprecated
|
|
76
72
|
storage = get_storage()
|
|
@@ -78,54 +74,48 @@ class Tabs(Resource):
|
|
|
78
74
|
tabs = request.json
|
|
79
75
|
b_types = json.dumps(tabs).encode("utf-8")
|
|
80
76
|
storage.file_set(TABS_FILENAME, b_types)
|
|
81
|
-
except Exception
|
|
82
|
-
logger.
|
|
83
|
-
logger.error(traceback.format_exc())
|
|
77
|
+
except Exception:
|
|
78
|
+
logger.exception("Unable to store tabs data:")
|
|
84
79
|
return http_error(
|
|
85
|
-
HTTPStatus.INTERNAL_SERVER_ERROR,
|
|
86
|
-
"Can't save tabs",
|
|
87
|
-
'something went wrong during tabs saving'
|
|
80
|
+
HTTPStatus.INTERNAL_SERVER_ERROR, "Can't save tabs", "something went wrong during tabs saving"
|
|
88
81
|
)
|
|
89
82
|
|
|
90
|
-
return
|
|
83
|
+
return "", 200
|
|
91
84
|
|
|
92
85
|
|
|
93
86
|
@ns_conf.route("/<tab_id>")
|
|
94
87
|
@ns_conf.param("tab_id", "id of tab")
|
|
95
88
|
class Tab(Resource):
|
|
96
89
|
@ns_conf.doc("get_tab")
|
|
97
|
-
@api_endpoint_metrics(
|
|
90
|
+
@api_endpoint_metrics("GET", "/tabs/tab")
|
|
98
91
|
def get(self, tab_id: int):
|
|
99
92
|
try:
|
|
100
93
|
tab_data = tabs_controller.get(int(tab_id))
|
|
101
94
|
except EntityNotExistsError:
|
|
102
|
-
return http_error(404,
|
|
95
|
+
return http_error(404, "Error", "The tab does not exist")
|
|
103
96
|
|
|
104
97
|
return tab_data, 200
|
|
105
98
|
|
|
106
99
|
@ns_conf.doc("put_tab")
|
|
107
|
-
@api_endpoint_metrics(
|
|
100
|
+
@api_endpoint_metrics("PUT", "/tabs/tab")
|
|
108
101
|
def put(self, tab_id: int):
|
|
109
102
|
if _is_request_valid() is False:
|
|
110
|
-
return http_error(400,
|
|
103
|
+
return http_error(400, "Error", "Invalid parameters")
|
|
111
104
|
data = request.json
|
|
112
105
|
try:
|
|
113
106
|
tab_meta = tabs_controller.modify(int(tab_id), **data)
|
|
114
107
|
except EntityNotExistsError:
|
|
115
|
-
return http_error(404,
|
|
108
|
+
return http_error(404, "Error", "The tab does not exist")
|
|
116
109
|
|
|
117
110
|
tabs_meta = tabs_controller._get_tabs_meta()
|
|
118
111
|
|
|
119
|
-
return {
|
|
120
|
-
'tab_meta': tab_meta,
|
|
121
|
-
'tabs_meta': tabs_meta
|
|
122
|
-
}, 200
|
|
112
|
+
return {"tab_meta": tab_meta, "tabs_meta": tabs_meta}, 200
|
|
123
113
|
|
|
124
114
|
@ns_conf.doc("delete_tab")
|
|
125
|
-
@api_endpoint_metrics(
|
|
115
|
+
@api_endpoint_metrics("DELETE", "/tabs/tab")
|
|
126
116
|
def delete(self, tab_id: int):
|
|
127
117
|
try:
|
|
128
118
|
tabs_controller.delete(int(tab_id))
|
|
129
119
|
except EntityNotExistsError:
|
|
130
|
-
return http_error(404,
|
|
131
|
-
return
|
|
120
|
+
return http_error(404, "Error", "The tab does not exist")
|
|
121
|
+
return "", 200
|
|
@@ -47,7 +47,7 @@ class ViewsList(Resource):
|
|
|
47
47
|
try:
|
|
48
48
|
project = session.database_controller.get_project(project_name)
|
|
49
49
|
except EntityNotExistsError:
|
|
50
|
-
return http_error(HTTPStatus.NOT_FOUND, "
|
|
50
|
+
return http_error(HTTPStatus.NOT_FOUND, "Project not found", f"Project name {project_name} does not exist")
|
|
51
51
|
|
|
52
52
|
if project.get_view(name) is not None:
|
|
53
53
|
return http_error(HTTPStatus.CONFLICT, "Name conflict", f"View with name {name} already exists.")
|
mindsdb/api/http/start.py
CHANGED
|
@@ -3,6 +3,10 @@ import gc
|
|
|
3
3
|
gc.disable()
|
|
4
4
|
|
|
5
5
|
from flask import Flask
|
|
6
|
+
from starlette.applications import Starlette
|
|
7
|
+
from starlette.routing import Mount
|
|
8
|
+
from a2wsgi import WSGIMiddleware
|
|
9
|
+
import uvicorn
|
|
6
10
|
|
|
7
11
|
from mindsdb.api.http.initialize import initialize_app
|
|
8
12
|
from mindsdb.interfaces.storage import db
|
|
@@ -11,13 +15,6 @@ from mindsdb.utilities.config import config
|
|
|
11
15
|
from mindsdb.utilities.functions import init_lexer_parsers
|
|
12
16
|
from mindsdb.integrations.libs.ml_exec_base import process_cache
|
|
13
17
|
from mindsdb.api.common.middleware import PATAuthMiddleware
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
from starlette.applications import Starlette
|
|
17
|
-
from starlette.routing import Mount
|
|
18
|
-
from starlette.middleware.wsgi import WSGIMiddleware
|
|
19
|
-
import uvicorn
|
|
20
|
-
|
|
21
18
|
from mindsdb.api.a2a import get_a2a_app
|
|
22
19
|
from mindsdb.api.mcp import get_mcp_app
|
|
23
20
|
|
|
@@ -48,7 +45,16 @@ def start(verbose, app: Flask = None):
|
|
|
48
45
|
routes.append(Mount("/mcp", app=mcp))
|
|
49
46
|
|
|
50
47
|
# Root app LAST so it won't shadow the others
|
|
51
|
-
routes.append(
|
|
48
|
+
routes.append(
|
|
49
|
+
Mount(
|
|
50
|
+
"/",
|
|
51
|
+
app=WSGIMiddleware(
|
|
52
|
+
app,
|
|
53
|
+
workers=config["api"]["http"]["a2wsgi"]["workers"],
|
|
54
|
+
send_queue_size=config["api"]["http"]["a2wsgi"]["send_queue_size"],
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
)
|
|
52
58
|
|
|
53
59
|
# Setting logging to None makes uvicorn use the existing logging configuration
|
|
54
60
|
uvicorn.run(Starlette(routes=routes, debug=verbose), host=host, port=int(port), log_level=None, log_config=None)
|
mindsdb/api/mcp/__init__.py
CHANGED
|
@@ -98,7 +98,7 @@ def query(query: str, context: dict | None = None) -> dict[str, Any]:
|
|
|
98
98
|
return {"type": SQL_RESPONSE_TYPE.ERROR, "error_code": 0, "error_message": "Unknown response type"}
|
|
99
99
|
|
|
100
100
|
except Exception as e:
|
|
101
|
-
logger.
|
|
101
|
+
logger.exception("Error processing query:")
|
|
102
102
|
return {"type": SQL_RESPONSE_TYPE.ERROR, "error_code": 0, "error_message": str(e)}
|
|
103
103
|
|
|
104
104
|
|
|
@@ -138,6 +138,7 @@ def list_databases() -> list[str]:
|
|
|
138
138
|
return data
|
|
139
139
|
|
|
140
140
|
except Exception as e:
|
|
141
|
+
logger.exception("Error while retrieving list of databases")
|
|
141
142
|
return {
|
|
142
143
|
"type": "error",
|
|
143
144
|
"error_code": 0,
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
from mindsdb_sql_parser import parse_sql
|
|
2
|
-
from
|
|
2
|
+
from mindsdb_sql_parser.exceptions import ParsingException
|
|
3
|
+
from mindsdb_sql_parser.ast.base import ASTNode
|
|
3
4
|
|
|
4
5
|
import mindsdb.utilities.profiler as profiler
|
|
5
|
-
from mindsdb.api.executor.sql_query.result_set import Column
|
|
6
6
|
from mindsdb.api.executor.sql_query import SQLQuery
|
|
7
|
+
from mindsdb.api.executor.sql_query.result_set import Column
|
|
8
|
+
from mindsdb.api.executor.planner import utils as planner_utils
|
|
7
9
|
from mindsdb.api.executor.data_types.answer import ExecuteAnswer
|
|
8
10
|
from mindsdb.api.executor.command_executor import ExecuteCommands
|
|
9
|
-
from mindsdb.api.
|
|
11
|
+
from mindsdb.api.executor.exceptions import SqlSyntaxError
|
|
10
12
|
from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
|
|
11
13
|
from mindsdb.utilities import log
|
|
12
14
|
|
|
@@ -18,7 +20,7 @@ class Executor:
|
|
|
18
20
|
self.session = session
|
|
19
21
|
self.sqlserver = sqlserver
|
|
20
22
|
|
|
21
|
-
self.query = None
|
|
23
|
+
self.query: ASTNode = None
|
|
22
24
|
|
|
23
25
|
self.columns: list[Column] = []
|
|
24
26
|
self.params: list[Column] = []
|
|
@@ -32,14 +34,13 @@ class Executor:
|
|
|
32
34
|
self.sql = ""
|
|
33
35
|
self.sql_lower = ""
|
|
34
36
|
|
|
35
|
-
context = {
|
|
37
|
+
context = {"connection_id": self.sqlserver.connection_id}
|
|
36
38
|
self.command_executor = ExecuteCommands(self.session, context)
|
|
37
39
|
|
|
38
40
|
def change_default_db(self, new_db):
|
|
39
41
|
self.command_executor.change_default_db(new_db)
|
|
40
42
|
|
|
41
43
|
def stmt_prepare(self, sql):
|
|
42
|
-
|
|
43
44
|
self.parse(sql)
|
|
44
45
|
|
|
45
46
|
# if not params
|
|
@@ -57,11 +58,7 @@ class Executor:
|
|
|
57
58
|
|
|
58
59
|
sqlquery.prepare_query()
|
|
59
60
|
|
|
60
|
-
self.params = [Column(
|
|
61
|
-
name=p.value,
|
|
62
|
-
alias=p.value,
|
|
63
|
-
type=MYSQL_DATA_TYPE.TEXT
|
|
64
|
-
) for p in params]
|
|
61
|
+
self.params = [Column(name=p.value, alias=p.value, type=MYSQL_DATA_TYPE.TEXT) for p in params]
|
|
65
62
|
|
|
66
63
|
# TODO:
|
|
67
64
|
# select * from mindsdb.models doesn't invoke prepare_steps and columns_list is empty
|
|
@@ -90,17 +87,15 @@ class Executor:
|
|
|
90
87
|
|
|
91
88
|
try:
|
|
92
89
|
self.query = parse_sql(sql)
|
|
93
|
-
except
|
|
90
|
+
except ParsingException as mdb_error:
|
|
94
91
|
# not all statements are parsed by parse_sql
|
|
95
|
-
logger.warning(
|
|
96
|
-
logger.debug(f
|
|
97
|
-
|
|
98
|
-
raise ErSqlSyntaxError(
|
|
99
|
-
f"The SQL statement cannot be parsed - {sql}: {mdb_error}"
|
|
100
|
-
) from mdb_error
|
|
92
|
+
logger.warning("Failed to parse SQL query")
|
|
93
|
+
logger.debug(f"Query that cannot be parsed: {sql}")
|
|
101
94
|
|
|
102
|
-
|
|
103
|
-
|
|
95
|
+
raise SqlSyntaxError(f"The SQL statement cannot be parsed - {sql}: {mdb_error}") from mdb_error
|
|
96
|
+
except Exception:
|
|
97
|
+
logger.exception(f"Unexpected error while parsing SQL query: {sql}")
|
|
98
|
+
raise
|
|
104
99
|
|
|
105
100
|
@profiler.profile()
|
|
106
101
|
def do_execute(self):
|
|
@@ -20,6 +20,7 @@ import struct
|
|
|
20
20
|
import sys
|
|
21
21
|
import tempfile
|
|
22
22
|
import traceback
|
|
23
|
+
import logging
|
|
23
24
|
from functools import partial
|
|
24
25
|
from typing import List
|
|
25
26
|
from dataclasses import dataclass
|
|
@@ -67,11 +68,7 @@ from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import (
|
|
|
67
68
|
)
|
|
68
69
|
from mindsdb.api.executor.data_types.answer import ExecuteAnswer
|
|
69
70
|
from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
|
|
70
|
-
from mindsdb.api.
|
|
71
|
-
ErWrongCharset,
|
|
72
|
-
SqlApiException,
|
|
73
|
-
)
|
|
74
|
-
from mindsdb.api.executor import exceptions as exec_exc
|
|
71
|
+
from mindsdb.api.executor import exceptions as executor_exceptions
|
|
75
72
|
|
|
76
73
|
from mindsdb.api.common.middleware import check_auth
|
|
77
74
|
from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
|
|
@@ -81,7 +78,9 @@ from mindsdb.utilities.config import config
|
|
|
81
78
|
from mindsdb.utilities.context import context as ctx
|
|
82
79
|
from mindsdb.utilities.otel import increment_otel_query_request_counter
|
|
83
80
|
from mindsdb.utilities.wizards import make_ssl_cert
|
|
81
|
+
from mindsdb.utilities.exception import QueryError
|
|
84
82
|
from mindsdb.api.mysql.mysql_proxy.utilities.dump import dump_result_set_to_mysql, column_to_mysql_column_dict
|
|
83
|
+
from mindsdb.api.executor.exceptions import WrongCharsetError
|
|
85
84
|
|
|
86
85
|
logger = log.getLogger(__name__)
|
|
87
86
|
|
|
@@ -128,6 +127,14 @@ class SQLAnswer:
|
|
|
128
127
|
raise ValueError(f"Unsupported response type for dump HTTP response: {self.resp_type}")
|
|
129
128
|
|
|
130
129
|
|
|
130
|
+
class MysqlTCPServer(SocketServer.ThreadingTCPServer):
|
|
131
|
+
"""
|
|
132
|
+
Custom TCP Server with increased request queue size
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
request_queue_size = 30
|
|
136
|
+
|
|
137
|
+
|
|
131
138
|
class MysqlProxy(SocketServer.BaseRequestHandler):
|
|
132
139
|
"""
|
|
133
140
|
The Main Server controller class
|
|
@@ -428,7 +435,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
|
|
|
428
435
|
try:
|
|
429
436
|
return text.decode("utf-8")
|
|
430
437
|
except Exception:
|
|
431
|
-
raise
|
|
438
|
+
raise WrongCharsetError(f"SQL contains non utf-8 values: {text}")
|
|
432
439
|
|
|
433
440
|
def is_cloud_connection(self):
|
|
434
441
|
"""Determine source of connection. Must be call before handshake.
|
|
@@ -489,6 +496,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
|
|
|
489
496
|
|
|
490
497
|
@profiler.profile()
|
|
491
498
|
def process_query(self, sql) -> SQLAnswer:
|
|
499
|
+
log.log_ram_info(logger)
|
|
492
500
|
executor = Executor(session=self.session, sqlserver=self)
|
|
493
501
|
executor.query_execute(sql)
|
|
494
502
|
executor_answer = executor.executor_answer
|
|
@@ -647,8 +655,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
|
|
|
647
655
|
try:
|
|
648
656
|
success = p.get()
|
|
649
657
|
except Exception:
|
|
650
|
-
logger.
|
|
651
|
-
logger.error(traceback.format_exc())
|
|
658
|
+
logger.exception("Session closed, on packet read error:")
|
|
652
659
|
return
|
|
653
660
|
|
|
654
661
|
if success is False:
|
|
@@ -712,59 +719,28 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
|
|
|
712
719
|
else:
|
|
713
720
|
logger.warning("Command has no specific handler, return OK msg")
|
|
714
721
|
logger.debug(str(p))
|
|
715
|
-
# p.pprintPacket() TODO: Make a version of print packet
|
|
716
|
-
# that sends it to debug instead
|
|
717
722
|
response = SQLAnswer(RESPONSE_TYPE.OK)
|
|
718
723
|
|
|
719
|
-
except
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
)
|
|
728
|
-
|
|
729
|
-
except exec_exc.ExecutorException as e:
|
|
730
|
-
# unclassified
|
|
731
|
-
error_type = "expected"
|
|
732
|
-
|
|
733
|
-
if isinstance(e, exec_exc.NotSupportedYet):
|
|
734
|
-
error_code = ERR.ER_NOT_SUPPORTED_YET
|
|
735
|
-
elif isinstance(e, exec_exc.KeyColumnDoesNotExist):
|
|
736
|
-
error_code = ERR.ER_KEY_COLUMN_DOES_NOT_EXIST
|
|
737
|
-
elif isinstance(e, exec_exc.TableNotExistError):
|
|
738
|
-
error_code = ERR.ER_TABLE_EXISTS_ERROR
|
|
739
|
-
elif isinstance(e, exec_exc.WrongArgumentError):
|
|
740
|
-
error_code = ERR.ER_WRONG_ARGUMENTS
|
|
741
|
-
elif isinstance(e, exec_exc.LogicError):
|
|
742
|
-
error_code = ERR.ER_WRONG_USAGE
|
|
743
|
-
elif isinstance(e, (exec_exc.BadDbError, exec_exc.BadTableError)):
|
|
744
|
-
error_code = ERR.ER_BAD_DB_ERROR
|
|
724
|
+
except (QueryError, executor_exceptions.ExecutorException, executor_exceptions.UnknownError) as e:
|
|
725
|
+
error_type = "expected" if e.is_expected else "unexpected"
|
|
726
|
+
error_code = e.mysql_error_code
|
|
727
|
+
if e.is_expected:
|
|
728
|
+
if logger.isEnabledFor(logging.DEBUG):
|
|
729
|
+
logger.info("Query execution failed with expected error:", exc_info=True)
|
|
730
|
+
else:
|
|
731
|
+
logger.info(f"Query execution failed with expected error: {e}")
|
|
745
732
|
else:
|
|
746
|
-
|
|
747
|
-
|
|
733
|
+
logger.exception("Query execution failed with error")
|
|
748
734
|
response = SQLAnswer(
|
|
749
735
|
resp_type=RESPONSE_TYPE.ERROR,
|
|
750
736
|
error_code=error_code,
|
|
751
737
|
error_message=str(e),
|
|
752
738
|
)
|
|
753
|
-
except exec_exc.UnknownError as e:
|
|
754
|
-
# unclassified
|
|
755
|
-
error_type = "unexpected"
|
|
756
|
-
|
|
757
|
-
response = SQLAnswer(
|
|
758
|
-
resp_type=RESPONSE_TYPE.ERROR,
|
|
759
|
-
error_code=ERR.ER_UNKNOWN_ERROR,
|
|
760
|
-
error_message=str(e),
|
|
761
|
-
)
|
|
762
739
|
|
|
763
740
|
except Exception as e:
|
|
764
|
-
# any other exception
|
|
765
741
|
error_type = "unexpected"
|
|
766
742
|
error_traceback = traceback.format_exc()
|
|
767
|
-
logger.
|
|
743
|
+
logger.exception("ERROR while executing query:")
|
|
768
744
|
error_code = ERR.ER_SYNTAX_ERROR
|
|
769
745
|
response = SQLAnswer(
|
|
770
746
|
resp_type=RESPONSE_TYPE.ERROR,
|
|
@@ -856,7 +832,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
|
|
|
856
832
|
logger.info(f"Starting MindsDB Mysql proxy server on tcp://{host}:{port}")
|
|
857
833
|
|
|
858
834
|
SocketServer.TCPServer.allow_reuse_address = True
|
|
859
|
-
server =
|
|
835
|
+
server = MysqlTCPServer((host, port), MysqlProxy)
|
|
860
836
|
server.mindsdb_config = config
|
|
861
837
|
server.check_auth = partial(check_auth, config=config)
|
|
862
838
|
server.cert_path = cert_path
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
from .exceptions import *
|
|
@@ -10,7 +10,7 @@ from mindsdb.api.executor.sql_query import SQLQuery
|
|
|
10
10
|
from mindsdb.api.executor.sql_query.result_set import Column
|
|
11
11
|
from mindsdb.api.mysql.mysql_proxy.utilities.lightwood_dtype import dtype
|
|
12
12
|
from mindsdb.api.executor.command_executor import ExecuteCommands
|
|
13
|
-
from mindsdb.api.
|
|
13
|
+
from mindsdb.api.executor.exceptions import SqlSyntaxError
|
|
14
14
|
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_fields import POSTGRES_TYPES
|
|
15
15
|
from mindsdb.utilities import log
|
|
16
16
|
|
|
@@ -47,12 +47,10 @@ class Executor:
|
|
|
47
47
|
self.query = parse_sql(sql)
|
|
48
48
|
except Exception as mdb_error:
|
|
49
49
|
# not all statements are parsed by parse_sql
|
|
50
|
-
self.logger.warning(
|
|
51
|
-
self.logger.debug(f
|
|
50
|
+
self.logger.warning("Failed to parse SQL query")
|
|
51
|
+
self.logger.debug(f"Query that cannot be parsed: {sql}")
|
|
52
52
|
|
|
53
|
-
raise
|
|
54
|
-
f"The SQL statement cannot be parsed - {sql}: {mdb_error}"
|
|
55
|
-
) from mdb_error
|
|
53
|
+
raise SqlSyntaxError(f"The SQL statement cannot be parsed - {sql}: {mdb_error}") from mdb_error
|
|
56
54
|
|
|
57
55
|
def stmt_execute(self, param_values):
|
|
58
56
|
if self.is_executed:
|
|
@@ -102,20 +100,15 @@ class Executor:
|
|
|
102
100
|
"state_track": self.state_track,
|
|
103
101
|
"server_status": self.server_status,
|
|
104
102
|
"is_executed": self.is_executed,
|
|
105
|
-
"session": self.session.to_json()
|
|
106
|
-
|
|
103
|
+
"session": self.session.to_json(),
|
|
107
104
|
}
|
|
108
105
|
return params
|
|
109
106
|
|
|
110
107
|
def to_postgres_columns(self, columns):
|
|
111
|
-
|
|
112
108
|
result = []
|
|
113
109
|
|
|
114
|
-
database = (
|
|
115
|
-
None if self.session.database == "" else self.session.database.lower()
|
|
116
|
-
)
|
|
110
|
+
database = None if self.session.database == "" else self.session.database.lower()
|
|
117
111
|
for column_record in columns:
|
|
118
|
-
|
|
119
112
|
field_type = column_record.type
|
|
120
113
|
|
|
121
114
|
column_type = POSTGRES_TYPES.VARCHAR
|
|
@@ -7,8 +7,11 @@ import time
|
|
|
7
7
|
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_fields import PostgresField
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_identifiers import
|
|
11
|
-
PostgresBackendMessageIdentifier,
|
|
10
|
+
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_identifiers import (
|
|
11
|
+
PostgresBackendMessageIdentifier,
|
|
12
|
+
PostgresFrontendMessageIdentifier,
|
|
13
|
+
PostgresAuthType,
|
|
14
|
+
)
|
|
12
15
|
|
|
13
16
|
|
|
14
17
|
class PostgresEmptyDataException(Exception):
|
|
@@ -29,8 +32,11 @@ class UnsupportedPostgresMessageType(Exception):
|
|
|
29
32
|
|
|
30
33
|
class PostgresPacketReader:
|
|
31
34
|
def __init__(self, buffer: BinaryIO):
|
|
32
|
-
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_formats import
|
|
33
|
-
|
|
35
|
+
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_formats import (
|
|
36
|
+
FE_MESSAGE_MAP,
|
|
37
|
+
SUPPORTED_AUTH_TYPES,
|
|
38
|
+
)
|
|
39
|
+
|
|
34
40
|
self.fe_message_map = FE_MESSAGE_MAP
|
|
35
41
|
self.supported_auth_types = SUPPORTED_AUTH_TYPES
|
|
36
42
|
self.buffer = buffer
|
|
@@ -80,7 +86,7 @@ class PostgresPacketReader:
|
|
|
80
86
|
|
|
81
87
|
def read_parameters(self, n):
|
|
82
88
|
data = self.read_bytes(n)
|
|
83
|
-
return data.split(b
|
|
89
|
+
return data.split(b"\x00")
|
|
84
90
|
|
|
85
91
|
def read_verify_ssl_request(self):
|
|
86
92
|
self.logger.debug("reading ssl")
|
|
@@ -94,9 +100,9 @@ class PostgresPacketReader:
|
|
|
94
100
|
length = self.read_int32()
|
|
95
101
|
version = self.read_int32()
|
|
96
102
|
major_version = version >> 16
|
|
97
|
-
minor_version = version &
|
|
103
|
+
minor_version = version & 0xFFFF
|
|
98
104
|
message = self.read_parameters(length - 8)
|
|
99
|
-
self.logger.debug(
|
|
105
|
+
self.logger.debug("PSQL Startup Message %d.%d : %s" % (major_version, minor_version, message))
|
|
100
106
|
parameters = {}
|
|
101
107
|
while len(message) != 0:
|
|
102
108
|
key = message.pop(0)
|
|
@@ -111,7 +117,7 @@ class PostgresPacketReader:
|
|
|
111
117
|
auth_type = self.read_byte()
|
|
112
118
|
except PostgresEmptyDataException:
|
|
113
119
|
# No authentication parameters specified. Which is fine if we're local on a mindsdbuser
|
|
114
|
-
return
|
|
120
|
+
return ""
|
|
115
121
|
try:
|
|
116
122
|
auth_type = PostgresAuthType(auth_type)
|
|
117
123
|
except Exception as e:
|
|
@@ -119,27 +125,31 @@ class PostgresPacketReader:
|
|
|
119
125
|
if auth_type not in self.supported_auth_types:
|
|
120
126
|
raise UnsupportedPostgresAuthException("%s is not a supported auth type identifier" % auth_type)
|
|
121
127
|
length = self.read_int32()
|
|
122
|
-
password = strip_null_byte(
|
|
128
|
+
password = strip_null_byte(
|
|
129
|
+
self.read_bytes(length - 4), encoding=encoding
|
|
130
|
+
) # password. Do something with later. We read to clear buffer.
|
|
123
131
|
return password
|
|
124
132
|
|
|
125
133
|
def read_message(self):
|
|
126
134
|
try:
|
|
127
135
|
message_type = self.read_byte()
|
|
128
136
|
except PostgresEmptyDataException:
|
|
129
|
-
self.logger.
|
|
137
|
+
self.logger.warning("Postgres Proxy: Received empty data string")
|
|
130
138
|
return None
|
|
131
139
|
try:
|
|
132
140
|
message_type = PostgresFrontendMessageIdentifier(message_type)
|
|
133
141
|
except Exception as e:
|
|
134
142
|
raise UnsupportedPostgresMessageType(
|
|
135
|
-
"%s is not a supported frontend message identifier:\n%s" % (message_type, str(e))
|
|
143
|
+
"%s is not a supported frontend message identifier:\n%s" % (message_type, str(e))
|
|
144
|
+
)
|
|
136
145
|
|
|
137
146
|
if message_type in self.fe_message_map:
|
|
138
147
|
self.logger.debug("reading message type %s" % str(message_type.name))
|
|
139
148
|
return self.fe_message_map[message_type]().read(self)
|
|
140
149
|
else:
|
|
141
150
|
raise UnsupportedPostgresMessageType(
|
|
142
|
-
"%s is not a supported frontend message identifier" % message_type.value
|
|
151
|
+
"%s is not a supported frontend message identifier" % message_type.value
|
|
152
|
+
)
|
|
143
153
|
|
|
144
154
|
|
|
145
155
|
class PostgresPacketBuilder:
|
|
@@ -153,8 +163,8 @@ class PostgresPacketBuilder:
|
|
|
153
163
|
self.reset()
|
|
154
164
|
|
|
155
165
|
def reset(self):
|
|
156
|
-
self.identifier = b
|
|
157
|
-
self.pack_string =
|
|
166
|
+
self.identifier = b""
|
|
167
|
+
self.pack_string = ""
|
|
158
168
|
self.pack_args = []
|
|
159
169
|
self.length = 0
|
|
160
170
|
|
|
@@ -194,44 +204,46 @@ class PostgresPacketBuilder:
|
|
|
194
204
|
write_file.write(packed_binary)
|
|
195
205
|
|
|
196
206
|
def add_char(self, s: bytes):
|
|
197
|
-
self.pack_string +=
|
|
207
|
+
self.pack_string += "c"
|
|
198
208
|
if len(s) != 1:
|
|
199
209
|
raise Exception("Char must be of length 1 in add_char")
|
|
200
210
|
self.pack_args.append(s)
|
|
201
211
|
return self.add_length(1)
|
|
202
212
|
|
|
203
213
|
def add_string(self, s: bytes):
|
|
204
|
-
s = s + b
|
|
205
|
-
self.pack_string += str(len(s)) +
|
|
214
|
+
s = s + b"\x00"
|
|
215
|
+
self.pack_string += str(len(s)) + "s"
|
|
206
216
|
self.pack_args.append(s)
|
|
207
217
|
return self.add_length(len(s))
|
|
208
218
|
|
|
209
219
|
def add_int32(self, i):
|
|
210
|
-
self.pack_string +=
|
|
220
|
+
self.pack_string += "i"
|
|
211
221
|
self.pack_args.append(i)
|
|
212
222
|
return self.add_length(4)
|
|
213
223
|
|
|
214
224
|
def add_int16(self, h):
|
|
215
|
-
self.pack_string +=
|
|
225
|
+
self.pack_string += "h"
|
|
216
226
|
self.pack_args.append(h)
|
|
217
227
|
return self.add_length(2)
|
|
218
228
|
|
|
219
229
|
def add_bytes(self, b: bytes):
|
|
220
230
|
if len(b) == 1:
|
|
221
|
-
self.pack_string +=
|
|
231
|
+
self.pack_string += "s"
|
|
222
232
|
else:
|
|
223
|
-
self.pack_string += str(len(b)) +
|
|
233
|
+
self.pack_string += str(len(b)) + "s"
|
|
224
234
|
self.pack_args.append(b)
|
|
225
235
|
return self.add_length(len(b))
|
|
226
236
|
|
|
227
237
|
def add_field(self, field: PostgresField):
|
|
228
|
-
return
|
|
229
|
-
.
|
|
230
|
-
.
|
|
231
|
-
.
|
|
232
|
-
.
|
|
233
|
-
.
|
|
238
|
+
return (
|
|
239
|
+
self.add_string(field.name.encode())
|
|
240
|
+
.add_int32(field.table_id)
|
|
241
|
+
.add_int16(field.column_id)
|
|
242
|
+
.add_int32(field.object_id)
|
|
243
|
+
.add_int16(field.dt_size)
|
|
244
|
+
.add_int32(field.type_modifier)
|
|
234
245
|
.add_int16(field.format_code)
|
|
246
|
+
)
|
|
235
247
|
|
|
236
248
|
def add_fields(self, fields: Sequence[PostgresField]):
|
|
237
249
|
for field in fields:
|
|
@@ -239,7 +251,7 @@ class PostgresPacketBuilder:
|
|
|
239
251
|
return self
|
|
240
252
|
|
|
241
253
|
def add_column_value(self, val: bytes):
|
|
242
|
-
if val == b
|
|
254
|
+
if val == b"NULL":
|
|
243
255
|
self.add_int32(-1)
|
|
244
256
|
return self
|
|
245
257
|
|