MindsDB 25.9.1.2__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of MindsDB might be problematic. Click here for more details.

Files changed (120) hide show
  1. mindsdb/__about__.py +1 -1
  2. mindsdb/__main__.py +39 -20
  3. mindsdb/api/a2a/agent.py +7 -9
  4. mindsdb/api/a2a/common/server/server.py +3 -3
  5. mindsdb/api/a2a/common/server/task_manager.py +4 -4
  6. mindsdb/api/a2a/task_manager.py +15 -17
  7. mindsdb/api/common/middleware.py +9 -11
  8. mindsdb/api/executor/command_executor.py +2 -4
  9. mindsdb/api/executor/datahub/datanodes/datanode.py +2 -2
  10. mindsdb/api/executor/datahub/datanodes/integration_datanode.py +100 -48
  11. mindsdb/api/executor/datahub/datanodes/project_datanode.py +8 -4
  12. mindsdb/api/executor/datahub/datanodes/system_tables.py +1 -1
  13. mindsdb/api/executor/exceptions.py +29 -10
  14. mindsdb/api/executor/planner/plan_join.py +17 -3
  15. mindsdb/api/executor/sql_query/sql_query.py +74 -74
  16. mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +1 -2
  17. mindsdb/api/executor/sql_query/steps/subselect_step.py +0 -1
  18. mindsdb/api/executor/utilities/functions.py +6 -6
  19. mindsdb/api/executor/utilities/sql.py +32 -16
  20. mindsdb/api/http/gui.py +5 -11
  21. mindsdb/api/http/initialize.py +8 -10
  22. mindsdb/api/http/namespaces/agents.py +10 -12
  23. mindsdb/api/http/namespaces/analysis.py +13 -20
  24. mindsdb/api/http/namespaces/auth.py +1 -1
  25. mindsdb/api/http/namespaces/config.py +15 -11
  26. mindsdb/api/http/namespaces/databases.py +140 -201
  27. mindsdb/api/http/namespaces/file.py +15 -4
  28. mindsdb/api/http/namespaces/handlers.py +7 -2
  29. mindsdb/api/http/namespaces/knowledge_bases.py +8 -7
  30. mindsdb/api/http/namespaces/models.py +94 -126
  31. mindsdb/api/http/namespaces/projects.py +13 -22
  32. mindsdb/api/http/namespaces/sql.py +33 -25
  33. mindsdb/api/http/namespaces/tab.py +27 -37
  34. mindsdb/api/http/namespaces/views.py +1 -1
  35. mindsdb/api/http/start.py +14 -8
  36. mindsdb/api/mcp/__init__.py +2 -1
  37. mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +15 -20
  38. mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +26 -50
  39. mindsdb/api/mysql/mysql_proxy/utilities/__init__.py +0 -1
  40. mindsdb/api/postgres/postgres_proxy/executor/executor.py +6 -13
  41. mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +40 -28
  42. mindsdb/integrations/handlers/byom_handler/byom_handler.py +168 -185
  43. mindsdb/integrations/handlers/chromadb_handler/chromadb_handler.py +11 -5
  44. mindsdb/integrations/handlers/file_handler/file_handler.py +7 -0
  45. mindsdb/integrations/handlers/lightwood_handler/functions.py +45 -79
  46. mindsdb/integrations/handlers/openai_handler/openai_handler.py +1 -1
  47. mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py +20 -2
  48. mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +18 -3
  49. mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +25 -12
  50. mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +2 -1
  51. mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +1 -0
  52. mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +1 -0
  53. mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +4 -4
  54. mindsdb/integrations/libs/api_handler.py +10 -10
  55. mindsdb/integrations/libs/base.py +4 -4
  56. mindsdb/integrations/libs/llm/utils.py +2 -2
  57. mindsdb/integrations/libs/ml_handler_process/create_engine_process.py +4 -7
  58. mindsdb/integrations/libs/ml_handler_process/func_call_process.py +2 -7
  59. mindsdb/integrations/libs/ml_handler_process/learn_process.py +37 -47
  60. mindsdb/integrations/libs/ml_handler_process/update_engine_process.py +4 -7
  61. mindsdb/integrations/libs/ml_handler_process/update_process.py +2 -7
  62. mindsdb/integrations/libs/process_cache.py +132 -140
  63. mindsdb/integrations/libs/response.py +18 -12
  64. mindsdb/integrations/libs/vectordatabase_handler.py +26 -0
  65. mindsdb/integrations/utilities/files/file_reader.py +6 -7
  66. mindsdb/integrations/utilities/rag/config_loader.py +37 -26
  67. mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +59 -9
  68. mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py +4 -4
  69. mindsdb/integrations/utilities/rag/retrievers/sql_retriever.py +55 -133
  70. mindsdb/integrations/utilities/rag/settings.py +58 -133
  71. mindsdb/integrations/utilities/rag/splitters/file_splitter.py +5 -15
  72. mindsdb/interfaces/agents/agents_controller.py +2 -1
  73. mindsdb/interfaces/agents/constants.py +0 -2
  74. mindsdb/interfaces/agents/litellm_server.py +34 -58
  75. mindsdb/interfaces/agents/mcp_client_agent.py +10 -10
  76. mindsdb/interfaces/agents/mindsdb_database_agent.py +5 -5
  77. mindsdb/interfaces/agents/run_mcp_agent.py +12 -21
  78. mindsdb/interfaces/chatbot/chatbot_task.py +20 -23
  79. mindsdb/interfaces/chatbot/polling.py +30 -18
  80. mindsdb/interfaces/data_catalog/data_catalog_loader.py +10 -10
  81. mindsdb/interfaces/database/integrations.py +19 -2
  82. mindsdb/interfaces/file/file_controller.py +6 -6
  83. mindsdb/interfaces/functions/controller.py +1 -1
  84. mindsdb/interfaces/functions/to_markdown.py +2 -2
  85. mindsdb/interfaces/jobs/jobs_controller.py +5 -5
  86. mindsdb/interfaces/jobs/scheduler.py +3 -8
  87. mindsdb/interfaces/knowledge_base/controller.py +54 -25
  88. mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +40 -61
  89. mindsdb/interfaces/model/model_controller.py +170 -166
  90. mindsdb/interfaces/query_context/context_controller.py +14 -2
  91. mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +6 -4
  92. mindsdb/interfaces/skills/retrieval_tool.py +43 -50
  93. mindsdb/interfaces/skills/skill_tool.py +2 -2
  94. mindsdb/interfaces/skills/sql_agent.py +25 -19
  95. mindsdb/interfaces/storage/fs.py +114 -169
  96. mindsdb/interfaces/storage/json.py +19 -18
  97. mindsdb/interfaces/storage/model_fs.py +54 -92
  98. mindsdb/interfaces/tabs/tabs_controller.py +49 -72
  99. mindsdb/interfaces/tasks/task_monitor.py +3 -9
  100. mindsdb/interfaces/tasks/task_thread.py +7 -9
  101. mindsdb/interfaces/triggers/trigger_task.py +7 -13
  102. mindsdb/interfaces/triggers/triggers_controller.py +47 -50
  103. mindsdb/migrations/migrate.py +16 -16
  104. mindsdb/utilities/api_status.py +58 -0
  105. mindsdb/utilities/config.py +49 -0
  106. mindsdb/utilities/exception.py +40 -1
  107. mindsdb/utilities/fs.py +0 -1
  108. mindsdb/utilities/hooks/profiling.py +17 -14
  109. mindsdb/utilities/langfuse.py +40 -45
  110. mindsdb/utilities/log.py +272 -0
  111. mindsdb/utilities/ml_task_queue/consumer.py +52 -58
  112. mindsdb/utilities/ml_task_queue/producer.py +26 -30
  113. mindsdb/utilities/render/sqlalchemy_render.py +8 -7
  114. mindsdb/utilities/utils.py +2 -2
  115. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/METADATA +266 -261
  116. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/RECORD +119 -119
  117. mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -14
  118. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/WHEEL +0 -0
  119. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/licenses/LICENSE +0 -0
  120. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/top_level.txt +0 -0
@@ -1,5 +1,4 @@
1
1
  import json
2
- import traceback
3
2
  from http import HTTPStatus
4
3
 
5
4
  from flask import request
@@ -29,20 +28,20 @@ def _is_request_valid() -> bool:
29
28
  if (
30
29
  isinstance(data, dict) is False
31
30
  or len(data.keys()) == 0
32
- or len(set(data.keys()) - {'index', 'name', 'content'}) != 0
31
+ or len(set(data.keys()) - {"index", "name", "content"}) != 0
33
32
  ):
34
33
  return False
35
34
  return True
36
35
 
37
36
 
38
- @ns_conf.route('/')
37
+ @ns_conf.route("/")
39
38
  class Tabs(Resource):
40
- @ns_conf.doc('get_tabs')
41
- @api_endpoint_metrics('GET', '/tabs')
39
+ @ns_conf.doc("get_tabs")
40
+ @api_endpoint_metrics("GET", "/tabs")
42
41
  def get(self):
43
- mode = request.args.get('mode')
42
+ mode = request.args.get("mode")
44
43
 
45
- if mode == 'new':
44
+ if mode == "new":
46
45
  return tabs_controller.get_all(), 200
47
46
  else:
48
47
  # deprecated
@@ -51,26 +50,23 @@ class Tabs(Resource):
51
50
  try:
52
51
  raw_data = storage.file_get(TABS_FILENAME)
53
52
  tabs = json.loads(raw_data)
54
- except Exception as e:
55
- logger.warning("unable to get tabs data - %s", e)
53
+ except Exception:
54
+ logger.warning("unable to get tabs data - %s", exc_info=True)
56
55
  return {}, 200
57
56
  return tabs, 200
58
57
 
59
- @ns_conf.doc('save_tab')
60
- @api_endpoint_metrics('POST', '/tabs')
58
+ @ns_conf.doc("save_tab")
59
+ @api_endpoint_metrics("POST", "/tabs")
61
60
  def post(self):
62
- mode = request.args.get('mode')
61
+ mode = request.args.get("mode")
63
62
 
64
- if mode == 'new':
63
+ if mode == "new":
65
64
  if _is_request_valid() is False:
66
- return http_error(400, 'Error', 'Invalid parameters')
65
+ return http_error(400, "Error", "Invalid parameters")
67
66
  data = request.json
68
67
  tab_meta = tabs_controller.add(**data)
69
68
  tabs_meta = tabs_controller._get_tabs_meta()
70
- return {
71
- 'tab_meta': tab_meta,
72
- 'tabs_meta': tabs_meta
73
- }, 200
69
+ return {"tab_meta": tab_meta, "tabs_meta": tabs_meta}, 200
74
70
  else:
75
71
  # deprecated
76
72
  storage = get_storage()
@@ -78,54 +74,48 @@ class Tabs(Resource):
78
74
  tabs = request.json
79
75
  b_types = json.dumps(tabs).encode("utf-8")
80
76
  storage.file_set(TABS_FILENAME, b_types)
81
- except Exception as e:
82
- logger.error("unable to store tabs data - %s", e)
83
- logger.error(traceback.format_exc())
77
+ except Exception:
78
+ logger.exception("Unable to store tabs data:")
84
79
  return http_error(
85
- HTTPStatus.INTERNAL_SERVER_ERROR,
86
- "Can't save tabs",
87
- 'something went wrong during tabs saving'
80
+ HTTPStatus.INTERNAL_SERVER_ERROR, "Can't save tabs", "something went wrong during tabs saving"
88
81
  )
89
82
 
90
- return '', 200
83
+ return "", 200
91
84
 
92
85
 
93
86
  @ns_conf.route("/<tab_id>")
94
87
  @ns_conf.param("tab_id", "id of tab")
95
88
  class Tab(Resource):
96
89
  @ns_conf.doc("get_tab")
97
- @api_endpoint_metrics('GET', '/tabs/tab')
90
+ @api_endpoint_metrics("GET", "/tabs/tab")
98
91
  def get(self, tab_id: int):
99
92
  try:
100
93
  tab_data = tabs_controller.get(int(tab_id))
101
94
  except EntityNotExistsError:
102
- return http_error(404, 'Error', 'The tab does not exist')
95
+ return http_error(404, "Error", "The tab does not exist")
103
96
 
104
97
  return tab_data, 200
105
98
 
106
99
  @ns_conf.doc("put_tab")
107
- @api_endpoint_metrics('PUT', '/tabs/tab')
100
+ @api_endpoint_metrics("PUT", "/tabs/tab")
108
101
  def put(self, tab_id: int):
109
102
  if _is_request_valid() is False:
110
- return http_error(400, 'Error', 'Invalid parameters')
103
+ return http_error(400, "Error", "Invalid parameters")
111
104
  data = request.json
112
105
  try:
113
106
  tab_meta = tabs_controller.modify(int(tab_id), **data)
114
107
  except EntityNotExistsError:
115
- return http_error(404, 'Error', 'The tab does not exist')
108
+ return http_error(404, "Error", "The tab does not exist")
116
109
 
117
110
  tabs_meta = tabs_controller._get_tabs_meta()
118
111
 
119
- return {
120
- 'tab_meta': tab_meta,
121
- 'tabs_meta': tabs_meta
122
- }, 200
112
+ return {"tab_meta": tab_meta, "tabs_meta": tabs_meta}, 200
123
113
 
124
114
  @ns_conf.doc("delete_tab")
125
- @api_endpoint_metrics('DELETE', '/tabs/tab')
115
+ @api_endpoint_metrics("DELETE", "/tabs/tab")
126
116
  def delete(self, tab_id: int):
127
117
  try:
128
118
  tabs_controller.delete(int(tab_id))
129
119
  except EntityNotExistsError:
130
- return http_error(404, 'Error', 'The tab does not exist')
131
- return '', 200
120
+ return http_error(404, "Error", "The tab does not exist")
121
+ return "", 200
@@ -47,7 +47,7 @@ class ViewsList(Resource):
47
47
  try:
48
48
  project = session.database_controller.get_project(project_name)
49
49
  except EntityNotExistsError:
50
- return http_error(HTTPStatus.NOT_FOUND, "Not found", f"Project name {project_name} does not exist")
50
+ return http_error(HTTPStatus.NOT_FOUND, "Project not found", f"Project name {project_name} does not exist")
51
51
 
52
52
  if project.get_view(name) is not None:
53
53
  return http_error(HTTPStatus.CONFLICT, "Name conflict", f"View with name {name} already exists.")
mindsdb/api/http/start.py CHANGED
@@ -3,6 +3,10 @@ import gc
3
3
  gc.disable()
4
4
 
5
5
  from flask import Flask
6
+ from starlette.applications import Starlette
7
+ from starlette.routing import Mount
8
+ from a2wsgi import WSGIMiddleware
9
+ import uvicorn
6
10
 
7
11
  from mindsdb.api.http.initialize import initialize_app
8
12
  from mindsdb.interfaces.storage import db
@@ -11,13 +15,6 @@ from mindsdb.utilities.config import config
11
15
  from mindsdb.utilities.functions import init_lexer_parsers
12
16
  from mindsdb.integrations.libs.ml_exec_base import process_cache
13
17
  from mindsdb.api.common.middleware import PATAuthMiddleware
14
-
15
-
16
- from starlette.applications import Starlette
17
- from starlette.routing import Mount
18
- from starlette.middleware.wsgi import WSGIMiddleware
19
- import uvicorn
20
-
21
18
  from mindsdb.api.a2a import get_a2a_app
22
19
  from mindsdb.api.mcp import get_mcp_app
23
20
 
@@ -48,7 +45,16 @@ def start(verbose, app: Flask = None):
48
45
  routes.append(Mount("/mcp", app=mcp))
49
46
 
50
47
  # Root app LAST so it won't shadow the others
51
- routes.append(Mount("/", app=WSGIMiddleware(app)))
48
+ routes.append(
49
+ Mount(
50
+ "/",
51
+ app=WSGIMiddleware(
52
+ app,
53
+ workers=config["api"]["http"]["a2wsgi"]["workers"],
54
+ send_queue_size=config["api"]["http"]["a2wsgi"]["send_queue_size"],
55
+ ),
56
+ )
57
+ )
52
58
 
53
59
  # Setting logging to None makes uvicorn use the existing logging configuration
54
60
  uvicorn.run(Starlette(routes=routes, debug=verbose), host=host, port=int(port), log_level=None, log_config=None)
@@ -98,7 +98,7 @@ def query(query: str, context: dict | None = None) -> dict[str, Any]:
98
98
  return {"type": SQL_RESPONSE_TYPE.ERROR, "error_code": 0, "error_message": "Unknown response type"}
99
99
 
100
100
  except Exception as e:
101
- logger.error(f"Error processing query: {str(e)}")
101
+ logger.exception("Error processing query:")
102
102
  return {"type": SQL_RESPONSE_TYPE.ERROR, "error_code": 0, "error_message": str(e)}
103
103
 
104
104
 
@@ -138,6 +138,7 @@ def list_databases() -> list[str]:
138
138
  return data
139
139
 
140
140
  except Exception as e:
141
+ logger.exception("Error while retrieving list of databases")
141
142
  return {
142
143
  "type": "error",
143
144
  "error_code": 0,
@@ -1,12 +1,14 @@
1
1
  from mindsdb_sql_parser import parse_sql
2
- from mindsdb.api.executor.planner import utils as planner_utils
2
+ from mindsdb_sql_parser.exceptions import ParsingException
3
+ from mindsdb_sql_parser.ast.base import ASTNode
3
4
 
4
5
  import mindsdb.utilities.profiler as profiler
5
- from mindsdb.api.executor.sql_query.result_set import Column
6
6
  from mindsdb.api.executor.sql_query import SQLQuery
7
+ from mindsdb.api.executor.sql_query.result_set import Column
8
+ from mindsdb.api.executor.planner import utils as planner_utils
7
9
  from mindsdb.api.executor.data_types.answer import ExecuteAnswer
8
10
  from mindsdb.api.executor.command_executor import ExecuteCommands
9
- from mindsdb.api.mysql.mysql_proxy.utilities import ErSqlSyntaxError
11
+ from mindsdb.api.executor.exceptions import SqlSyntaxError
10
12
  from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
11
13
  from mindsdb.utilities import log
12
14
 
@@ -18,7 +20,7 @@ class Executor:
18
20
  self.session = session
19
21
  self.sqlserver = sqlserver
20
22
 
21
- self.query = None
23
+ self.query: ASTNode = None
22
24
 
23
25
  self.columns: list[Column] = []
24
26
  self.params: list[Column] = []
@@ -32,14 +34,13 @@ class Executor:
32
34
  self.sql = ""
33
35
  self.sql_lower = ""
34
36
 
35
- context = {'connection_id': self.sqlserver.connection_id}
37
+ context = {"connection_id": self.sqlserver.connection_id}
36
38
  self.command_executor = ExecuteCommands(self.session, context)
37
39
 
38
40
  def change_default_db(self, new_db):
39
41
  self.command_executor.change_default_db(new_db)
40
42
 
41
43
  def stmt_prepare(self, sql):
42
-
43
44
  self.parse(sql)
44
45
 
45
46
  # if not params
@@ -57,11 +58,7 @@ class Executor:
57
58
 
58
59
  sqlquery.prepare_query()
59
60
 
60
- self.params = [Column(
61
- name=p.value,
62
- alias=p.value,
63
- type=MYSQL_DATA_TYPE.TEXT
64
- ) for p in params]
61
+ self.params = [Column(name=p.value, alias=p.value, type=MYSQL_DATA_TYPE.TEXT) for p in params]
65
62
 
66
63
  # TODO:
67
64
  # select * from mindsdb.models doesn't invoke prepare_steps and columns_list is empty
@@ -90,17 +87,15 @@ class Executor:
90
87
 
91
88
  try:
92
89
  self.query = parse_sql(sql)
93
- except Exception as mdb_error:
90
+ except ParsingException as mdb_error:
94
91
  # not all statements are parsed by parse_sql
95
- logger.warning('Failed to parse SQL query')
96
- logger.debug(f'Query that cannot be parsed: {sql}')
97
-
98
- raise ErSqlSyntaxError(
99
- f"The SQL statement cannot be parsed - {sql}: {mdb_error}"
100
- ) from mdb_error
92
+ logger.warning("Failed to parse SQL query")
93
+ logger.debug(f"Query that cannot be parsed: {sql}")
101
94
 
102
- # == a place for workarounds ==
103
- # or run sql in integration without parsing
95
+ raise SqlSyntaxError(f"The SQL statement cannot be parsed - {sql}: {mdb_error}") from mdb_error
96
+ except Exception:
97
+ logger.exception(f"Unexpected error while parsing SQL query: {sql}")
98
+ raise
104
99
 
105
100
  @profiler.profile()
106
101
  def do_execute(self):
@@ -20,6 +20,7 @@ import struct
20
20
  import sys
21
21
  import tempfile
22
22
  import traceback
23
+ import logging
23
24
  from functools import partial
24
25
  from typing import List
25
26
  from dataclasses import dataclass
@@ -67,11 +68,7 @@ from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import (
67
68
  )
68
69
  from mindsdb.api.executor.data_types.answer import ExecuteAnswer
69
70
  from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
70
- from mindsdb.api.mysql.mysql_proxy.utilities import (
71
- ErWrongCharset,
72
- SqlApiException,
73
- )
74
- from mindsdb.api.executor import exceptions as exec_exc
71
+ from mindsdb.api.executor import exceptions as executor_exceptions
75
72
 
76
73
  from mindsdb.api.common.middleware import check_auth
77
74
  from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
@@ -81,7 +78,9 @@ from mindsdb.utilities.config import config
81
78
  from mindsdb.utilities.context import context as ctx
82
79
  from mindsdb.utilities.otel import increment_otel_query_request_counter
83
80
  from mindsdb.utilities.wizards import make_ssl_cert
81
+ from mindsdb.utilities.exception import QueryError
84
82
  from mindsdb.api.mysql.mysql_proxy.utilities.dump import dump_result_set_to_mysql, column_to_mysql_column_dict
83
+ from mindsdb.api.executor.exceptions import WrongCharsetError
85
84
 
86
85
  logger = log.getLogger(__name__)
87
86
 
@@ -128,6 +127,14 @@ class SQLAnswer:
128
127
  raise ValueError(f"Unsupported response type for dump HTTP response: {self.resp_type}")
129
128
 
130
129
 
130
+ class MysqlTCPServer(SocketServer.ThreadingTCPServer):
131
+ """
132
+ Custom TCP Server with increased request queue size
133
+ """
134
+
135
+ request_queue_size = 30
136
+
137
+
131
138
  class MysqlProxy(SocketServer.BaseRequestHandler):
132
139
  """
133
140
  The Main Server controller class
@@ -428,7 +435,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
428
435
  try:
429
436
  return text.decode("utf-8")
430
437
  except Exception:
431
- raise ErWrongCharset(f"SQL contains non utf-8 values: {text}")
438
+ raise WrongCharsetError(f"SQL contains non utf-8 values: {text}")
432
439
 
433
440
  def is_cloud_connection(self):
434
441
  """Determine source of connection. Must be call before handshake.
@@ -489,6 +496,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
489
496
 
490
497
  @profiler.profile()
491
498
  def process_query(self, sql) -> SQLAnswer:
499
+ log.log_ram_info(logger)
492
500
  executor = Executor(session=self.session, sqlserver=self)
493
501
  executor.query_execute(sql)
494
502
  executor_answer = executor.executor_answer
@@ -647,8 +655,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
647
655
  try:
648
656
  success = p.get()
649
657
  except Exception:
650
- logger.error("Session closed, on packet read error")
651
- logger.error(traceback.format_exc())
658
+ logger.exception("Session closed, on packet read error:")
652
659
  return
653
660
 
654
661
  if success is False:
@@ -712,59 +719,28 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
712
719
  else:
713
720
  logger.warning("Command has no specific handler, return OK msg")
714
721
  logger.debug(str(p))
715
- # p.pprintPacket() TODO: Make a version of print packet
716
- # that sends it to debug instead
717
722
  response = SQLAnswer(RESPONSE_TYPE.OK)
718
723
 
719
- except SqlApiException as e:
720
- # classified error
721
- error_type = "expected"
722
-
723
- response = SQLAnswer(
724
- resp_type=RESPONSE_TYPE.ERROR,
725
- error_code=e.err_code,
726
- error_message=str(e),
727
- )
728
-
729
- except exec_exc.ExecutorException as e:
730
- # unclassified
731
- error_type = "expected"
732
-
733
- if isinstance(e, exec_exc.NotSupportedYet):
734
- error_code = ERR.ER_NOT_SUPPORTED_YET
735
- elif isinstance(e, exec_exc.KeyColumnDoesNotExist):
736
- error_code = ERR.ER_KEY_COLUMN_DOES_NOT_EXIST
737
- elif isinstance(e, exec_exc.TableNotExistError):
738
- error_code = ERR.ER_TABLE_EXISTS_ERROR
739
- elif isinstance(e, exec_exc.WrongArgumentError):
740
- error_code = ERR.ER_WRONG_ARGUMENTS
741
- elif isinstance(e, exec_exc.LogicError):
742
- error_code = ERR.ER_WRONG_USAGE
743
- elif isinstance(e, (exec_exc.BadDbError, exec_exc.BadTableError)):
744
- error_code = ERR.ER_BAD_DB_ERROR
724
+ except (QueryError, executor_exceptions.ExecutorException, executor_exceptions.UnknownError) as e:
725
+ error_type = "expected" if e.is_expected else "unexpected"
726
+ error_code = e.mysql_error_code
727
+ if e.is_expected:
728
+ if logger.isEnabledFor(logging.DEBUG):
729
+ logger.info("Query execution failed with expected error:", exc_info=True)
730
+ else:
731
+ logger.info(f"Query execution failed with expected error: {e}")
745
732
  else:
746
- error_code = ERR.ER_SYNTAX_ERROR
747
-
733
+ logger.exception("Query execution failed with error")
748
734
  response = SQLAnswer(
749
735
  resp_type=RESPONSE_TYPE.ERROR,
750
736
  error_code=error_code,
751
737
  error_message=str(e),
752
738
  )
753
- except exec_exc.UnknownError as e:
754
- # unclassified
755
- error_type = "unexpected"
756
-
757
- response = SQLAnswer(
758
- resp_type=RESPONSE_TYPE.ERROR,
759
- error_code=ERR.ER_UNKNOWN_ERROR,
760
- error_message=str(e),
761
- )
762
739
 
763
740
  except Exception as e:
764
- # any other exception
765
741
  error_type = "unexpected"
766
742
  error_traceback = traceback.format_exc()
767
- logger.error(f"ERROR while executing query\n{error_traceback}\n{e}")
743
+ logger.exception("ERROR while executing query:")
768
744
  error_code = ERR.ER_SYNTAX_ERROR
769
745
  response = SQLAnswer(
770
746
  resp_type=RESPONSE_TYPE.ERROR,
@@ -856,7 +832,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
856
832
  logger.info(f"Starting MindsDB Mysql proxy server on tcp://{host}:{port}")
857
833
 
858
834
  SocketServer.TCPServer.allow_reuse_address = True
859
- server = SocketServer.ThreadingTCPServer((host, port), MysqlProxy)
835
+ server = MysqlTCPServer((host, port), MysqlProxy)
860
836
  server.mindsdb_config = config
861
837
  server.check_auth = partial(check_auth, config=config)
862
838
  server.cert_path = cert_path
@@ -1 +0,0 @@
1
- from .exceptions import *
@@ -10,7 +10,7 @@ from mindsdb.api.executor.sql_query import SQLQuery
10
10
  from mindsdb.api.executor.sql_query.result_set import Column
11
11
  from mindsdb.api.mysql.mysql_proxy.utilities.lightwood_dtype import dtype
12
12
  from mindsdb.api.executor.command_executor import ExecuteCommands
13
- from mindsdb.api.mysql.mysql_proxy.utilities import SqlApiException
13
+ from mindsdb.api.executor.exceptions import SqlSyntaxError
14
14
  from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_fields import POSTGRES_TYPES
15
15
  from mindsdb.utilities import log
16
16
 
@@ -47,12 +47,10 @@ class Executor:
47
47
  self.query = parse_sql(sql)
48
48
  except Exception as mdb_error:
49
49
  # not all statements are parsed by parse_sql
50
- self.logger.warning('Failed to parse SQL query')
51
- self.logger.debug(f'Query that cannot be parsed: {sql}')
50
+ self.logger.warning("Failed to parse SQL query")
51
+ self.logger.debug(f"Query that cannot be parsed: {sql}")
52
52
 
53
- raise SqlApiException(
54
- f"The SQL statement cannot be parsed - {sql}: {mdb_error}"
55
- ) from mdb_error
53
+ raise SqlSyntaxError(f"The SQL statement cannot be parsed - {sql}: {mdb_error}") from mdb_error
56
54
 
57
55
  def stmt_execute(self, param_values):
58
56
  if self.is_executed:
@@ -102,20 +100,15 @@ class Executor:
102
100
  "state_track": self.state_track,
103
101
  "server_status": self.server_status,
104
102
  "is_executed": self.is_executed,
105
- "session": self.session.to_json()
106
-
103
+ "session": self.session.to_json(),
107
104
  }
108
105
  return params
109
106
 
110
107
  def to_postgres_columns(self, columns):
111
-
112
108
  result = []
113
109
 
114
- database = (
115
- None if self.session.database == "" else self.session.database.lower()
116
- )
110
+ database = None if self.session.database == "" else self.session.database.lower()
117
111
  for column_record in columns:
118
-
119
112
  field_type = column_record.type
120
113
 
121
114
  column_type = POSTGRES_TYPES.VARCHAR
@@ -7,8 +7,11 @@ import time
7
7
  from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_fields import PostgresField
8
8
 
9
9
 
10
- from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_identifiers import \
11
- PostgresBackendMessageIdentifier, PostgresFrontendMessageIdentifier, PostgresAuthType
10
+ from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_identifiers import (
11
+ PostgresBackendMessageIdentifier,
12
+ PostgresFrontendMessageIdentifier,
13
+ PostgresAuthType,
14
+ )
12
15
 
13
16
 
14
17
  class PostgresEmptyDataException(Exception):
@@ -29,8 +32,11 @@ class UnsupportedPostgresMessageType(Exception):
29
32
 
30
33
  class PostgresPacketReader:
31
34
  def __init__(self, buffer: BinaryIO):
32
- from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_formats import FE_MESSAGE_MAP, \
33
- SUPPORTED_AUTH_TYPES
35
+ from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_formats import (
36
+ FE_MESSAGE_MAP,
37
+ SUPPORTED_AUTH_TYPES,
38
+ )
39
+
34
40
  self.fe_message_map = FE_MESSAGE_MAP
35
41
  self.supported_auth_types = SUPPORTED_AUTH_TYPES
36
42
  self.buffer = buffer
@@ -80,7 +86,7 @@ class PostgresPacketReader:
80
86
 
81
87
  def read_parameters(self, n):
82
88
  data = self.read_bytes(n)
83
- return data.split(b'\x00')
89
+ return data.split(b"\x00")
84
90
 
85
91
  def read_verify_ssl_request(self):
86
92
  self.logger.debug("reading ssl")
@@ -94,9 +100,9 @@ class PostgresPacketReader:
94
100
  length = self.read_int32()
95
101
  version = self.read_int32()
96
102
  major_version = version >> 16
97
- minor_version = version & 0xffff
103
+ minor_version = version & 0xFFFF
98
104
  message = self.read_parameters(length - 8)
99
- self.logger.debug('PSQL Startup Message %d.%d : %s' % (major_version, minor_version, message))
105
+ self.logger.debug("PSQL Startup Message %d.%d : %s" % (major_version, minor_version, message))
100
106
  parameters = {}
101
107
  while len(message) != 0:
102
108
  key = message.pop(0)
@@ -111,7 +117,7 @@ class PostgresPacketReader:
111
117
  auth_type = self.read_byte()
112
118
  except PostgresEmptyDataException:
113
119
  # No authentication parameters specified. Which is fine if we're local on a mindsdbuser
114
- return ''
120
+ return ""
115
121
  try:
116
122
  auth_type = PostgresAuthType(auth_type)
117
123
  except Exception as e:
@@ -119,27 +125,31 @@ class PostgresPacketReader:
119
125
  if auth_type not in self.supported_auth_types:
120
126
  raise UnsupportedPostgresAuthException("%s is not a supported auth type identifier" % auth_type)
121
127
  length = self.read_int32()
122
- password = strip_null_byte(self.read_bytes(length - 4), encoding=encoding) # password. Do something with later. We read to clear buffer.
128
+ password = strip_null_byte(
129
+ self.read_bytes(length - 4), encoding=encoding
130
+ ) # password. Do something with later. We read to clear buffer.
123
131
  return password
124
132
 
125
133
  def read_message(self):
126
134
  try:
127
135
  message_type = self.read_byte()
128
136
  except PostgresEmptyDataException:
129
- self.logger.warn("Postgres Proxy: Received empty data string")
137
+ self.logger.warning("Postgres Proxy: Received empty data string")
130
138
  return None
131
139
  try:
132
140
  message_type = PostgresFrontendMessageIdentifier(message_type)
133
141
  except Exception as e:
134
142
  raise UnsupportedPostgresMessageType(
135
- "%s is not a supported frontend message identifier:\n%s" % (message_type, str(e)))
143
+ "%s is not a supported frontend message identifier:\n%s" % (message_type, str(e))
144
+ )
136
145
 
137
146
  if message_type in self.fe_message_map:
138
147
  self.logger.debug("reading message type %s" % str(message_type.name))
139
148
  return self.fe_message_map[message_type]().read(self)
140
149
  else:
141
150
  raise UnsupportedPostgresMessageType(
142
- "%s is not a supported frontend message identifier" % message_type.value)
151
+ "%s is not a supported frontend message identifier" % message_type.value
152
+ )
143
153
 
144
154
 
145
155
  class PostgresPacketBuilder:
@@ -153,8 +163,8 @@ class PostgresPacketBuilder:
153
163
  self.reset()
154
164
 
155
165
  def reset(self):
156
- self.identifier = b''
157
- self.pack_string = ''
166
+ self.identifier = b""
167
+ self.pack_string = ""
158
168
  self.pack_args = []
159
169
  self.length = 0
160
170
 
@@ -194,44 +204,46 @@ class PostgresPacketBuilder:
194
204
  write_file.write(packed_binary)
195
205
 
196
206
  def add_char(self, s: bytes):
197
- self.pack_string += 'c'
207
+ self.pack_string += "c"
198
208
  if len(s) != 1:
199
209
  raise Exception("Char must be of length 1 in add_char")
200
210
  self.pack_args.append(s)
201
211
  return self.add_length(1)
202
212
 
203
213
  def add_string(self, s: bytes):
204
- s = s + b'\x00'
205
- self.pack_string += str(len(s)) + 's'
214
+ s = s + b"\x00"
215
+ self.pack_string += str(len(s)) + "s"
206
216
  self.pack_args.append(s)
207
217
  return self.add_length(len(s))
208
218
 
209
219
  def add_int32(self, i):
210
- self.pack_string += 'i'
220
+ self.pack_string += "i"
211
221
  self.pack_args.append(i)
212
222
  return self.add_length(4)
213
223
 
214
224
  def add_int16(self, h):
215
- self.pack_string += 'h'
225
+ self.pack_string += "h"
216
226
  self.pack_args.append(h)
217
227
  return self.add_length(2)
218
228
 
219
229
  def add_bytes(self, b: bytes):
220
230
  if len(b) == 1:
221
- self.pack_string += 's'
231
+ self.pack_string += "s"
222
232
  else:
223
- self.pack_string += str(len(b)) + 's'
233
+ self.pack_string += str(len(b)) + "s"
224
234
  self.pack_args.append(b)
225
235
  return self.add_length(len(b))
226
236
 
227
237
  def add_field(self, field: PostgresField):
228
- return self.add_string(field.name.encode()) \
229
- .add_int32(field.table_id) \
230
- .add_int16(field.column_id) \
231
- .add_int32(field.object_id) \
232
- .add_int16(field.dt_size) \
233
- .add_int32(field.type_modifier) \
238
+ return (
239
+ self.add_string(field.name.encode())
240
+ .add_int32(field.table_id)
241
+ .add_int16(field.column_id)
242
+ .add_int32(field.object_id)
243
+ .add_int16(field.dt_size)
244
+ .add_int32(field.type_modifier)
234
245
  .add_int16(field.format_code)
246
+ )
235
247
 
236
248
  def add_fields(self, fields: Sequence[PostgresField]):
237
249
  for field in fields:
@@ -239,7 +251,7 @@ class PostgresPacketBuilder:
239
251
  return self
240
252
 
241
253
  def add_column_value(self, val: bytes):
242
- if val == b'NULL':
254
+ if val == b"NULL":
243
255
  self.add_int32(-1)
244
256
  return self
245
257