MindsDB 25.9.2.0a1__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of MindsDB might be problematic. Click here for more details.

Files changed (116) hide show
  1. mindsdb/__about__.py +1 -1
  2. mindsdb/__main__.py +39 -20
  3. mindsdb/api/a2a/agent.py +7 -9
  4. mindsdb/api/a2a/common/server/server.py +3 -3
  5. mindsdb/api/a2a/common/server/task_manager.py +4 -4
  6. mindsdb/api/a2a/task_manager.py +15 -17
  7. mindsdb/api/common/middleware.py +9 -11
  8. mindsdb/api/executor/command_executor.py +2 -4
  9. mindsdb/api/executor/datahub/datanodes/datanode.py +2 -2
  10. mindsdb/api/executor/datahub/datanodes/integration_datanode.py +100 -48
  11. mindsdb/api/executor/datahub/datanodes/project_datanode.py +8 -4
  12. mindsdb/api/executor/datahub/datanodes/system_tables.py +1 -1
  13. mindsdb/api/executor/exceptions.py +29 -10
  14. mindsdb/api/executor/planner/plan_join.py +17 -3
  15. mindsdb/api/executor/sql_query/sql_query.py +74 -74
  16. mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +1 -2
  17. mindsdb/api/executor/sql_query/steps/subselect_step.py +0 -1
  18. mindsdb/api/executor/utilities/functions.py +6 -6
  19. mindsdb/api/executor/utilities/sql.py +32 -16
  20. mindsdb/api/http/gui.py +5 -11
  21. mindsdb/api/http/initialize.py +8 -10
  22. mindsdb/api/http/namespaces/agents.py +10 -12
  23. mindsdb/api/http/namespaces/analysis.py +13 -20
  24. mindsdb/api/http/namespaces/auth.py +1 -1
  25. mindsdb/api/http/namespaces/config.py +15 -11
  26. mindsdb/api/http/namespaces/databases.py +140 -201
  27. mindsdb/api/http/namespaces/file.py +15 -4
  28. mindsdb/api/http/namespaces/handlers.py +7 -2
  29. mindsdb/api/http/namespaces/knowledge_bases.py +8 -7
  30. mindsdb/api/http/namespaces/models.py +94 -126
  31. mindsdb/api/http/namespaces/projects.py +13 -22
  32. mindsdb/api/http/namespaces/sql.py +33 -25
  33. mindsdb/api/http/namespaces/tab.py +27 -37
  34. mindsdb/api/http/namespaces/views.py +1 -1
  35. mindsdb/api/http/start.py +14 -8
  36. mindsdb/api/mcp/__init__.py +2 -1
  37. mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +15 -20
  38. mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +26 -50
  39. mindsdb/api/mysql/mysql_proxy/utilities/__init__.py +0 -1
  40. mindsdb/api/postgres/postgres_proxy/executor/executor.py +6 -13
  41. mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +40 -28
  42. mindsdb/integrations/handlers/byom_handler/byom_handler.py +168 -185
  43. mindsdb/integrations/handlers/file_handler/file_handler.py +7 -0
  44. mindsdb/integrations/handlers/lightwood_handler/functions.py +45 -79
  45. mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +13 -1
  46. mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +25 -12
  47. mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +2 -1
  48. mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +1 -0
  49. mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +1 -0
  50. mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +4 -4
  51. mindsdb/integrations/libs/api_handler.py +10 -10
  52. mindsdb/integrations/libs/base.py +4 -4
  53. mindsdb/integrations/libs/llm/utils.py +2 -2
  54. mindsdb/integrations/libs/ml_handler_process/create_engine_process.py +4 -7
  55. mindsdb/integrations/libs/ml_handler_process/func_call_process.py +2 -7
  56. mindsdb/integrations/libs/ml_handler_process/learn_process.py +37 -47
  57. mindsdb/integrations/libs/ml_handler_process/update_engine_process.py +4 -7
  58. mindsdb/integrations/libs/ml_handler_process/update_process.py +2 -7
  59. mindsdb/integrations/libs/process_cache.py +132 -140
  60. mindsdb/integrations/libs/response.py +18 -12
  61. mindsdb/integrations/libs/vectordatabase_handler.py +26 -0
  62. mindsdb/integrations/utilities/files/file_reader.py +6 -7
  63. mindsdb/integrations/utilities/rag/config_loader.py +37 -26
  64. mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +59 -9
  65. mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py +4 -4
  66. mindsdb/integrations/utilities/rag/retrievers/sql_retriever.py +55 -133
  67. mindsdb/integrations/utilities/rag/settings.py +58 -133
  68. mindsdb/integrations/utilities/rag/splitters/file_splitter.py +5 -15
  69. mindsdb/interfaces/agents/agents_controller.py +2 -1
  70. mindsdb/interfaces/agents/constants.py +0 -2
  71. mindsdb/interfaces/agents/litellm_server.py +34 -58
  72. mindsdb/interfaces/agents/mcp_client_agent.py +10 -10
  73. mindsdb/interfaces/agents/mindsdb_database_agent.py +5 -5
  74. mindsdb/interfaces/agents/run_mcp_agent.py +12 -21
  75. mindsdb/interfaces/chatbot/chatbot_task.py +20 -23
  76. mindsdb/interfaces/chatbot/polling.py +30 -18
  77. mindsdb/interfaces/data_catalog/data_catalog_loader.py +10 -10
  78. mindsdb/interfaces/database/integrations.py +19 -2
  79. mindsdb/interfaces/file/file_controller.py +6 -6
  80. mindsdb/interfaces/functions/controller.py +1 -1
  81. mindsdb/interfaces/functions/to_markdown.py +2 -2
  82. mindsdb/interfaces/jobs/jobs_controller.py +5 -5
  83. mindsdb/interfaces/jobs/scheduler.py +3 -8
  84. mindsdb/interfaces/knowledge_base/controller.py +50 -23
  85. mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +40 -61
  86. mindsdb/interfaces/model/model_controller.py +170 -166
  87. mindsdb/interfaces/query_context/context_controller.py +14 -2
  88. mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +6 -4
  89. mindsdb/interfaces/skills/retrieval_tool.py +43 -50
  90. mindsdb/interfaces/skills/skill_tool.py +2 -2
  91. mindsdb/interfaces/skills/sql_agent.py +25 -19
  92. mindsdb/interfaces/storage/fs.py +114 -169
  93. mindsdb/interfaces/storage/json.py +19 -18
  94. mindsdb/interfaces/tabs/tabs_controller.py +49 -72
  95. mindsdb/interfaces/tasks/task_monitor.py +3 -9
  96. mindsdb/interfaces/tasks/task_thread.py +7 -9
  97. mindsdb/interfaces/triggers/trigger_task.py +7 -13
  98. mindsdb/interfaces/triggers/triggers_controller.py +47 -50
  99. mindsdb/migrations/migrate.py +16 -16
  100. mindsdb/utilities/api_status.py +58 -0
  101. mindsdb/utilities/config.py +49 -0
  102. mindsdb/utilities/exception.py +40 -1
  103. mindsdb/utilities/fs.py +0 -1
  104. mindsdb/utilities/hooks/profiling.py +17 -14
  105. mindsdb/utilities/langfuse.py +40 -45
  106. mindsdb/utilities/log.py +272 -0
  107. mindsdb/utilities/ml_task_queue/consumer.py +52 -58
  108. mindsdb/utilities/ml_task_queue/producer.py +26 -30
  109. mindsdb/utilities/render/sqlalchemy_render.py +7 -6
  110. mindsdb/utilities/utils.py +2 -2
  111. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/METADATA +269 -264
  112. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/RECORD +115 -115
  113. mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -14
  114. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/WHEEL +0 -0
  115. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/licenses/LICENSE +0 -0
  116. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/top_level.txt +0 -0
mindsdb/__about__.py CHANGED
@@ -1,6 +1,6 @@
1
1
  __title__ = "MindsDB"
2
2
  __package_name__ = "mindsdb"
3
- __version__ = "25.9.2.0a1"
3
+ __version__ = "25.9.3rc1"
4
4
  __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
5
5
  __email__ = "jorge@mindsdb.com"
6
6
  __author__ = "MindsDB Inc"
mindsdb/__main__.py CHANGED
@@ -8,8 +8,8 @@ import atexit
8
8
  import signal
9
9
  import psutil
10
10
  import asyncio
11
- import traceback
12
11
  import threading
12
+ import shutil
13
13
  from enum import Enum
14
14
  from dataclasses import dataclass, field
15
15
  from typing import Callable, Optional, Tuple, List
@@ -39,6 +39,7 @@ from mindsdb.utilities.fs import clean_process_marks, clean_unlinked_process_mar
39
39
  from mindsdb.utilities.context import context as ctx
40
40
  from mindsdb.utilities.auth import register_oauth_client, get_aws_meta_data
41
41
  from mindsdb.utilities.sentry import sentry_sdk # noqa: F401
42
+ from mindsdb.utilities.api_status import set_api_status
42
43
 
43
44
  try:
44
45
  import torch.multiprocessing as mp
@@ -152,6 +153,16 @@ def close_api_gracefully(trunc_processes_struct):
152
153
  sys.exit(0)
153
154
 
154
155
 
156
+ def clean_mindsdb_tmp_dir():
157
+ """Clean the MindsDB tmp dir at exit."""
158
+ temp_dir = config["paths"]["tmp"]
159
+ for file in temp_dir.iterdir():
160
+ if file.is_dir():
161
+ shutil.rmtree(file)
162
+ else:
163
+ file.unlink()
164
+
165
+
155
166
  def set_error_model_status_by_pids(unexisting_pids: List[int]):
156
167
  """Models have id of its traiing process in the 'training_metadata' field.
157
168
  If the pid does not exist, we should set the model status to "error".
@@ -217,19 +228,20 @@ def create_permanent_integrations():
217
228
  """
218
229
  integration_name = "files"
219
230
  existing = db.session.query(db.Integration).filter_by(name=integration_name, company_id=None).first()
220
- if existing is None:
221
- integration_record = db.Integration(
222
- name=integration_name,
223
- data={},
224
- engine=integration_name,
225
- company_id=None,
226
- )
227
- db.session.add(integration_record)
228
- try:
229
- db.session.commit()
230
- except Exception as e:
231
- logger.error(f"Failed to commit permanent integration {integration_name}: {e}")
232
- db.session.rollback()
231
+ if existing is not None:
232
+ return
233
+ integration_record = db.Integration(
234
+ name=integration_name,
235
+ data={},
236
+ engine=integration_name,
237
+ company_id=None,
238
+ )
239
+ db.session.add(integration_record)
240
+ try:
241
+ db.session.commit()
242
+ except Exception:
243
+ logger.exception(f"Failed to create permanent integration '{integration_name}' in the internal database.")
244
+ db.session.rollback()
233
245
 
234
246
 
235
247
  def validate_default_project() -> None:
@@ -289,7 +301,7 @@ def start_process(trunc_process_data: TrunkProcessData) -> None:
289
301
  )
290
302
  trunc_process_data.process.start()
291
303
  except Exception as e:
292
- logger.error(f"Failed to start {trunc_process_data.name} API with exception {e}\n{traceback.format_exc()}")
304
+ logger.exception(f"Failed to start '{trunc_process_data.name}' API process due to unexpected error:")
293
305
  close_api_gracefully(trunc_processes_struct)
294
306
  raise e
295
307
 
@@ -363,8 +375,8 @@ if __name__ == "__main__":
363
375
  if environment == "aws_marketplace":
364
376
  try:
365
377
  register_oauth_client()
366
- except Exception as e:
367
- logger.error(f"Something went wrong during client register: {e}")
378
+ except Exception:
379
+ logger.exception("Something went wrong during client register:")
368
380
  elif environment != "local":
369
381
  try:
370
382
  aws_meta_data = get_aws_meta_data()
@@ -384,6 +396,7 @@ if __name__ == "__main__":
384
396
  logger.info(f"Version: {mindsdb_version}")
385
397
  logger.info(f"Configuration file: {config.config_path or 'absent'}")
386
398
  logger.info(f"Storage path: {config.paths['root']}")
399
+ log.log_system_info(logger)
387
400
  logger.debug(f"User config: {config.user_config}")
388
401
  logger.debug(f"System config: {config.auto_config}")
389
402
  logger.debug(f"Env config: {config.env_config}")
@@ -391,13 +404,12 @@ if __name__ == "__main__":
391
404
  is_cloud = config.is_cloud
392
405
  unexisting_pids = clean_unlinked_process_marks()
393
406
  if not is_cloud:
394
- logger.debug("Applying database migrations")
395
407
  try:
396
408
  from mindsdb.migrations import migrate
397
409
 
398
410
  migrate.migrate_to_head()
399
- except Exception as e:
400
- logger.error(f"Error! Something went wrong during DB migrations: {e}")
411
+ except Exception:
412
+ logger.exception("Failed to apply database migrations. This may prevent MindsDB from operating correctly:")
401
413
 
402
414
  validate_default_project()
403
415
 
@@ -484,8 +496,12 @@ if __name__ == "__main__":
484
496
  if trunc_process_data.started is True or trunc_process_data.need_to_run is False:
485
497
  continue
486
498
  start_process(trunc_process_data)
499
+ # Set status for APIs without ports (they don't go through wait_api_start)
500
+ if trunc_process_data.port is None:
501
+ set_api_status(trunc_process_data.name, True)
487
502
 
488
503
  atexit.register(close_api_gracefully, trunc_processes_struct=trunc_processes_struct)
504
+ atexit.register(clean_mindsdb_tmp_dir)
489
505
 
490
506
  async def wait_api_start(api_name, pid, port):
491
507
  timeout = 60
@@ -494,6 +510,9 @@ if __name__ == "__main__":
494
510
  while (time.time() - start_time) < timeout and started is False:
495
511
  await asyncio.sleep(0.5)
496
512
  started = is_pid_listen_port(pid, port)
513
+
514
+ set_api_status(api_name, started)
515
+
497
516
  return api_name, port, started
498
517
 
499
518
  async def wait_apis_start():
mindsdb/api/a2a/agent.py CHANGED
@@ -73,17 +73,15 @@ class MindsDBAgent:
73
73
  "parts": [{"type": "text", "text": error_msg}],
74
74
  }
75
75
  except requests.exceptions.RequestException as e:
76
- error_msg = f"Error connecting to MindsDB: {str(e)}"
77
- logger.error(error_msg)
76
+ logger.exception("Error connecting to MindsDB:")
78
77
  return {
79
- "content": error_msg,
78
+ "content": f"Error connecting to MindsDB: {e}",
80
79
  "parts": [{"type": "text", "text": error_msg}],
81
80
  }
82
81
  except Exception as e:
83
- error_msg = f"Error: {str(e)}"
84
- logger.error(error_msg)
82
+ logger.exception("Error: ")
85
83
  return {
86
- "content": error_msg,
84
+ "content": f"Error: {e}",
87
85
  "parts": [{"type": "text", "text": error_msg}],
88
86
  }
89
87
 
@@ -102,7 +100,7 @@ class MindsDBAgent:
102
100
  try:
103
101
  yield json.loads(payload)
104
102
  except Exception as e:
105
- logger.error(f"Failed to parse SSE JSON payload: {e}; line: {payload}")
103
+ logger.exception(f"Failed to parse SSE JSON payload: {e}; line: {payload}")
106
104
  # Ignore comments or control lines
107
105
  # Signal the end of the stream
108
106
  yield {"is_task_complete": True}
@@ -129,13 +127,13 @@ class MindsDBAgent:
129
127
  wrapped_chunk = {"is_task_complete": False, "content": content_value, "metadata": {}}
130
128
  yield wrapped_chunk
131
129
  except Exception as e:
132
- logger.error(f"Error in streaming: {str(e)}")
130
+ logger.exception(f"Error in streaming: {e}")
133
131
  yield {
134
132
  "is_task_complete": True,
135
133
  "parts": [
136
134
  {
137
135
  "type": "text",
138
- "text": f"Error: {str(e)}",
136
+ "text": f"Error: {e}",
139
137
  }
140
138
  ],
141
139
  "metadata": {
@@ -118,7 +118,7 @@ class A2AServer:
118
118
  elif isinstance(e, ValidationError):
119
119
  json_rpc_error = InvalidRequestError(data=json.loads(e.json()))
120
120
  else:
121
- logger.error(f"Unhandled exception: {e}")
121
+ logger.exception("Unhandled exception:")
122
122
  json_rpc_error = InternalError()
123
123
 
124
124
  response = JSONRPCResponse(id=None, error=json_rpc_error)
@@ -137,8 +137,8 @@ class A2AServer:
137
137
  else:
138
138
  data = json.dumps(item)
139
139
  except Exception as e:
140
- logger.error(f"Serialization error in SSE stream: {e}")
141
- data = json.dumps({"error": f"Serialization error: {str(e)}"})
140
+ logger.exception("Serialization error in SSE stream:")
141
+ data = json.dumps({"error": f"Serialization error: {e}"})
142
142
  yield {"data": data}
143
143
 
144
144
  # Add robust SSE headers for compatibility
@@ -152,8 +152,8 @@ class InMemoryTaskManager(TaskManager):
152
152
  task_notification_params.id,
153
153
  task_notification_params.pushNotificationConfig,
154
154
  )
155
- except Exception as e:
156
- logger.error(f"Error while setting push notification info: {e}")
155
+ except Exception:
156
+ logger.exception("Error while setting push notification info:")
157
157
  return JSONRPCResponse(
158
158
  id=request.id,
159
159
  error=InternalError(message="An error occurred while setting push notification info"),
@@ -169,8 +169,8 @@ class InMemoryTaskManager(TaskManager):
169
169
 
170
170
  try:
171
171
  notification_info = await self.get_push_notification_info(task_params.id)
172
- except Exception as e:
173
- logger.error(f"Error while getting push notification info: {e}")
172
+ except Exception:
173
+ logger.exception("Error while getting push notification info:")
174
174
  return GetTaskPushNotificationResponse(
175
175
  id=request.id,
176
176
  error=InternalError(message="An error occurred while getting push notification info"),
@@ -1,4 +1,8 @@
1
- from typing import AsyncIterable, Dict
1
+ import time
2
+ import logging
3
+ import asyncio
4
+ from typing import AsyncIterable, Dict, Union
5
+
2
6
  from mindsdb.api.a2a.common.types import (
3
7
  SendTaskRequest,
4
8
  TaskSendParams,
@@ -20,11 +24,6 @@ from mindsdb.api.a2a.common.server.task_manager import InMemoryTaskManager
20
24
  from mindsdb.api.a2a.agent import MindsDBAgent
21
25
  from mindsdb.api.a2a.utils import to_serializable, convert_a2a_message_to_qa_format
22
26
 
23
- from typing import Union
24
- import logging
25
- import asyncio
26
- import time
27
- import traceback
28
27
 
29
28
  logger = logging.getLogger(__name__)
30
29
 
@@ -80,11 +79,11 @@ class AgentTaskManager(InMemoryTaskManager):
80
79
  task = await self.upsert_task(task_send_params)
81
80
  logger.info(f"Task created/updated with history length: {len(task.history) if task.history else 0}")
82
81
  except Exception as e:
83
- logger.error(f"Error creating task: {str(e)}")
82
+ logger.exception("Error creating task:")
84
83
  error_result = to_serializable(
85
84
  {
86
85
  "id": request.id,
87
- "error": to_serializable(InternalError(message=f"Error creating task: {str(e)}")),
86
+ "error": to_serializable(InternalError(message=f"Error creating task: {e}")),
88
87
  }
89
88
  )
90
89
  yield error_result
@@ -149,14 +148,14 @@ class AgentTaskManager(InMemoryTaskManager):
149
148
  return
150
149
 
151
150
  except Exception as e:
152
- logger.error(f"Error invoking agent: {e}")
151
+ logger.exception("Error invoking agent:")
153
152
  error_result = to_serializable(
154
153
  {
155
154
  "id": request.id,
156
155
  "error": to_serializable(
157
156
  JSONRPCResponse(
158
157
  id=request.id,
159
- error=to_serializable(InternalError(message=f"Error invoking agent: {str(e)}")),
158
+ error=to_serializable(InternalError(message=f"Error invoking agent: {e}")),
160
159
  )
161
160
  ),
162
161
  }
@@ -182,11 +181,10 @@ class AgentTaskManager(InMemoryTaskManager):
182
181
  item["artifact"]["parts"] = [to_serializable(p) for p in item["artifact"]["parts"]]
183
182
  yield to_serializable(item)
184
183
  except Exception as e:
185
- logger.error(f"An error occurred while streaming the response: {e}")
186
- logger.error(traceback.format_exc())
187
- error_text = f"An error occurred while streaming the response: {str(e)}"
184
+ error_text = "An error occurred while streaming the response:"
185
+ logger.exception(error_text)
188
186
  # Ensure all parts are plain dicts
189
- parts = [{"type": "text", "text": error_text}]
187
+ parts = [{"type": "text", "text": f"{error_text} {e}"}]
190
188
  parts = [to_serializable(part) for part in parts]
191
189
  artifact = {
192
190
  "parts": parts,
@@ -333,11 +331,11 @@ class AgentTaskManager(InMemoryTaskManager):
333
331
  yield response
334
332
  except Exception as e:
335
333
  # If an error occurs, yield an error response
336
- logger.error(f"Error in on_send_task_subscribe: {str(e)}")
334
+ logger.exception(f"Error in on_send_task_subscribe: {e}")
337
335
  error_result = to_serializable(
338
336
  {
339
337
  "id": request.id,
340
- "error": to_serializable(InternalError(message=f"Error processing streaming request: {str(e)}")),
338
+ "error": to_serializable(InternalError(message=f"Error processing streaming request: {e}")),
341
339
  }
342
340
  )
343
341
  yield error_result
@@ -463,7 +461,7 @@ class AgentTaskManager(InMemoryTaskManager):
463
461
  )
464
462
  return to_serializable(SendTaskResponse(id=request.id, result=task))
465
463
  except Exception as e:
466
- logger.error(f"Error invoking agent: {e}")
464
+ logger.exception("Error invoking agent:")
467
465
  result_text = f"Error invoking agent: {e}"
468
466
  parts = [{"type": "text", "text": result_text}]
469
467
 
@@ -1,13 +1,13 @@
1
+ import os
2
+ import hmac
3
+ import secrets
4
+ import hashlib
5
+ from http import HTTPStatus
6
+ from typing import Optional
7
+
1
8
  from starlette.middleware.base import BaseHTTPMiddleware
2
9
  from starlette.responses import JSONResponse
3
10
  from starlette.requests import Request
4
- from http import HTTPStatus
5
- from typing import Optional
6
- import secrets
7
- import hmac
8
- import hashlib
9
- import os
10
- import traceback
11
11
 
12
12
  from mindsdb.utilities import log
13
13
  from mindsdb.utilities.config import config
@@ -100,7 +100,5 @@ def check_auth(username, password, scramble_func, salt, company_id, config):
100
100
 
101
101
  logger.info(f"Check auth, user={username}: Ok")
102
102
  return {"success": True, "username": username}
103
- except Exception as e:
104
- logger.error(f"Check auth, user={username}: ERROR")
105
- logger.error(e)
106
- logger.error(traceback.format_exc())
103
+ except Exception:
104
+ logger.exception(f"Check auth, user={username}: ERROR")
@@ -757,8 +757,6 @@ class ExecuteCommands:
757
757
  except EntityNotExistsError:
758
758
  if statement.if_exists is False:
759
759
  raise
760
- except Exception as e:
761
- raise e
762
760
 
763
761
  return ExecuteAnswer()
764
762
 
@@ -844,7 +842,7 @@ class ExecuteCommands:
844
842
  try:
845
843
  sqlquery = SQLQuery(statement.data, session=self.session, database=database_name)
846
844
  except Exception as e:
847
- raise Exception(f'Nested query failed to execute with error: "{e}", please check and try again.')
845
+ raise Exception(f'Nested query failed to execute with error: "{e}", please check and try again.') from e
848
846
  df = sqlquery.fetched_data.to_df()
849
847
  df.columns = [str(t.alias) if hasattr(t, "alias") else str(t.parts[-1]) for t in statement.data.targets]
850
848
 
@@ -1210,7 +1208,7 @@ class ExecuteCommands:
1210
1208
  ast_drop = DropMLEngine(name=Identifier(name))
1211
1209
  self.answer_drop_ml_engine(ast_drop)
1212
1210
  logger.info(msg)
1213
- raise ExecutorException(msg)
1211
+ raise ExecutorException(msg) from e
1214
1212
 
1215
1213
  return ExecuteAnswer()
1216
1214
 
@@ -4,7 +4,7 @@ from mindsdb.api.executor.datahub.classes.response import DataHubResponse
4
4
 
5
5
 
6
6
  class DataNode:
7
- type = 'meta'
7
+ type = "meta"
8
8
 
9
9
  def __init__(self):
10
10
  pass
@@ -21,5 +21,5 @@ class DataNode:
21
21
  def get_table_columns_names(self, table_name: str, schema_name: str | None = None) -> list[str]:
22
22
  pass
23
23
 
24
- def query(self, query=None, native_query=None, session=None) -> DataHubResponse:
24
+ def query(self, query=None, session=None) -> DataHubResponse:
25
25
  pass
@@ -1,5 +1,6 @@
1
1
  import time
2
2
  import inspect
3
+ import functools
3
4
  from dataclasses import astuple
4
5
  from typing import Iterable, List
5
6
 
@@ -20,7 +21,7 @@ from mindsdb.integrations.utilities.utils import get_class_name
20
21
  from mindsdb.metrics import metrics
21
22
  from mindsdb.utilities import log
22
23
  from mindsdb.utilities.profiler import profiler
23
- from mindsdb.utilities.exception import format_db_error_message
24
+ from mindsdb.utilities.exception import QueryError
24
25
  from mindsdb.api.executor.datahub.datanodes.system_tables import infer_mysql_type
25
26
 
26
27
  logger = log.getLogger(__name__)
@@ -30,6 +31,51 @@ class DBHandlerException(Exception):
30
31
  pass
31
32
 
32
33
 
34
+ def collect_metrics(func):
35
+ """Decorator for collecting performance metrics if integration handler query.
36
+
37
+ The decorator measures:
38
+ - Query execution time using high-precision performance counter
39
+ - Response size (number of rows returned)
40
+
41
+ Args:
42
+ func: The function to be decorated (integration handler method)
43
+
44
+ Returns:
45
+ function: Wrapped function that includes metrics collection and error handling
46
+ """
47
+
48
+ @functools.wraps(func)
49
+ def wrapper(self, *args, **kwargs):
50
+ try:
51
+ time_before_query = time.perf_counter()
52
+ result = func(self, *args, **kwargs)
53
+
54
+ # metrics
55
+ handler_class_name = get_class_name(self.integration_handler)
56
+ elapsed_seconds = time.perf_counter() - time_before_query
57
+ query_time_with_labels = metrics.INTEGRATION_HANDLER_QUERY_TIME.labels(handler_class_name, result.type)
58
+ query_time_with_labels.observe(elapsed_seconds)
59
+
60
+ num_rows = 0
61
+ if result.data_frame is not None:
62
+ num_rows = len(result.data_frame.index)
63
+ response_size_with_labels = metrics.INTEGRATION_HANDLER_RESPONSE_SIZE.labels(
64
+ handler_class_name, result.type
65
+ )
66
+ response_size_with_labels.observe(num_rows)
67
+ logger.debug(f"Handler '{handler_class_name}' returned {num_rows} rows in {elapsed_seconds:.3f} seconds")
68
+ except Exception as e:
69
+ msg = str(e).strip()
70
+ if msg == "":
71
+ msg = e.__class__.__name__
72
+ msg = f"[{self.ds_type}/{self.integration_name}]: {msg}"
73
+ raise DBHandlerException(msg) from e
74
+ return result
75
+
76
+ return wrapper
77
+
78
+
33
79
  class IntegrationDataNode(DataNode):
34
80
  type = "integration"
35
81
 
@@ -46,16 +92,10 @@ class IntegrationDataNode(DataNode):
46
92
  response = self.integration_handler.get_tables()
47
93
  if response.type == RESPONSE_TYPE.TABLE:
48
94
  result_dict = response.data_frame.to_dict(orient="records")
49
- result = []
50
- for row in result_dict:
51
- result.append(TablesRow.from_dict(row))
52
- return result
95
+ return [TablesRow.from_dict(row) for row in result_dict]
53
96
  else:
54
97
  raise Exception(f"Can't get tables: {response.error_message}")
55
98
 
56
- result_dict = response.data_frame.to_dict(orient="records")
57
- return [TablesRow.from_dict(row) for row in result_dict]
58
-
59
99
  def get_table_columns_df(self, table_name: str, schema_name: str | None = None) -> pd.DataFrame:
60
100
  """Get a DataFrame containing representation of information_schema.columns for the specified table.
61
101
 
@@ -214,50 +254,54 @@ class IntegrationDataNode(DataNode):
214
254
  return self.integration_handler.query_stream(query, fetch_size=fetch_size)
215
255
 
216
256
  @profiler.profile()
217
- def query(self, query: ASTNode | None = None, native_query: str | None = None, session=None) -> DataHubResponse:
218
- try:
219
- time_before_query = time.perf_counter()
220
- if query is not None:
221
- result: HandlerResponse = self.integration_handler.query(query)
222
- else:
223
- # try to fetch native query
224
- result: HandlerResponse = self.integration_handler.native_query(native_query)
257
+ def query(self, query: ASTNode | str = None, session=None) -> DataHubResponse:
258
+ """Execute a query against the integration data source.
225
259
 
226
- # metrics
227
- elapsed_seconds = time.perf_counter() - time_before_query
228
- query_time_with_labels = metrics.INTEGRATION_HANDLER_QUERY_TIME.labels(
229
- get_class_name(self.integration_handler), result.type
230
- )
231
- query_time_with_labels.observe(elapsed_seconds)
260
+ This method processes SQL queries either as ASTNode objects or raw SQL strings
232
261
 
233
- num_rows = 0
234
- if result.data_frame is not None:
235
- num_rows = len(result.data_frame.index)
236
- response_size_with_labels = metrics.INTEGRATION_HANDLER_RESPONSE_SIZE.labels(
237
- get_class_name(self.integration_handler), result.type
238
- )
239
- response_size_with_labels.observe(num_rows)
240
- except Exception as e:
241
- msg = str(e).strip()
242
- if msg == "":
243
- msg = e.__class__.__name__
244
- msg = f"[{self.ds_type}/{self.integration_name}]: {msg}"
245
- raise DBHandlerException(msg) from e
262
+ Args:
263
+ query (ASTNode | str, optional): The query to execute. Can be either:
264
+ - ASTNode: A parsed SQL query object
265
+ - str: Raw SQL query string
266
+ session: Session object (currently unused but kept for compatibility)
267
+
268
+ Returns:
269
+ DataHubResponse: Response object
270
+
271
+ Raises:
272
+ NotImplementedError: If query is not ASTNode or str type
273
+ Exception: If the query execution fails with an error response
274
+ """
275
+ if isinstance(query, ASTNode):
276
+ result: HandlerResponse = self.query_integration_handler(query=query)
277
+ elif isinstance(query, str):
278
+ result: HandlerResponse = self.native_query_integration(query=query)
279
+ else:
280
+ raise NotImplementedError("Thew query argument must be ASTNode or string type")
246
281
 
247
282
  if result.type == RESPONSE_TYPE.ERROR:
248
- failed_sql_query = native_query
249
- if query is not None:
250
- failed_sql_query = query.to_string()
251
-
252
- raise Exception(
253
- format_db_error_message(
254
- db_name=self.integration_handler.name,
255
- db_type=self.integration_handler.__class__.name,
256
- db_error_msg=result.error_message,
257
- failed_query=failed_sql_query,
258
- )
283
+ if isinstance(query, ASTNode):
284
+ try:
285
+ query_str = query.to_string()
286
+ except Exception:
287
+ # most likely it is CreateTable with exotic column types
288
+ query_str = "can't be dump"
289
+ else:
290
+ query_str = query
291
+
292
+ exception = QueryError(
293
+ db_name=self.integration_handler.name,
294
+ db_type=self.integration_handler.__class__.name,
295
+ db_error_msg=result.error_message,
296
+ failed_query=query_str,
297
+ is_acceptable=result.is_acceptable_error,
259
298
  )
260
299
 
300
+ if result.exception is None:
301
+ raise exception
302
+ else:
303
+ raise exception from result.exception
304
+
261
305
  if result.type == RESPONSE_TYPE.OK:
262
306
  return DataHubResponse(affected_rows=result.affected_rows)
263
307
 
@@ -271,8 +315,8 @@ class IntegrationDataNode(DataNode):
271
315
  # replace python's Nan, np.NaN, np.nan and pd.NA to None
272
316
  # TODO keep all NAN to the end of processing, bacause replacing also changes dtypes
273
317
  df.replace([np.NaN, pd.NA, pd.NaT], None, inplace=True)
274
- except Exception as e:
275
- logger.error(f"Issue with clearing DF from NaN values: {e}")
318
+ except Exception:
319
+ logger.exception("Issue with clearing DF from NaN values:")
276
320
  # endregion
277
321
 
278
322
  columns_info = [{"name": k, "type": v} for k, v in df.dtypes.items()]
@@ -280,3 +324,11 @@ class IntegrationDataNode(DataNode):
280
324
  return DataHubResponse(
281
325
  data_frame=df, columns=columns_info, affected_rows=result.affected_rows, mysql_types=result.mysql_types
282
326
  )
327
+
328
+ @collect_metrics
329
+ def query_integration_handler(self, query: ASTNode) -> HandlerResponse:
330
+ return self.integration_handler.query(query)
331
+
332
+ @collect_metrics
333
+ def native_query_integration(self, query: str) -> HandlerResponse:
334
+ return self.integration_handler.native_query(query)
@@ -2,6 +2,7 @@ from copy import deepcopy
2
2
  from dataclasses import astuple
3
3
 
4
4
  import pandas as pd
5
+ from mindsdb_sql_parser.ast.base import ASTNode
5
6
  from mindsdb_sql_parser import parse_sql
6
7
  from mindsdb_sql_parser.ast import (
7
8
  BinaryOperation,
@@ -99,9 +100,9 @@ class ProjectDataNode(DataNode):
99
100
 
100
101
  return ml_handler.predict(model_name, df, project_name=self.project.name, version=version, params=params)
101
102
 
102
- def query(self, query=None, native_query=None, session=None) -> DataHubResponse:
103
- if query is None and native_query is not None:
104
- query = parse_sql(native_query)
103
+ def query(self, query: ASTNode | str = None, session=None) -> DataHubResponse:
104
+ if isinstance(query, str):
105
+ query = parse_sql(query)
105
106
 
106
107
  if isinstance(query, Update):
107
108
  query_table = query.table.parts[0].lower()
@@ -132,7 +133,10 @@ class ProjectDataNode(DataNode):
132
133
  case [query_table, str(version)], [is_quoted, _] if version.isdigit():
133
134
  ...
134
135
  case _:
135
- raise ValueError("Table name should contain only one part")
136
+ raise EntityNotExistsError(
137
+ f"Table '{query.from_table}' not found in the database. The project database support only single-part names",
138
+ self.project.name,
139
+ )
136
140
 
137
141
  if not is_quoted:
138
142
  query_table = query_table.lower()
@@ -131,7 +131,7 @@ class TablesTable(Table):
131
131
  row.TABLE_SCHEMA = ds_name
132
132
  data.append(row.to_list())
133
133
  except Exception:
134
- logger.error(f"Can't get tables from '{ds_name}'")
134
+ logger.exception(f"Can't get tables from '{ds_name}'")
135
135
 
136
136
  for project_name in inf_schema.get_projects_names():
137
137
  if databases is not None and project_name not in databases: