MindsDB 25.9.1.2__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +39 -20
- mindsdb/api/a2a/agent.py +7 -9
- mindsdb/api/a2a/common/server/server.py +3 -3
- mindsdb/api/a2a/common/server/task_manager.py +4 -4
- mindsdb/api/a2a/task_manager.py +15 -17
- mindsdb/api/common/middleware.py +9 -11
- mindsdb/api/executor/command_executor.py +2 -4
- mindsdb/api/executor/datahub/datanodes/datanode.py +2 -2
- mindsdb/api/executor/datahub/datanodes/integration_datanode.py +100 -48
- mindsdb/api/executor/datahub/datanodes/project_datanode.py +8 -4
- mindsdb/api/executor/datahub/datanodes/system_tables.py +1 -1
- mindsdb/api/executor/exceptions.py +29 -10
- mindsdb/api/executor/planner/plan_join.py +17 -3
- mindsdb/api/executor/sql_query/sql_query.py +74 -74
- mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +1 -2
- mindsdb/api/executor/sql_query/steps/subselect_step.py +0 -1
- mindsdb/api/executor/utilities/functions.py +6 -6
- mindsdb/api/executor/utilities/sql.py +32 -16
- mindsdb/api/http/gui.py +5 -11
- mindsdb/api/http/initialize.py +8 -10
- mindsdb/api/http/namespaces/agents.py +10 -12
- mindsdb/api/http/namespaces/analysis.py +13 -20
- mindsdb/api/http/namespaces/auth.py +1 -1
- mindsdb/api/http/namespaces/config.py +15 -11
- mindsdb/api/http/namespaces/databases.py +140 -201
- mindsdb/api/http/namespaces/file.py +15 -4
- mindsdb/api/http/namespaces/handlers.py +7 -2
- mindsdb/api/http/namespaces/knowledge_bases.py +8 -7
- mindsdb/api/http/namespaces/models.py +94 -126
- mindsdb/api/http/namespaces/projects.py +13 -22
- mindsdb/api/http/namespaces/sql.py +33 -25
- mindsdb/api/http/namespaces/tab.py +27 -37
- mindsdb/api/http/namespaces/views.py +1 -1
- mindsdb/api/http/start.py +14 -8
- mindsdb/api/mcp/__init__.py +2 -1
- mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +15 -20
- mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +26 -50
- mindsdb/api/mysql/mysql_proxy/utilities/__init__.py +0 -1
- mindsdb/api/postgres/postgres_proxy/executor/executor.py +6 -13
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +40 -28
- mindsdb/integrations/handlers/byom_handler/byom_handler.py +168 -185
- mindsdb/integrations/handlers/chromadb_handler/chromadb_handler.py +11 -5
- mindsdb/integrations/handlers/file_handler/file_handler.py +7 -0
- mindsdb/integrations/handlers/lightwood_handler/functions.py +45 -79
- mindsdb/integrations/handlers/openai_handler/openai_handler.py +1 -1
- mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py +20 -2
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +18 -3
- mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +25 -12
- mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +2 -1
- mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +1 -0
- mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +4 -4
- mindsdb/integrations/libs/api_handler.py +10 -10
- mindsdb/integrations/libs/base.py +4 -4
- mindsdb/integrations/libs/llm/utils.py +2 -2
- mindsdb/integrations/libs/ml_handler_process/create_engine_process.py +4 -7
- mindsdb/integrations/libs/ml_handler_process/func_call_process.py +2 -7
- mindsdb/integrations/libs/ml_handler_process/learn_process.py +37 -47
- mindsdb/integrations/libs/ml_handler_process/update_engine_process.py +4 -7
- mindsdb/integrations/libs/ml_handler_process/update_process.py +2 -7
- mindsdb/integrations/libs/process_cache.py +132 -140
- mindsdb/integrations/libs/response.py +18 -12
- mindsdb/integrations/libs/vectordatabase_handler.py +26 -0
- mindsdb/integrations/utilities/files/file_reader.py +6 -7
- mindsdb/integrations/utilities/rag/config_loader.py +37 -26
- mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +59 -9
- mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py +4 -4
- mindsdb/integrations/utilities/rag/retrievers/sql_retriever.py +55 -133
- mindsdb/integrations/utilities/rag/settings.py +58 -133
- mindsdb/integrations/utilities/rag/splitters/file_splitter.py +5 -15
- mindsdb/interfaces/agents/agents_controller.py +2 -1
- mindsdb/interfaces/agents/constants.py +0 -2
- mindsdb/interfaces/agents/litellm_server.py +34 -58
- mindsdb/interfaces/agents/mcp_client_agent.py +10 -10
- mindsdb/interfaces/agents/mindsdb_database_agent.py +5 -5
- mindsdb/interfaces/agents/run_mcp_agent.py +12 -21
- mindsdb/interfaces/chatbot/chatbot_task.py +20 -23
- mindsdb/interfaces/chatbot/polling.py +30 -18
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +10 -10
- mindsdb/interfaces/database/integrations.py +19 -2
- mindsdb/interfaces/file/file_controller.py +6 -6
- mindsdb/interfaces/functions/controller.py +1 -1
- mindsdb/interfaces/functions/to_markdown.py +2 -2
- mindsdb/interfaces/jobs/jobs_controller.py +5 -5
- mindsdb/interfaces/jobs/scheduler.py +3 -8
- mindsdb/interfaces/knowledge_base/controller.py +54 -25
- mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +40 -61
- mindsdb/interfaces/model/model_controller.py +170 -166
- mindsdb/interfaces/query_context/context_controller.py +14 -2
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +6 -4
- mindsdb/interfaces/skills/retrieval_tool.py +43 -50
- mindsdb/interfaces/skills/skill_tool.py +2 -2
- mindsdb/interfaces/skills/sql_agent.py +25 -19
- mindsdb/interfaces/storage/fs.py +114 -169
- mindsdb/interfaces/storage/json.py +19 -18
- mindsdb/interfaces/storage/model_fs.py +54 -92
- mindsdb/interfaces/tabs/tabs_controller.py +49 -72
- mindsdb/interfaces/tasks/task_monitor.py +3 -9
- mindsdb/interfaces/tasks/task_thread.py +7 -9
- mindsdb/interfaces/triggers/trigger_task.py +7 -13
- mindsdb/interfaces/triggers/triggers_controller.py +47 -50
- mindsdb/migrations/migrate.py +16 -16
- mindsdb/utilities/api_status.py +58 -0
- mindsdb/utilities/config.py +49 -0
- mindsdb/utilities/exception.py +40 -1
- mindsdb/utilities/fs.py +0 -1
- mindsdb/utilities/hooks/profiling.py +17 -14
- mindsdb/utilities/langfuse.py +40 -45
- mindsdb/utilities/log.py +272 -0
- mindsdb/utilities/ml_task_queue/consumer.py +52 -58
- mindsdb/utilities/ml_task_queue/producer.py +26 -30
- mindsdb/utilities/render/sqlalchemy_render.py +8 -7
- mindsdb/utilities/utils.py +2 -2
- {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/METADATA +266 -261
- {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/RECORD +119 -119
- mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -14
- {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/WHEEL +0 -0
- {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/top_level.txt +0 -0
mindsdb/__about__.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
__title__ = "MindsDB"
|
|
2
2
|
__package_name__ = "mindsdb"
|
|
3
|
-
__version__ = "25.9.
|
|
3
|
+
__version__ = "25.9.3rc1"
|
|
4
4
|
__description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
|
|
5
5
|
__email__ = "jorge@mindsdb.com"
|
|
6
6
|
__author__ = "MindsDB Inc"
|
mindsdb/__main__.py
CHANGED
|
@@ -8,8 +8,8 @@ import atexit
|
|
|
8
8
|
import signal
|
|
9
9
|
import psutil
|
|
10
10
|
import asyncio
|
|
11
|
-
import traceback
|
|
12
11
|
import threading
|
|
12
|
+
import shutil
|
|
13
13
|
from enum import Enum
|
|
14
14
|
from dataclasses import dataclass, field
|
|
15
15
|
from typing import Callable, Optional, Tuple, List
|
|
@@ -39,6 +39,7 @@ from mindsdb.utilities.fs import clean_process_marks, clean_unlinked_process_mar
|
|
|
39
39
|
from mindsdb.utilities.context import context as ctx
|
|
40
40
|
from mindsdb.utilities.auth import register_oauth_client, get_aws_meta_data
|
|
41
41
|
from mindsdb.utilities.sentry import sentry_sdk # noqa: F401
|
|
42
|
+
from mindsdb.utilities.api_status import set_api_status
|
|
42
43
|
|
|
43
44
|
try:
|
|
44
45
|
import torch.multiprocessing as mp
|
|
@@ -152,6 +153,16 @@ def close_api_gracefully(trunc_processes_struct):
|
|
|
152
153
|
sys.exit(0)
|
|
153
154
|
|
|
154
155
|
|
|
156
|
+
def clean_mindsdb_tmp_dir():
|
|
157
|
+
"""Clean the MindsDB tmp dir at exit."""
|
|
158
|
+
temp_dir = config["paths"]["tmp"]
|
|
159
|
+
for file in temp_dir.iterdir():
|
|
160
|
+
if file.is_dir():
|
|
161
|
+
shutil.rmtree(file)
|
|
162
|
+
else:
|
|
163
|
+
file.unlink()
|
|
164
|
+
|
|
165
|
+
|
|
155
166
|
def set_error_model_status_by_pids(unexisting_pids: List[int]):
|
|
156
167
|
"""Models have id of its traiing process in the 'training_metadata' field.
|
|
157
168
|
If the pid does not exist, we should set the model status to "error".
|
|
@@ -217,19 +228,20 @@ def create_permanent_integrations():
|
|
|
217
228
|
"""
|
|
218
229
|
integration_name = "files"
|
|
219
230
|
existing = db.session.query(db.Integration).filter_by(name=integration_name, company_id=None).first()
|
|
220
|
-
if existing is None:
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
231
|
+
if existing is not None:
|
|
232
|
+
return
|
|
233
|
+
integration_record = db.Integration(
|
|
234
|
+
name=integration_name,
|
|
235
|
+
data={},
|
|
236
|
+
engine=integration_name,
|
|
237
|
+
company_id=None,
|
|
238
|
+
)
|
|
239
|
+
db.session.add(integration_record)
|
|
240
|
+
try:
|
|
241
|
+
db.session.commit()
|
|
242
|
+
except Exception:
|
|
243
|
+
logger.exception(f"Failed to create permanent integration '{integration_name}' in the internal database.")
|
|
244
|
+
db.session.rollback()
|
|
233
245
|
|
|
234
246
|
|
|
235
247
|
def validate_default_project() -> None:
|
|
@@ -289,7 +301,7 @@ def start_process(trunc_process_data: TrunkProcessData) -> None:
|
|
|
289
301
|
)
|
|
290
302
|
trunc_process_data.process.start()
|
|
291
303
|
except Exception as e:
|
|
292
|
-
logger.
|
|
304
|
+
logger.exception(f"Failed to start '{trunc_process_data.name}' API process due to unexpected error:")
|
|
293
305
|
close_api_gracefully(trunc_processes_struct)
|
|
294
306
|
raise e
|
|
295
307
|
|
|
@@ -363,8 +375,8 @@ if __name__ == "__main__":
|
|
|
363
375
|
if environment == "aws_marketplace":
|
|
364
376
|
try:
|
|
365
377
|
register_oauth_client()
|
|
366
|
-
except Exception
|
|
367
|
-
logger.
|
|
378
|
+
except Exception:
|
|
379
|
+
logger.exception("Something went wrong during client register:")
|
|
368
380
|
elif environment != "local":
|
|
369
381
|
try:
|
|
370
382
|
aws_meta_data = get_aws_meta_data()
|
|
@@ -384,6 +396,7 @@ if __name__ == "__main__":
|
|
|
384
396
|
logger.info(f"Version: {mindsdb_version}")
|
|
385
397
|
logger.info(f"Configuration file: {config.config_path or 'absent'}")
|
|
386
398
|
logger.info(f"Storage path: {config.paths['root']}")
|
|
399
|
+
log.log_system_info(logger)
|
|
387
400
|
logger.debug(f"User config: {config.user_config}")
|
|
388
401
|
logger.debug(f"System config: {config.auto_config}")
|
|
389
402
|
logger.debug(f"Env config: {config.env_config}")
|
|
@@ -391,13 +404,12 @@ if __name__ == "__main__":
|
|
|
391
404
|
is_cloud = config.is_cloud
|
|
392
405
|
unexisting_pids = clean_unlinked_process_marks()
|
|
393
406
|
if not is_cloud:
|
|
394
|
-
logger.debug("Applying database migrations")
|
|
395
407
|
try:
|
|
396
408
|
from mindsdb.migrations import migrate
|
|
397
409
|
|
|
398
410
|
migrate.migrate_to_head()
|
|
399
|
-
except Exception
|
|
400
|
-
logger.
|
|
411
|
+
except Exception:
|
|
412
|
+
logger.exception("Failed to apply database migrations. This may prevent MindsDB from operating correctly:")
|
|
401
413
|
|
|
402
414
|
validate_default_project()
|
|
403
415
|
|
|
@@ -484,8 +496,12 @@ if __name__ == "__main__":
|
|
|
484
496
|
if trunc_process_data.started is True or trunc_process_data.need_to_run is False:
|
|
485
497
|
continue
|
|
486
498
|
start_process(trunc_process_data)
|
|
499
|
+
# Set status for APIs without ports (they don't go through wait_api_start)
|
|
500
|
+
if trunc_process_data.port is None:
|
|
501
|
+
set_api_status(trunc_process_data.name, True)
|
|
487
502
|
|
|
488
503
|
atexit.register(close_api_gracefully, trunc_processes_struct=trunc_processes_struct)
|
|
504
|
+
atexit.register(clean_mindsdb_tmp_dir)
|
|
489
505
|
|
|
490
506
|
async def wait_api_start(api_name, pid, port):
|
|
491
507
|
timeout = 60
|
|
@@ -494,6 +510,9 @@ if __name__ == "__main__":
|
|
|
494
510
|
while (time.time() - start_time) < timeout and started is False:
|
|
495
511
|
await asyncio.sleep(0.5)
|
|
496
512
|
started = is_pid_listen_port(pid, port)
|
|
513
|
+
|
|
514
|
+
set_api_status(api_name, started)
|
|
515
|
+
|
|
497
516
|
return api_name, port, started
|
|
498
517
|
|
|
499
518
|
async def wait_apis_start():
|
mindsdb/api/a2a/agent.py
CHANGED
|
@@ -73,17 +73,15 @@ class MindsDBAgent:
|
|
|
73
73
|
"parts": [{"type": "text", "text": error_msg}],
|
|
74
74
|
}
|
|
75
75
|
except requests.exceptions.RequestException as e:
|
|
76
|
-
|
|
77
|
-
logger.error(error_msg)
|
|
76
|
+
logger.exception("Error connecting to MindsDB:")
|
|
78
77
|
return {
|
|
79
|
-
"content":
|
|
78
|
+
"content": f"Error connecting to MindsDB: {e}",
|
|
80
79
|
"parts": [{"type": "text", "text": error_msg}],
|
|
81
80
|
}
|
|
82
81
|
except Exception as e:
|
|
83
|
-
|
|
84
|
-
logger.error(error_msg)
|
|
82
|
+
logger.exception("Error: ")
|
|
85
83
|
return {
|
|
86
|
-
"content":
|
|
84
|
+
"content": f"Error: {e}",
|
|
87
85
|
"parts": [{"type": "text", "text": error_msg}],
|
|
88
86
|
}
|
|
89
87
|
|
|
@@ -102,7 +100,7 @@ class MindsDBAgent:
|
|
|
102
100
|
try:
|
|
103
101
|
yield json.loads(payload)
|
|
104
102
|
except Exception as e:
|
|
105
|
-
logger.
|
|
103
|
+
logger.exception(f"Failed to parse SSE JSON payload: {e}; line: {payload}")
|
|
106
104
|
# Ignore comments or control lines
|
|
107
105
|
# Signal the end of the stream
|
|
108
106
|
yield {"is_task_complete": True}
|
|
@@ -129,13 +127,13 @@ class MindsDBAgent:
|
|
|
129
127
|
wrapped_chunk = {"is_task_complete": False, "content": content_value, "metadata": {}}
|
|
130
128
|
yield wrapped_chunk
|
|
131
129
|
except Exception as e:
|
|
132
|
-
logger.
|
|
130
|
+
logger.exception(f"Error in streaming: {e}")
|
|
133
131
|
yield {
|
|
134
132
|
"is_task_complete": True,
|
|
135
133
|
"parts": [
|
|
136
134
|
{
|
|
137
135
|
"type": "text",
|
|
138
|
-
"text": f"Error: {
|
|
136
|
+
"text": f"Error: {e}",
|
|
139
137
|
}
|
|
140
138
|
],
|
|
141
139
|
"metadata": {
|
|
@@ -118,7 +118,7 @@ class A2AServer:
|
|
|
118
118
|
elif isinstance(e, ValidationError):
|
|
119
119
|
json_rpc_error = InvalidRequestError(data=json.loads(e.json()))
|
|
120
120
|
else:
|
|
121
|
-
logger.
|
|
121
|
+
logger.exception("Unhandled exception:")
|
|
122
122
|
json_rpc_error = InternalError()
|
|
123
123
|
|
|
124
124
|
response = JSONRPCResponse(id=None, error=json_rpc_error)
|
|
@@ -137,8 +137,8 @@ class A2AServer:
|
|
|
137
137
|
else:
|
|
138
138
|
data = json.dumps(item)
|
|
139
139
|
except Exception as e:
|
|
140
|
-
logger.
|
|
141
|
-
data = json.dumps({"error": f"Serialization error: {
|
|
140
|
+
logger.exception("Serialization error in SSE stream:")
|
|
141
|
+
data = json.dumps({"error": f"Serialization error: {e}"})
|
|
142
142
|
yield {"data": data}
|
|
143
143
|
|
|
144
144
|
# Add robust SSE headers for compatibility
|
|
@@ -152,8 +152,8 @@ class InMemoryTaskManager(TaskManager):
|
|
|
152
152
|
task_notification_params.id,
|
|
153
153
|
task_notification_params.pushNotificationConfig,
|
|
154
154
|
)
|
|
155
|
-
except Exception
|
|
156
|
-
logger.
|
|
155
|
+
except Exception:
|
|
156
|
+
logger.exception("Error while setting push notification info:")
|
|
157
157
|
return JSONRPCResponse(
|
|
158
158
|
id=request.id,
|
|
159
159
|
error=InternalError(message="An error occurred while setting push notification info"),
|
|
@@ -169,8 +169,8 @@ class InMemoryTaskManager(TaskManager):
|
|
|
169
169
|
|
|
170
170
|
try:
|
|
171
171
|
notification_info = await self.get_push_notification_info(task_params.id)
|
|
172
|
-
except Exception
|
|
173
|
-
logger.
|
|
172
|
+
except Exception:
|
|
173
|
+
logger.exception("Error while getting push notification info:")
|
|
174
174
|
return GetTaskPushNotificationResponse(
|
|
175
175
|
id=request.id,
|
|
176
176
|
error=InternalError(message="An error occurred while getting push notification info"),
|
mindsdb/api/a2a/task_manager.py
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
import time
|
|
2
|
+
import logging
|
|
3
|
+
import asyncio
|
|
4
|
+
from typing import AsyncIterable, Dict, Union
|
|
5
|
+
|
|
2
6
|
from mindsdb.api.a2a.common.types import (
|
|
3
7
|
SendTaskRequest,
|
|
4
8
|
TaskSendParams,
|
|
@@ -20,11 +24,6 @@ from mindsdb.api.a2a.common.server.task_manager import InMemoryTaskManager
|
|
|
20
24
|
from mindsdb.api.a2a.agent import MindsDBAgent
|
|
21
25
|
from mindsdb.api.a2a.utils import to_serializable, convert_a2a_message_to_qa_format
|
|
22
26
|
|
|
23
|
-
from typing import Union
|
|
24
|
-
import logging
|
|
25
|
-
import asyncio
|
|
26
|
-
import time
|
|
27
|
-
import traceback
|
|
28
27
|
|
|
29
28
|
logger = logging.getLogger(__name__)
|
|
30
29
|
|
|
@@ -80,11 +79,11 @@ class AgentTaskManager(InMemoryTaskManager):
|
|
|
80
79
|
task = await self.upsert_task(task_send_params)
|
|
81
80
|
logger.info(f"Task created/updated with history length: {len(task.history) if task.history else 0}")
|
|
82
81
|
except Exception as e:
|
|
83
|
-
logger.
|
|
82
|
+
logger.exception("Error creating task:")
|
|
84
83
|
error_result = to_serializable(
|
|
85
84
|
{
|
|
86
85
|
"id": request.id,
|
|
87
|
-
"error": to_serializable(InternalError(message=f"Error creating task: {
|
|
86
|
+
"error": to_serializable(InternalError(message=f"Error creating task: {e}")),
|
|
88
87
|
}
|
|
89
88
|
)
|
|
90
89
|
yield error_result
|
|
@@ -149,14 +148,14 @@ class AgentTaskManager(InMemoryTaskManager):
|
|
|
149
148
|
return
|
|
150
149
|
|
|
151
150
|
except Exception as e:
|
|
152
|
-
logger.
|
|
151
|
+
logger.exception("Error invoking agent:")
|
|
153
152
|
error_result = to_serializable(
|
|
154
153
|
{
|
|
155
154
|
"id": request.id,
|
|
156
155
|
"error": to_serializable(
|
|
157
156
|
JSONRPCResponse(
|
|
158
157
|
id=request.id,
|
|
159
|
-
error=to_serializable(InternalError(message=f"Error invoking agent: {
|
|
158
|
+
error=to_serializable(InternalError(message=f"Error invoking agent: {e}")),
|
|
160
159
|
)
|
|
161
160
|
),
|
|
162
161
|
}
|
|
@@ -182,11 +181,10 @@ class AgentTaskManager(InMemoryTaskManager):
|
|
|
182
181
|
item["artifact"]["parts"] = [to_serializable(p) for p in item["artifact"]["parts"]]
|
|
183
182
|
yield to_serializable(item)
|
|
184
183
|
except Exception as e:
|
|
185
|
-
|
|
186
|
-
logger.
|
|
187
|
-
error_text = f"An error occurred while streaming the response: {str(e)}"
|
|
184
|
+
error_text = "An error occurred while streaming the response:"
|
|
185
|
+
logger.exception(error_text)
|
|
188
186
|
# Ensure all parts are plain dicts
|
|
189
|
-
parts = [{"type": "text", "text": error_text}]
|
|
187
|
+
parts = [{"type": "text", "text": f"{error_text} {e}"}]
|
|
190
188
|
parts = [to_serializable(part) for part in parts]
|
|
191
189
|
artifact = {
|
|
192
190
|
"parts": parts,
|
|
@@ -333,11 +331,11 @@ class AgentTaskManager(InMemoryTaskManager):
|
|
|
333
331
|
yield response
|
|
334
332
|
except Exception as e:
|
|
335
333
|
# If an error occurs, yield an error response
|
|
336
|
-
logger.
|
|
334
|
+
logger.exception(f"Error in on_send_task_subscribe: {e}")
|
|
337
335
|
error_result = to_serializable(
|
|
338
336
|
{
|
|
339
337
|
"id": request.id,
|
|
340
|
-
"error": to_serializable(InternalError(message=f"Error processing streaming request: {
|
|
338
|
+
"error": to_serializable(InternalError(message=f"Error processing streaming request: {e}")),
|
|
341
339
|
}
|
|
342
340
|
)
|
|
343
341
|
yield error_result
|
|
@@ -463,7 +461,7 @@ class AgentTaskManager(InMemoryTaskManager):
|
|
|
463
461
|
)
|
|
464
462
|
return to_serializable(SendTaskResponse(id=request.id, result=task))
|
|
465
463
|
except Exception as e:
|
|
466
|
-
logger.
|
|
464
|
+
logger.exception("Error invoking agent:")
|
|
467
465
|
result_text = f"Error invoking agent: {e}"
|
|
468
466
|
parts = [{"type": "text", "text": result_text}]
|
|
469
467
|
|
mindsdb/api/common/middleware.py
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import hmac
|
|
3
|
+
import secrets
|
|
4
|
+
import hashlib
|
|
5
|
+
from http import HTTPStatus
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
1
8
|
from starlette.middleware.base import BaseHTTPMiddleware
|
|
2
9
|
from starlette.responses import JSONResponse
|
|
3
10
|
from starlette.requests import Request
|
|
4
|
-
from http import HTTPStatus
|
|
5
|
-
from typing import Optional
|
|
6
|
-
import secrets
|
|
7
|
-
import hmac
|
|
8
|
-
import hashlib
|
|
9
|
-
import os
|
|
10
|
-
import traceback
|
|
11
11
|
|
|
12
12
|
from mindsdb.utilities import log
|
|
13
13
|
from mindsdb.utilities.config import config
|
|
@@ -100,7 +100,5 @@ def check_auth(username, password, scramble_func, salt, company_id, config):
|
|
|
100
100
|
|
|
101
101
|
logger.info(f"Check auth, user={username}: Ok")
|
|
102
102
|
return {"success": True, "username": username}
|
|
103
|
-
except Exception
|
|
104
|
-
logger.
|
|
105
|
-
logger.error(e)
|
|
106
|
-
logger.error(traceback.format_exc())
|
|
103
|
+
except Exception:
|
|
104
|
+
logger.exception(f"Check auth, user={username}: ERROR")
|
|
@@ -757,8 +757,6 @@ class ExecuteCommands:
|
|
|
757
757
|
except EntityNotExistsError:
|
|
758
758
|
if statement.if_exists is False:
|
|
759
759
|
raise
|
|
760
|
-
except Exception as e:
|
|
761
|
-
raise e
|
|
762
760
|
|
|
763
761
|
return ExecuteAnswer()
|
|
764
762
|
|
|
@@ -844,7 +842,7 @@ class ExecuteCommands:
|
|
|
844
842
|
try:
|
|
845
843
|
sqlquery = SQLQuery(statement.data, session=self.session, database=database_name)
|
|
846
844
|
except Exception as e:
|
|
847
|
-
raise Exception(f'Nested query failed to execute with error: "{e}", please check and try again.')
|
|
845
|
+
raise Exception(f'Nested query failed to execute with error: "{e}", please check and try again.') from e
|
|
848
846
|
df = sqlquery.fetched_data.to_df()
|
|
849
847
|
df.columns = [str(t.alias) if hasattr(t, "alias") else str(t.parts[-1]) for t in statement.data.targets]
|
|
850
848
|
|
|
@@ -1210,7 +1208,7 @@ class ExecuteCommands:
|
|
|
1210
1208
|
ast_drop = DropMLEngine(name=Identifier(name))
|
|
1211
1209
|
self.answer_drop_ml_engine(ast_drop)
|
|
1212
1210
|
logger.info(msg)
|
|
1213
|
-
raise ExecutorException(msg)
|
|
1211
|
+
raise ExecutorException(msg) from e
|
|
1214
1212
|
|
|
1215
1213
|
return ExecuteAnswer()
|
|
1216
1214
|
|
|
@@ -4,7 +4,7 @@ from mindsdb.api.executor.datahub.classes.response import DataHubResponse
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class DataNode:
|
|
7
|
-
type =
|
|
7
|
+
type = "meta"
|
|
8
8
|
|
|
9
9
|
def __init__(self):
|
|
10
10
|
pass
|
|
@@ -21,5 +21,5 @@ class DataNode:
|
|
|
21
21
|
def get_table_columns_names(self, table_name: str, schema_name: str | None = None) -> list[str]:
|
|
22
22
|
pass
|
|
23
23
|
|
|
24
|
-
def query(self, query=None,
|
|
24
|
+
def query(self, query=None, session=None) -> DataHubResponse:
|
|
25
25
|
pass
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import time
|
|
2
2
|
import inspect
|
|
3
|
+
import functools
|
|
3
4
|
from dataclasses import astuple
|
|
4
5
|
from typing import Iterable, List
|
|
5
6
|
|
|
@@ -20,7 +21,7 @@ from mindsdb.integrations.utilities.utils import get_class_name
|
|
|
20
21
|
from mindsdb.metrics import metrics
|
|
21
22
|
from mindsdb.utilities import log
|
|
22
23
|
from mindsdb.utilities.profiler import profiler
|
|
23
|
-
from mindsdb.utilities.exception import
|
|
24
|
+
from mindsdb.utilities.exception import QueryError
|
|
24
25
|
from mindsdb.api.executor.datahub.datanodes.system_tables import infer_mysql_type
|
|
25
26
|
|
|
26
27
|
logger = log.getLogger(__name__)
|
|
@@ -30,6 +31,51 @@ class DBHandlerException(Exception):
|
|
|
30
31
|
pass
|
|
31
32
|
|
|
32
33
|
|
|
34
|
+
def collect_metrics(func):
|
|
35
|
+
"""Decorator for collecting performance metrics if integration handler query.
|
|
36
|
+
|
|
37
|
+
The decorator measures:
|
|
38
|
+
- Query execution time using high-precision performance counter
|
|
39
|
+
- Response size (number of rows returned)
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
func: The function to be decorated (integration handler method)
|
|
43
|
+
|
|
44
|
+
Returns:
|
|
45
|
+
function: Wrapped function that includes metrics collection and error handling
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
@functools.wraps(func)
|
|
49
|
+
def wrapper(self, *args, **kwargs):
|
|
50
|
+
try:
|
|
51
|
+
time_before_query = time.perf_counter()
|
|
52
|
+
result = func(self, *args, **kwargs)
|
|
53
|
+
|
|
54
|
+
# metrics
|
|
55
|
+
handler_class_name = get_class_name(self.integration_handler)
|
|
56
|
+
elapsed_seconds = time.perf_counter() - time_before_query
|
|
57
|
+
query_time_with_labels = metrics.INTEGRATION_HANDLER_QUERY_TIME.labels(handler_class_name, result.type)
|
|
58
|
+
query_time_with_labels.observe(elapsed_seconds)
|
|
59
|
+
|
|
60
|
+
num_rows = 0
|
|
61
|
+
if result.data_frame is not None:
|
|
62
|
+
num_rows = len(result.data_frame.index)
|
|
63
|
+
response_size_with_labels = metrics.INTEGRATION_HANDLER_RESPONSE_SIZE.labels(
|
|
64
|
+
handler_class_name, result.type
|
|
65
|
+
)
|
|
66
|
+
response_size_with_labels.observe(num_rows)
|
|
67
|
+
logger.debug(f"Handler '{handler_class_name}' returned {num_rows} rows in {elapsed_seconds:.3f} seconds")
|
|
68
|
+
except Exception as e:
|
|
69
|
+
msg = str(e).strip()
|
|
70
|
+
if msg == "":
|
|
71
|
+
msg = e.__class__.__name__
|
|
72
|
+
msg = f"[{self.ds_type}/{self.integration_name}]: {msg}"
|
|
73
|
+
raise DBHandlerException(msg) from e
|
|
74
|
+
return result
|
|
75
|
+
|
|
76
|
+
return wrapper
|
|
77
|
+
|
|
78
|
+
|
|
33
79
|
class IntegrationDataNode(DataNode):
|
|
34
80
|
type = "integration"
|
|
35
81
|
|
|
@@ -46,16 +92,10 @@ class IntegrationDataNode(DataNode):
|
|
|
46
92
|
response = self.integration_handler.get_tables()
|
|
47
93
|
if response.type == RESPONSE_TYPE.TABLE:
|
|
48
94
|
result_dict = response.data_frame.to_dict(orient="records")
|
|
49
|
-
|
|
50
|
-
for row in result_dict:
|
|
51
|
-
result.append(TablesRow.from_dict(row))
|
|
52
|
-
return result
|
|
95
|
+
return [TablesRow.from_dict(row) for row in result_dict]
|
|
53
96
|
else:
|
|
54
97
|
raise Exception(f"Can't get tables: {response.error_message}")
|
|
55
98
|
|
|
56
|
-
result_dict = response.data_frame.to_dict(orient="records")
|
|
57
|
-
return [TablesRow.from_dict(row) for row in result_dict]
|
|
58
|
-
|
|
59
99
|
def get_table_columns_df(self, table_name: str, schema_name: str | None = None) -> pd.DataFrame:
|
|
60
100
|
"""Get a DataFrame containing representation of information_schema.columns for the specified table.
|
|
61
101
|
|
|
@@ -214,50 +254,54 @@ class IntegrationDataNode(DataNode):
|
|
|
214
254
|
return self.integration_handler.query_stream(query, fetch_size=fetch_size)
|
|
215
255
|
|
|
216
256
|
@profiler.profile()
|
|
217
|
-
def query(self, query: ASTNode |
|
|
218
|
-
|
|
219
|
-
time_before_query = time.perf_counter()
|
|
220
|
-
if query is not None:
|
|
221
|
-
result: HandlerResponse = self.integration_handler.query(query)
|
|
222
|
-
else:
|
|
223
|
-
# try to fetch native query
|
|
224
|
-
result: HandlerResponse = self.integration_handler.native_query(native_query)
|
|
257
|
+
def query(self, query: ASTNode | str = None, session=None) -> DataHubResponse:
|
|
258
|
+
"""Execute a query against the integration data source.
|
|
225
259
|
|
|
226
|
-
|
|
227
|
-
elapsed_seconds = time.perf_counter() - time_before_query
|
|
228
|
-
query_time_with_labels = metrics.INTEGRATION_HANDLER_QUERY_TIME.labels(
|
|
229
|
-
get_class_name(self.integration_handler), result.type
|
|
230
|
-
)
|
|
231
|
-
query_time_with_labels.observe(elapsed_seconds)
|
|
260
|
+
This method processes SQL queries either as ASTNode objects or raw SQL strings
|
|
232
261
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
262
|
+
Args:
|
|
263
|
+
query (ASTNode | str, optional): The query to execute. Can be either:
|
|
264
|
+
- ASTNode: A parsed SQL query object
|
|
265
|
+
- str: Raw SQL query string
|
|
266
|
+
session: Session object (currently unused but kept for compatibility)
|
|
267
|
+
|
|
268
|
+
Returns:
|
|
269
|
+
DataHubResponse: Response object
|
|
270
|
+
|
|
271
|
+
Raises:
|
|
272
|
+
NotImplementedError: If query is not ASTNode or str type
|
|
273
|
+
Exception: If the query execution fails with an error response
|
|
274
|
+
"""
|
|
275
|
+
if isinstance(query, ASTNode):
|
|
276
|
+
result: HandlerResponse = self.query_integration_handler(query=query)
|
|
277
|
+
elif isinstance(query, str):
|
|
278
|
+
result: HandlerResponse = self.native_query_integration(query=query)
|
|
279
|
+
else:
|
|
280
|
+
raise NotImplementedError("Thew query argument must be ASTNode or string type")
|
|
246
281
|
|
|
247
282
|
if result.type == RESPONSE_TYPE.ERROR:
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
283
|
+
if isinstance(query, ASTNode):
|
|
284
|
+
try:
|
|
285
|
+
query_str = query.to_string()
|
|
286
|
+
except Exception:
|
|
287
|
+
# most likely it is CreateTable with exotic column types
|
|
288
|
+
query_str = "can't be dump"
|
|
289
|
+
else:
|
|
290
|
+
query_str = query
|
|
291
|
+
|
|
292
|
+
exception = QueryError(
|
|
293
|
+
db_name=self.integration_handler.name,
|
|
294
|
+
db_type=self.integration_handler.__class__.name,
|
|
295
|
+
db_error_msg=result.error_message,
|
|
296
|
+
failed_query=query_str,
|
|
297
|
+
is_acceptable=result.is_acceptable_error,
|
|
259
298
|
)
|
|
260
299
|
|
|
300
|
+
if result.exception is None:
|
|
301
|
+
raise exception
|
|
302
|
+
else:
|
|
303
|
+
raise exception from result.exception
|
|
304
|
+
|
|
261
305
|
if result.type == RESPONSE_TYPE.OK:
|
|
262
306
|
return DataHubResponse(affected_rows=result.affected_rows)
|
|
263
307
|
|
|
@@ -271,8 +315,8 @@ class IntegrationDataNode(DataNode):
|
|
|
271
315
|
# replace python's Nan, np.NaN, np.nan and pd.NA to None
|
|
272
316
|
# TODO keep all NAN to the end of processing, bacause replacing also changes dtypes
|
|
273
317
|
df.replace([np.NaN, pd.NA, pd.NaT], None, inplace=True)
|
|
274
|
-
except Exception
|
|
275
|
-
logger.
|
|
318
|
+
except Exception:
|
|
319
|
+
logger.exception("Issue with clearing DF from NaN values:")
|
|
276
320
|
# endregion
|
|
277
321
|
|
|
278
322
|
columns_info = [{"name": k, "type": v} for k, v in df.dtypes.items()]
|
|
@@ -280,3 +324,11 @@ class IntegrationDataNode(DataNode):
|
|
|
280
324
|
return DataHubResponse(
|
|
281
325
|
data_frame=df, columns=columns_info, affected_rows=result.affected_rows, mysql_types=result.mysql_types
|
|
282
326
|
)
|
|
327
|
+
|
|
328
|
+
@collect_metrics
|
|
329
|
+
def query_integration_handler(self, query: ASTNode) -> HandlerResponse:
|
|
330
|
+
return self.integration_handler.query(query)
|
|
331
|
+
|
|
332
|
+
@collect_metrics
|
|
333
|
+
def native_query_integration(self, query: str) -> HandlerResponse:
|
|
334
|
+
return self.integration_handler.native_query(query)
|
|
@@ -2,6 +2,7 @@ from copy import deepcopy
|
|
|
2
2
|
from dataclasses import astuple
|
|
3
3
|
|
|
4
4
|
import pandas as pd
|
|
5
|
+
from mindsdb_sql_parser.ast.base import ASTNode
|
|
5
6
|
from mindsdb_sql_parser import parse_sql
|
|
6
7
|
from mindsdb_sql_parser.ast import (
|
|
7
8
|
BinaryOperation,
|
|
@@ -99,9 +100,9 @@ class ProjectDataNode(DataNode):
|
|
|
99
100
|
|
|
100
101
|
return ml_handler.predict(model_name, df, project_name=self.project.name, version=version, params=params)
|
|
101
102
|
|
|
102
|
-
def query(self, query=
|
|
103
|
-
if query
|
|
104
|
-
query = parse_sql(
|
|
103
|
+
def query(self, query: ASTNode | str = None, session=None) -> DataHubResponse:
|
|
104
|
+
if isinstance(query, str):
|
|
105
|
+
query = parse_sql(query)
|
|
105
106
|
|
|
106
107
|
if isinstance(query, Update):
|
|
107
108
|
query_table = query.table.parts[0].lower()
|
|
@@ -132,7 +133,10 @@ class ProjectDataNode(DataNode):
|
|
|
132
133
|
case [query_table, str(version)], [is_quoted, _] if version.isdigit():
|
|
133
134
|
...
|
|
134
135
|
case _:
|
|
135
|
-
raise
|
|
136
|
+
raise EntityNotExistsError(
|
|
137
|
+
f"Table '{query.from_table}' not found in the database. The project database support only single-part names",
|
|
138
|
+
self.project.name,
|
|
139
|
+
)
|
|
136
140
|
|
|
137
141
|
if not is_quoted:
|
|
138
142
|
query_table = query_table.lower()
|
|
@@ -131,7 +131,7 @@ class TablesTable(Table):
|
|
|
131
131
|
row.TABLE_SCHEMA = ds_name
|
|
132
132
|
data.append(row.to_list())
|
|
133
133
|
except Exception:
|
|
134
|
-
logger.
|
|
134
|
+
logger.exception(f"Can't get tables from '{ds_name}'")
|
|
135
135
|
|
|
136
136
|
for project_name in inf_schema.get_projects_names():
|
|
137
137
|
if databases is not None and project_name not in databases:
|