MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +40 -29
- mindsdb/api/a2a/__init__.py +1 -1
- mindsdb/api/a2a/agent.py +16 -10
- mindsdb/api/a2a/common/server/server.py +7 -3
- mindsdb/api/a2a/common/server/task_manager.py +12 -5
- mindsdb/api/a2a/common/types.py +66 -0
- mindsdb/api/a2a/task_manager.py +65 -17
- mindsdb/api/common/middleware.py +10 -12
- mindsdb/api/executor/command_executor.py +51 -40
- mindsdb/api/executor/datahub/datanodes/datanode.py +2 -2
- mindsdb/api/executor/datahub/datanodes/information_schema_datanode.py +7 -13
- mindsdb/api/executor/datahub/datanodes/integration_datanode.py +101 -49
- mindsdb/api/executor/datahub/datanodes/project_datanode.py +8 -4
- mindsdb/api/executor/datahub/datanodes/system_tables.py +3 -2
- mindsdb/api/executor/exceptions.py +29 -10
- mindsdb/api/executor/planner/plan_join.py +17 -3
- mindsdb/api/executor/planner/query_prepare.py +2 -20
- mindsdb/api/executor/sql_query/sql_query.py +74 -74
- mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +1 -2
- mindsdb/api/executor/sql_query/steps/subselect_step.py +0 -1
- mindsdb/api/executor/utilities/functions.py +6 -6
- mindsdb/api/executor/utilities/sql.py +37 -20
- mindsdb/api/http/gui.py +5 -11
- mindsdb/api/http/initialize.py +75 -61
- mindsdb/api/http/namespaces/agents.py +10 -15
- mindsdb/api/http/namespaces/analysis.py +13 -20
- mindsdb/api/http/namespaces/auth.py +1 -1
- mindsdb/api/http/namespaces/chatbots.py +0 -5
- mindsdb/api/http/namespaces/config.py +15 -11
- mindsdb/api/http/namespaces/databases.py +140 -201
- mindsdb/api/http/namespaces/file.py +17 -4
- mindsdb/api/http/namespaces/handlers.py +17 -7
- mindsdb/api/http/namespaces/knowledge_bases.py +28 -7
- mindsdb/api/http/namespaces/models.py +94 -126
- mindsdb/api/http/namespaces/projects.py +13 -22
- mindsdb/api/http/namespaces/sql.py +33 -25
- mindsdb/api/http/namespaces/tab.py +27 -37
- mindsdb/api/http/namespaces/views.py +1 -1
- mindsdb/api/http/start.py +16 -10
- mindsdb/api/mcp/__init__.py +2 -1
- mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +15 -20
- mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +26 -50
- mindsdb/api/mysql/mysql_proxy/utilities/__init__.py +0 -1
- mindsdb/api/mysql/mysql_proxy/utilities/dump.py +8 -2
- mindsdb/integrations/handlers/byom_handler/byom_handler.py +165 -190
- mindsdb/integrations/handlers/databricks_handler/databricks_handler.py +98 -46
- mindsdb/integrations/handlers/druid_handler/druid_handler.py +32 -40
- mindsdb/integrations/handlers/file_handler/file_handler.py +7 -0
- mindsdb/integrations/handlers/gitlab_handler/gitlab_handler.py +5 -2
- mindsdb/integrations/handlers/lightwood_handler/functions.py +45 -79
- mindsdb/integrations/handlers/mssql_handler/mssql_handler.py +438 -100
- mindsdb/integrations/handlers/mssql_handler/requirements_odbc.txt +3 -0
- mindsdb/integrations/handlers/mysql_handler/mysql_handler.py +235 -3
- mindsdb/integrations/handlers/oracle_handler/__init__.py +2 -0
- mindsdb/integrations/handlers/oracle_handler/connection_args.py +7 -1
- mindsdb/integrations/handlers/oracle_handler/oracle_handler.py +321 -16
- mindsdb/integrations/handlers/oracle_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +14 -2
- mindsdb/integrations/handlers/shopify_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +80 -13
- mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +2 -1
- mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +1 -0
- mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +4 -4
- mindsdb/integrations/handlers/zendesk_handler/zendesk_tables.py +144 -111
- mindsdb/integrations/libs/api_handler.py +10 -10
- mindsdb/integrations/libs/base.py +4 -4
- mindsdb/integrations/libs/llm/utils.py +2 -2
- mindsdb/integrations/libs/ml_handler_process/create_engine_process.py +4 -7
- mindsdb/integrations/libs/ml_handler_process/func_call_process.py +2 -7
- mindsdb/integrations/libs/ml_handler_process/learn_process.py +37 -47
- mindsdb/integrations/libs/ml_handler_process/update_engine_process.py +4 -7
- mindsdb/integrations/libs/ml_handler_process/update_process.py +2 -7
- mindsdb/integrations/libs/process_cache.py +132 -140
- mindsdb/integrations/libs/response.py +18 -12
- mindsdb/integrations/libs/vectordatabase_handler.py +26 -0
- mindsdb/integrations/utilities/files/file_reader.py +6 -7
- mindsdb/integrations/utilities/handlers/auth_utilities/snowflake/__init__.py +1 -0
- mindsdb/integrations/utilities/handlers/auth_utilities/snowflake/snowflake_jwt_gen.py +151 -0
- mindsdb/integrations/utilities/rag/config_loader.py +37 -26
- mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +83 -30
- mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py +4 -4
- mindsdb/integrations/utilities/rag/retrievers/sql_retriever.py +55 -133
- mindsdb/integrations/utilities/rag/settings.py +58 -133
- mindsdb/integrations/utilities/rag/splitters/file_splitter.py +5 -15
- mindsdb/interfaces/agents/agents_controller.py +2 -3
- mindsdb/interfaces/agents/constants.py +0 -2
- mindsdb/interfaces/agents/litellm_server.py +34 -58
- mindsdb/interfaces/agents/mcp_client_agent.py +10 -10
- mindsdb/interfaces/agents/mindsdb_database_agent.py +5 -5
- mindsdb/interfaces/agents/run_mcp_agent.py +12 -21
- mindsdb/interfaces/chatbot/chatbot_task.py +20 -23
- mindsdb/interfaces/chatbot/polling.py +30 -18
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +16 -17
- mindsdb/interfaces/data_catalog/data_catalog_reader.py +15 -4
- mindsdb/interfaces/database/data_handlers_cache.py +190 -0
- mindsdb/interfaces/database/database.py +3 -3
- mindsdb/interfaces/database/integrations.py +7 -110
- mindsdb/interfaces/database/projects.py +2 -6
- mindsdb/interfaces/database/views.py +1 -4
- mindsdb/interfaces/file/file_controller.py +6 -6
- mindsdb/interfaces/functions/controller.py +1 -1
- mindsdb/interfaces/functions/to_markdown.py +2 -2
- mindsdb/interfaces/jobs/jobs_controller.py +5 -9
- mindsdb/interfaces/jobs/scheduler.py +3 -9
- mindsdb/interfaces/knowledge_base/controller.py +244 -128
- mindsdb/interfaces/knowledge_base/evaluate.py +36 -41
- mindsdb/interfaces/knowledge_base/executor.py +11 -0
- mindsdb/interfaces/knowledge_base/llm_client.py +51 -17
- mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +40 -61
- mindsdb/interfaces/model/model_controller.py +172 -168
- mindsdb/interfaces/query_context/context_controller.py +14 -2
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +10 -14
- mindsdb/interfaces/skills/retrieval_tool.py +43 -50
- mindsdb/interfaces/skills/skill_tool.py +2 -2
- mindsdb/interfaces/skills/skills_controller.py +1 -4
- mindsdb/interfaces/skills/sql_agent.py +25 -19
- mindsdb/interfaces/storage/db.py +16 -6
- mindsdb/interfaces/storage/fs.py +114 -169
- mindsdb/interfaces/storage/json.py +19 -18
- mindsdb/interfaces/tabs/tabs_controller.py +49 -72
- mindsdb/interfaces/tasks/task_monitor.py +3 -9
- mindsdb/interfaces/tasks/task_thread.py +7 -9
- mindsdb/interfaces/triggers/trigger_task.py +7 -13
- mindsdb/interfaces/triggers/triggers_controller.py +47 -52
- mindsdb/migrations/migrate.py +16 -16
- mindsdb/utilities/api_status.py +58 -0
- mindsdb/utilities/config.py +68 -2
- mindsdb/utilities/exception.py +40 -1
- mindsdb/utilities/fs.py +0 -1
- mindsdb/utilities/hooks/profiling.py +17 -14
- mindsdb/utilities/json_encoder.py +24 -10
- mindsdb/utilities/langfuse.py +40 -45
- mindsdb/utilities/log.py +272 -0
- mindsdb/utilities/ml_task_queue/consumer.py +52 -58
- mindsdb/utilities/ml_task_queue/producer.py +26 -30
- mindsdb/utilities/render/sqlalchemy_render.py +22 -20
- mindsdb/utilities/starters.py +0 -10
- mindsdb/utilities/utils.py +2 -2
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0.dist-info}/METADATA +286 -267
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0.dist-info}/RECORD +145 -159
- mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -14
- mindsdb/api/postgres/__init__.py +0 -0
- mindsdb/api/postgres/postgres_proxy/__init__.py +0 -0
- mindsdb/api/postgres/postgres_proxy/executor/__init__.py +0 -1
- mindsdb/api/postgres/postgres_proxy/executor/executor.py +0 -189
- mindsdb/api/postgres/postgres_proxy/postgres_packets/__init__.py +0 -0
- mindsdb/api/postgres/postgres_proxy/postgres_packets/errors.py +0 -322
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_fields.py +0 -34
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message.py +0 -31
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message_formats.py +0 -1265
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message_identifiers.py +0 -31
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +0 -253
- mindsdb/api/postgres/postgres_proxy/postgres_proxy.py +0 -477
- mindsdb/api/postgres/postgres_proxy/utilities/__init__.py +0 -10
- mindsdb/api/postgres/start.py +0 -11
- mindsdb/integrations/handlers/mssql_handler/tests/__init__.py +0 -0
- mindsdb/integrations/handlers/mssql_handler/tests/test_mssql_handler.py +0 -169
- mindsdb/integrations/handlers/oracle_handler/tests/__init__.py +0 -0
- mindsdb/integrations/handlers/oracle_handler/tests/test_oracle_handler.py +0 -32
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0.dist-info}/WHEEL +0 -0
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0.dist-info}/top_level.txt +0 -0
|
@@ -49,6 +49,7 @@ from mindsdb_sql_parser.ast.mindsdb import (
|
|
|
49
49
|
CreateDatabase,
|
|
50
50
|
CreateJob,
|
|
51
51
|
CreateKnowledgeBase,
|
|
52
|
+
AlterKnowledgeBase,
|
|
52
53
|
CreateMLEngine,
|
|
53
54
|
CreatePredictor,
|
|
54
55
|
CreateSkill,
|
|
@@ -196,11 +197,13 @@ def match_two_part_name(
|
|
|
196
197
|
ValueError: If the identifier does not contain one or two parts, or if ensure_lower_case is True and the name is not lowercase.
|
|
197
198
|
"""
|
|
198
199
|
db_name = None
|
|
200
|
+
|
|
199
201
|
match identifier.parts, identifier.is_quoted:
|
|
200
202
|
case [name], [is_quoted]:
|
|
201
203
|
...
|
|
202
|
-
case [db_name, name], [
|
|
203
|
-
|
|
204
|
+
case [db_name, name], [db_is_quoted, is_quoted]:
|
|
205
|
+
if not db_is_quoted:
|
|
206
|
+
db_name = db_name.lower()
|
|
204
207
|
case _:
|
|
205
208
|
raise ValueError(f"Only single-part or two-part names are allowed: {identifier}")
|
|
206
209
|
if not is_quoted:
|
|
@@ -655,6 +658,8 @@ class ExecuteCommands:
|
|
|
655
658
|
return self.answer_drop_chatbot(statement, database_name)
|
|
656
659
|
elif statement_type is CreateKnowledgeBase:
|
|
657
660
|
return self.answer_create_kb(statement, database_name)
|
|
661
|
+
elif statement_type is AlterKnowledgeBase:
|
|
662
|
+
return self.answer_alter_kb(statement, database_name)
|
|
658
663
|
elif statement_type is DropKnowledgeBase:
|
|
659
664
|
return self.answer_drop_kb(statement, database_name)
|
|
660
665
|
elif statement_type is CreateSkill:
|
|
@@ -710,9 +715,7 @@ class ExecuteCommands:
|
|
|
710
715
|
|
|
711
716
|
def answer_create_trigger(self, statement, database_name):
|
|
712
717
|
triggers_controller = TriggersController()
|
|
713
|
-
project_name, trigger_name = match_two_part_name(
|
|
714
|
-
statement.name, ensure_lower_case=True, default_db_name=database_name
|
|
715
|
-
)
|
|
718
|
+
project_name, trigger_name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
716
719
|
|
|
717
720
|
triggers_controller.add(
|
|
718
721
|
trigger_name,
|
|
@@ -726,9 +729,7 @@ class ExecuteCommands:
|
|
|
726
729
|
def answer_drop_trigger(self, statement, database_name):
|
|
727
730
|
triggers_controller = TriggersController()
|
|
728
731
|
|
|
729
|
-
|
|
730
|
-
trigger_name = statement.name.parts[-1]
|
|
731
|
-
project_name = name.parts[-2] if len(name.parts) > 1 else database_name
|
|
732
|
+
project_name, trigger_name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
732
733
|
|
|
733
734
|
triggers_controller.delete(trigger_name, project_name)
|
|
734
735
|
|
|
@@ -736,9 +737,7 @@ class ExecuteCommands:
|
|
|
736
737
|
|
|
737
738
|
def answer_create_job(self, statement: CreateJob, database_name):
|
|
738
739
|
jobs_controller = JobsController()
|
|
739
|
-
project_name, job_name = match_two_part_name(
|
|
740
|
-
statement.name, ensure_lower_case=True, default_db_name=database_name
|
|
741
|
-
)
|
|
740
|
+
project_name, job_name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
742
741
|
|
|
743
742
|
try:
|
|
744
743
|
jobs_controller.create(job_name, project_name, statement)
|
|
@@ -757,14 +756,12 @@ class ExecuteCommands:
|
|
|
757
756
|
except EntityNotExistsError:
|
|
758
757
|
if statement.if_exists is False:
|
|
759
758
|
raise
|
|
760
|
-
except Exception as e:
|
|
761
|
-
raise e
|
|
762
759
|
|
|
763
760
|
return ExecuteAnswer()
|
|
764
761
|
|
|
765
762
|
def answer_create_chatbot(self, statement, database_name):
|
|
766
763
|
chatbot_controller = ChatBotController()
|
|
767
|
-
project_name, name = match_two_part_name(statement.name,
|
|
764
|
+
project_name, name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
768
765
|
|
|
769
766
|
is_running = statement.params.pop("is_running", True)
|
|
770
767
|
|
|
@@ -796,9 +793,7 @@ class ExecuteCommands:
|
|
|
796
793
|
def answer_update_chatbot(self, statement, database_name):
|
|
797
794
|
chatbot_controller = ChatBotController()
|
|
798
795
|
|
|
799
|
-
name = statement.name
|
|
800
|
-
name_no_project = name.parts[-1]
|
|
801
|
-
project_name = name.parts[-2] if len(name.parts) > 1 else database_name
|
|
796
|
+
project_name, name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
802
797
|
|
|
803
798
|
# From SET keyword parameters
|
|
804
799
|
updated_name = statement.params.pop("name", None)
|
|
@@ -815,7 +810,7 @@ class ExecuteCommands:
|
|
|
815
810
|
database_id = database["id"]
|
|
816
811
|
|
|
817
812
|
updated_chatbot = chatbot_controller.update_chatbot(
|
|
818
|
-
|
|
813
|
+
name,
|
|
819
814
|
project_name=project_name,
|
|
820
815
|
name=updated_name,
|
|
821
816
|
model_name=model_name,
|
|
@@ -825,16 +820,15 @@ class ExecuteCommands:
|
|
|
825
820
|
params=statement.params,
|
|
826
821
|
)
|
|
827
822
|
if updated_chatbot is None:
|
|
828
|
-
raise ExecutorException(f"Chatbot with name {
|
|
823
|
+
raise ExecutorException(f"Chatbot with name {name} not found")
|
|
829
824
|
return ExecuteAnswer()
|
|
830
825
|
|
|
831
826
|
def answer_drop_chatbot(self, statement, database_name):
|
|
832
827
|
chatbot_controller = ChatBotController()
|
|
833
828
|
|
|
834
|
-
name = statement.name
|
|
835
|
-
project_name = name.parts[-2] if len(name.parts) > 1 else database_name
|
|
829
|
+
project_name, name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
836
830
|
|
|
837
|
-
chatbot_controller.delete_chatbot(name
|
|
831
|
+
chatbot_controller.delete_chatbot(name, project_name=project_name)
|
|
838
832
|
return ExecuteAnswer()
|
|
839
833
|
|
|
840
834
|
def answer_evaluate_metric(self, statement, database_name):
|
|
@@ -844,7 +838,7 @@ class ExecuteCommands:
|
|
|
844
838
|
try:
|
|
845
839
|
sqlquery = SQLQuery(statement.data, session=self.session, database=database_name)
|
|
846
840
|
except Exception as e:
|
|
847
|
-
raise Exception(f'Nested query failed to execute with error: "{e}", please check and try again.')
|
|
841
|
+
raise Exception(f'Nested query failed to execute with error: "{e}", please check and try again.') from e
|
|
848
842
|
df = sqlquery.fetched_data.to_df()
|
|
849
843
|
df.columns = [str(t.alias) if hasattr(t, "alias") else str(t.parts[-1]) for t in statement.data.targets]
|
|
850
844
|
|
|
@@ -974,6 +968,10 @@ class ExecuteCommands:
|
|
|
974
968
|
identifier.is_quoted = [False] + identifier.is_quoted
|
|
975
969
|
|
|
976
970
|
database_name, model_name, model_version = resolve_model_identifier(identifier)
|
|
971
|
+
# at least two part in identifier
|
|
972
|
+
identifier.parts[0] = database_name
|
|
973
|
+
identifier.parts[1] = model_name
|
|
974
|
+
|
|
977
975
|
if database_name is None:
|
|
978
976
|
database_name = database_name
|
|
979
977
|
|
|
@@ -1161,7 +1159,7 @@ class ExecuteCommands:
|
|
|
1161
1159
|
Raises:
|
|
1162
1160
|
ValueError: If the ml_engine name format is invalid.
|
|
1163
1161
|
"""
|
|
1164
|
-
name = match_one_part_name(statement.name
|
|
1162
|
+
name = match_one_part_name(statement.name)
|
|
1165
1163
|
|
|
1166
1164
|
handler = statement.handler
|
|
1167
1165
|
params = statement.params
|
|
@@ -1210,7 +1208,7 @@ class ExecuteCommands:
|
|
|
1210
1208
|
ast_drop = DropMLEngine(name=Identifier(name))
|
|
1211
1209
|
self.answer_drop_ml_engine(ast_drop)
|
|
1212
1210
|
logger.info(msg)
|
|
1213
|
-
raise ExecutorException(msg)
|
|
1211
|
+
raise ExecutorException(msg) from e
|
|
1214
1212
|
|
|
1215
1213
|
return ExecuteAnswer()
|
|
1216
1214
|
|
|
@@ -1247,7 +1245,7 @@ class ExecuteCommands:
|
|
|
1247
1245
|
Returns:
|
|
1248
1246
|
ExecuteAnswer: 'ok' answer
|
|
1249
1247
|
"""
|
|
1250
|
-
database_name = match_one_part_name(statement.name
|
|
1248
|
+
database_name = match_one_part_name(statement.name)
|
|
1251
1249
|
|
|
1252
1250
|
engine = (statement.engine or "mindsdb").lower()
|
|
1253
1251
|
|
|
@@ -1336,9 +1334,7 @@ class ExecuteCommands:
|
|
|
1336
1334
|
Returns:
|
|
1337
1335
|
ExecuteAnswer: answer for the command
|
|
1338
1336
|
"""
|
|
1339
|
-
project_name, view_name = match_two_part_name(
|
|
1340
|
-
statement.name, default_db_name=database_name, ensure_lower_case=isinstance(statement, CreateView)
|
|
1341
|
-
)
|
|
1337
|
+
project_name, view_name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
1342
1338
|
|
|
1343
1339
|
query_str = statement.query_str
|
|
1344
1340
|
|
|
@@ -1390,10 +1386,15 @@ class ExecuteCommands:
|
|
|
1390
1386
|
case _:
|
|
1391
1387
|
raise ValueError(f"Invalid view name: {name}")
|
|
1392
1388
|
|
|
1389
|
+
if not db_name_quoted:
|
|
1390
|
+
database_name = database_name.lower()
|
|
1391
|
+
if not view_name_quoted:
|
|
1392
|
+
view_name = view_name.lower()
|
|
1393
|
+
|
|
1393
1394
|
project = self.session.database_controller.get_project(database_name, db_name_quoted)
|
|
1394
1395
|
|
|
1395
1396
|
try:
|
|
1396
|
-
project.drop_view(view_name, strict_case=
|
|
1397
|
+
project.drop_view(view_name, strict_case=True)
|
|
1397
1398
|
except EntityNotExistsError:
|
|
1398
1399
|
if statement.if_exists is not True:
|
|
1399
1400
|
raise
|
|
@@ -1407,9 +1408,7 @@ class ExecuteCommands:
|
|
|
1407
1408
|
"Please pass the model parameters as a JSON object in the embedding_model field."
|
|
1408
1409
|
)
|
|
1409
1410
|
|
|
1410
|
-
project_name, kb_name = match_two_part_name(
|
|
1411
|
-
statement.name, ensure_lower_case=True, default_db_name=database_name
|
|
1412
|
-
)
|
|
1411
|
+
project_name, kb_name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
1413
1412
|
|
|
1414
1413
|
if statement.storage is not None:
|
|
1415
1414
|
if len(statement.storage.parts) != 2:
|
|
@@ -1433,6 +1432,20 @@ class ExecuteCommands:
|
|
|
1433
1432
|
|
|
1434
1433
|
return ExecuteAnswer()
|
|
1435
1434
|
|
|
1435
|
+
def answer_alter_kb(self, statement: AlterKnowledgeBase, database_name: str):
|
|
1436
|
+
project_name, kb_name = match_two_part_name(
|
|
1437
|
+
statement.name, ensure_lower_case=True, default_db_name=database_name
|
|
1438
|
+
)
|
|
1439
|
+
|
|
1440
|
+
# update the knowledge base
|
|
1441
|
+
self.session.kb_controller.update(
|
|
1442
|
+
name=kb_name,
|
|
1443
|
+
project_name=project_name,
|
|
1444
|
+
params=statement.params,
|
|
1445
|
+
)
|
|
1446
|
+
|
|
1447
|
+
return ExecuteAnswer()
|
|
1448
|
+
|
|
1436
1449
|
def answer_drop_kb(self, statement: DropKnowledgeBase, database_name: str) -> ExecuteAnswer:
|
|
1437
1450
|
project_name, kb_name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
1438
1451
|
|
|
@@ -1446,7 +1459,7 @@ class ExecuteCommands:
|
|
|
1446
1459
|
return ExecuteAnswer()
|
|
1447
1460
|
|
|
1448
1461
|
def answer_create_skill(self, statement, database_name):
|
|
1449
|
-
project_name, name = match_two_part_name(statement.name,
|
|
1462
|
+
project_name, name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
1450
1463
|
|
|
1451
1464
|
try:
|
|
1452
1465
|
_ = self.session.skills_controller.add_skill(name, project_name, statement.type, statement.params)
|
|
@@ -1482,7 +1495,7 @@ class ExecuteCommands:
|
|
|
1482
1495
|
return ExecuteAnswer()
|
|
1483
1496
|
|
|
1484
1497
|
def answer_create_agent(self, statement, database_name):
|
|
1485
|
-
project_name, name = match_two_part_name(statement.name,
|
|
1498
|
+
project_name, name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
1486
1499
|
|
|
1487
1500
|
skills = statement.params.pop("skills", [])
|
|
1488
1501
|
provider = statement.params.pop("provider", None)
|
|
@@ -1538,11 +1551,9 @@ class ExecuteCommands:
|
|
|
1538
1551
|
|
|
1539
1552
|
@mark_process("learn")
|
|
1540
1553
|
def answer_create_predictor(self, statement: CreatePredictor, database_name: str):
|
|
1541
|
-
integration_name, model_name = match_two_part_name(
|
|
1542
|
-
statement.name, ensure_lower_case=True, default_db_name=database_name
|
|
1543
|
-
)
|
|
1554
|
+
integration_name, model_name = match_two_part_name(statement.name, default_db_name=database_name)
|
|
1544
1555
|
|
|
1545
|
-
statement.name.parts = [integration_name
|
|
1556
|
+
statement.name.parts = [integration_name, model_name]
|
|
1546
1557
|
statement.name.is_quoted = [False, False]
|
|
1547
1558
|
|
|
1548
1559
|
ml_integration_name = "lightwood" # default
|
|
@@ -2018,7 +2029,7 @@ class ExecuteCommands:
|
|
|
2018
2029
|
else:
|
|
2019
2030
|
# drop model
|
|
2020
2031
|
try:
|
|
2021
|
-
project = self.session.database_controller.get_project(project_name)
|
|
2032
|
+
project = self.session.database_controller.get_project(project_name, strict_case=True)
|
|
2022
2033
|
project.drop_model(model_name)
|
|
2023
2034
|
except Exception as e:
|
|
2024
2035
|
if not statement.if_exists:
|
|
@@ -4,7 +4,7 @@ from mindsdb.api.executor.datahub.classes.response import DataHubResponse
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class DataNode:
|
|
7
|
-
type =
|
|
7
|
+
type = "meta"
|
|
8
8
|
|
|
9
9
|
def __init__(self):
|
|
10
10
|
pass
|
|
@@ -21,5 +21,5 @@ class DataNode:
|
|
|
21
21
|
def get_table_columns_names(self, table_name: str, schema_name: str | None = None) -> list[str]:
|
|
22
22
|
pass
|
|
23
23
|
|
|
24
|
-
def query(self, query=None,
|
|
24
|
+
def query(self, query=None, session=None) -> DataHubResponse:
|
|
25
25
|
pass
|
|
@@ -94,17 +94,7 @@ class InformationSchemaDataNode(DataNode):
|
|
|
94
94
|
self.integration_controller = session.integration_controller
|
|
95
95
|
self.project_controller = ProjectController()
|
|
96
96
|
self.database_controller = session.database_controller
|
|
97
|
-
|
|
98
|
-
self.persis_datanodes = {"log": self.database_controller.logs_db_controller}
|
|
99
|
-
|
|
100
|
-
databases = self.database_controller.get_dict()
|
|
101
|
-
if "files" in databases:
|
|
102
|
-
self.persis_datanodes["files"] = IntegrationDataNode(
|
|
103
|
-
"files",
|
|
104
|
-
ds_type="file",
|
|
105
|
-
integration_controller=self.session.integration_controller,
|
|
106
|
-
)
|
|
107
|
-
|
|
97
|
+
self.persist_datanodes_names = ("log", "files")
|
|
108
98
|
self.tables = {t.name: t for t in self.tables_list}
|
|
109
99
|
|
|
110
100
|
def __getitem__(self, key):
|
|
@@ -119,8 +109,12 @@ class InformationSchemaDataNode(DataNode):
|
|
|
119
109
|
if name_lower == "log":
|
|
120
110
|
return self.database_controller.get_system_db("log")
|
|
121
111
|
|
|
122
|
-
if name_lower
|
|
123
|
-
return
|
|
112
|
+
if name_lower == "files":
|
|
113
|
+
return IntegrationDataNode(
|
|
114
|
+
"files",
|
|
115
|
+
ds_type="file",
|
|
116
|
+
integration_controller=self.session.integration_controller,
|
|
117
|
+
)
|
|
124
118
|
|
|
125
119
|
existing_databases_meta = self.database_controller.get_dict() # filter_type='project'
|
|
126
120
|
database_name = None
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import time
|
|
2
2
|
import inspect
|
|
3
|
+
import functools
|
|
3
4
|
from dataclasses import astuple
|
|
4
5
|
from typing import Iterable, List
|
|
5
6
|
|
|
@@ -20,7 +21,7 @@ from mindsdb.integrations.utilities.utils import get_class_name
|
|
|
20
21
|
from mindsdb.metrics import metrics
|
|
21
22
|
from mindsdb.utilities import log
|
|
22
23
|
from mindsdb.utilities.profiler import profiler
|
|
23
|
-
from mindsdb.utilities.exception import
|
|
24
|
+
from mindsdb.utilities.exception import QueryError
|
|
24
25
|
from mindsdb.api.executor.datahub.datanodes.system_tables import infer_mysql_type
|
|
25
26
|
|
|
26
27
|
logger = log.getLogger(__name__)
|
|
@@ -30,6 +31,51 @@ class DBHandlerException(Exception):
|
|
|
30
31
|
pass
|
|
31
32
|
|
|
32
33
|
|
|
34
|
+
def collect_metrics(func):
|
|
35
|
+
"""Decorator for collecting performance metrics if integration handler query.
|
|
36
|
+
|
|
37
|
+
The decorator measures:
|
|
38
|
+
- Query execution time using high-precision performance counter
|
|
39
|
+
- Response size (number of rows returned)
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
func: The function to be decorated (integration handler method)
|
|
43
|
+
|
|
44
|
+
Returns:
|
|
45
|
+
function: Wrapped function that includes metrics collection and error handling
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
@functools.wraps(func)
|
|
49
|
+
def wrapper(self, *args, **kwargs):
|
|
50
|
+
try:
|
|
51
|
+
time_before_query = time.perf_counter()
|
|
52
|
+
result = func(self, *args, **kwargs)
|
|
53
|
+
|
|
54
|
+
# metrics
|
|
55
|
+
handler_class_name = get_class_name(self.integration_handler)
|
|
56
|
+
elapsed_seconds = time.perf_counter() - time_before_query
|
|
57
|
+
query_time_with_labels = metrics.INTEGRATION_HANDLER_QUERY_TIME.labels(handler_class_name, result.type)
|
|
58
|
+
query_time_with_labels.observe(elapsed_seconds)
|
|
59
|
+
|
|
60
|
+
num_rows = 0
|
|
61
|
+
if result.data_frame is not None:
|
|
62
|
+
num_rows = len(result.data_frame.index)
|
|
63
|
+
response_size_with_labels = metrics.INTEGRATION_HANDLER_RESPONSE_SIZE.labels(
|
|
64
|
+
handler_class_name, result.type
|
|
65
|
+
)
|
|
66
|
+
response_size_with_labels.observe(num_rows)
|
|
67
|
+
logger.debug(f"Handler '{handler_class_name}' returned {num_rows} rows in {elapsed_seconds:.3f} seconds")
|
|
68
|
+
except Exception as e:
|
|
69
|
+
msg = str(e).strip()
|
|
70
|
+
if msg == "":
|
|
71
|
+
msg = e.__class__.__name__
|
|
72
|
+
msg = f"[{self.ds_type}/{self.integration_name}]: {msg}"
|
|
73
|
+
raise DBHandlerException(msg) from e
|
|
74
|
+
return result
|
|
75
|
+
|
|
76
|
+
return wrapper
|
|
77
|
+
|
|
78
|
+
|
|
33
79
|
class IntegrationDataNode(DataNode):
|
|
34
80
|
type = "integration"
|
|
35
81
|
|
|
@@ -46,16 +92,10 @@ class IntegrationDataNode(DataNode):
|
|
|
46
92
|
response = self.integration_handler.get_tables()
|
|
47
93
|
if response.type == RESPONSE_TYPE.TABLE:
|
|
48
94
|
result_dict = response.data_frame.to_dict(orient="records")
|
|
49
|
-
|
|
50
|
-
for row in result_dict:
|
|
51
|
-
result.append(TablesRow.from_dict(row))
|
|
52
|
-
return result
|
|
95
|
+
return [TablesRow.from_dict(row) for row in result_dict]
|
|
53
96
|
else:
|
|
54
97
|
raise Exception(f"Can't get tables: {response.error_message}")
|
|
55
98
|
|
|
56
|
-
result_dict = response.data_frame.to_dict(orient="records")
|
|
57
|
-
return [TablesRow.from_dict(row) for row in result_dict]
|
|
58
|
-
|
|
59
99
|
def get_table_columns_df(self, table_name: str, schema_name: str | None = None) -> pd.DataFrame:
|
|
60
100
|
"""Get a DataFrame containing representation of information_schema.columns for the specified table.
|
|
61
101
|
|
|
@@ -84,7 +124,7 @@ class IntegrationDataNode(DataNode):
|
|
|
84
124
|
df.columns = [name.upper() for name in df.columns]
|
|
85
125
|
if "FIELD" not in df.columns or "TYPE" not in df.columns:
|
|
86
126
|
logger.warning(
|
|
87
|
-
f"Response from the handler's `get_columns` call does not contain required columns:
|
|
127
|
+
f"Response from the handler's `get_columns` call does not contain required columns: {list(df.columns)}"
|
|
88
128
|
)
|
|
89
129
|
return pd.DataFrame([], columns=astuple(INF_SCHEMA_COLUMNS_NAMES))
|
|
90
130
|
|
|
@@ -214,50 +254,54 @@ class IntegrationDataNode(DataNode):
|
|
|
214
254
|
return self.integration_handler.query_stream(query, fetch_size=fetch_size)
|
|
215
255
|
|
|
216
256
|
@profiler.profile()
|
|
217
|
-
def query(self, query: ASTNode |
|
|
218
|
-
|
|
219
|
-
time_before_query = time.perf_counter()
|
|
220
|
-
if query is not None:
|
|
221
|
-
result: HandlerResponse = self.integration_handler.query(query)
|
|
222
|
-
else:
|
|
223
|
-
# try to fetch native query
|
|
224
|
-
result: HandlerResponse = self.integration_handler.native_query(native_query)
|
|
257
|
+
def query(self, query: ASTNode | str = None, session=None) -> DataHubResponse:
|
|
258
|
+
"""Execute a query against the integration data source.
|
|
225
259
|
|
|
226
|
-
|
|
227
|
-
elapsed_seconds = time.perf_counter() - time_before_query
|
|
228
|
-
query_time_with_labels = metrics.INTEGRATION_HANDLER_QUERY_TIME.labels(
|
|
229
|
-
get_class_name(self.integration_handler), result.type
|
|
230
|
-
)
|
|
231
|
-
query_time_with_labels.observe(elapsed_seconds)
|
|
260
|
+
This method processes SQL queries either as ASTNode objects or raw SQL strings
|
|
232
261
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
262
|
+
Args:
|
|
263
|
+
query (ASTNode | str, optional): The query to execute. Can be either:
|
|
264
|
+
- ASTNode: A parsed SQL query object
|
|
265
|
+
- str: Raw SQL query string
|
|
266
|
+
session: Session object (currently unused but kept for compatibility)
|
|
267
|
+
|
|
268
|
+
Returns:
|
|
269
|
+
DataHubResponse: Response object
|
|
270
|
+
|
|
271
|
+
Raises:
|
|
272
|
+
NotImplementedError: If query is not ASTNode or str type
|
|
273
|
+
Exception: If the query execution fails with an error response
|
|
274
|
+
"""
|
|
275
|
+
if isinstance(query, ASTNode):
|
|
276
|
+
result: HandlerResponse = self.query_integration_handler(query=query)
|
|
277
|
+
elif isinstance(query, str):
|
|
278
|
+
result: HandlerResponse = self.native_query_integration(query=query)
|
|
279
|
+
else:
|
|
280
|
+
raise NotImplementedError("Thew query argument must be ASTNode or string type")
|
|
246
281
|
|
|
247
282
|
if result.type == RESPONSE_TYPE.ERROR:
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
283
|
+
if isinstance(query, ASTNode):
|
|
284
|
+
try:
|
|
285
|
+
query_str = query.to_string()
|
|
286
|
+
except Exception:
|
|
287
|
+
# most likely it is CreateTable with exotic column types
|
|
288
|
+
query_str = "can't be dump"
|
|
289
|
+
else:
|
|
290
|
+
query_str = query
|
|
291
|
+
|
|
292
|
+
exception = QueryError(
|
|
293
|
+
db_name=self.integration_handler.name,
|
|
294
|
+
db_type=self.integration_handler.__class__.name,
|
|
295
|
+
db_error_msg=result.error_message,
|
|
296
|
+
failed_query=query_str,
|
|
297
|
+
is_expected=result.is_expected_error,
|
|
259
298
|
)
|
|
260
299
|
|
|
300
|
+
if result.exception is None:
|
|
301
|
+
raise exception
|
|
302
|
+
else:
|
|
303
|
+
raise exception from result.exception
|
|
304
|
+
|
|
261
305
|
if result.type == RESPONSE_TYPE.OK:
|
|
262
306
|
return DataHubResponse(affected_rows=result.affected_rows)
|
|
263
307
|
|
|
@@ -271,8 +315,8 @@ class IntegrationDataNode(DataNode):
|
|
|
271
315
|
# replace python's Nan, np.NaN, np.nan and pd.NA to None
|
|
272
316
|
# TODO keep all NAN to the end of processing, bacause replacing also changes dtypes
|
|
273
317
|
df.replace([np.NaN, pd.NA, pd.NaT], None, inplace=True)
|
|
274
|
-
except Exception
|
|
275
|
-
logger.
|
|
318
|
+
except Exception:
|
|
319
|
+
logger.exception("Issue with clearing DF from NaN values:")
|
|
276
320
|
# endregion
|
|
277
321
|
|
|
278
322
|
columns_info = [{"name": k, "type": v} for k, v in df.dtypes.items()]
|
|
@@ -280,3 +324,11 @@ class IntegrationDataNode(DataNode):
|
|
|
280
324
|
return DataHubResponse(
|
|
281
325
|
data_frame=df, columns=columns_info, affected_rows=result.affected_rows, mysql_types=result.mysql_types
|
|
282
326
|
)
|
|
327
|
+
|
|
328
|
+
@collect_metrics
|
|
329
|
+
def query_integration_handler(self, query: ASTNode) -> HandlerResponse:
|
|
330
|
+
return self.integration_handler.query(query)
|
|
331
|
+
|
|
332
|
+
@collect_metrics
|
|
333
|
+
def native_query_integration(self, query: str) -> HandlerResponse:
|
|
334
|
+
return self.integration_handler.native_query(query)
|
|
@@ -2,6 +2,7 @@ from copy import deepcopy
|
|
|
2
2
|
from dataclasses import astuple
|
|
3
3
|
|
|
4
4
|
import pandas as pd
|
|
5
|
+
from mindsdb_sql_parser.ast.base import ASTNode
|
|
5
6
|
from mindsdb_sql_parser import parse_sql
|
|
6
7
|
from mindsdb_sql_parser.ast import (
|
|
7
8
|
BinaryOperation,
|
|
@@ -99,9 +100,9 @@ class ProjectDataNode(DataNode):
|
|
|
99
100
|
|
|
100
101
|
return ml_handler.predict(model_name, df, project_name=self.project.name, version=version, params=params)
|
|
101
102
|
|
|
102
|
-
def query(self, query=
|
|
103
|
-
if query
|
|
104
|
-
query = parse_sql(
|
|
103
|
+
def query(self, query: ASTNode | str = None, session=None) -> DataHubResponse:
|
|
104
|
+
if isinstance(query, str):
|
|
105
|
+
query = parse_sql(query)
|
|
105
106
|
|
|
106
107
|
if isinstance(query, Update):
|
|
107
108
|
query_table = query.table.parts[0].lower()
|
|
@@ -132,7 +133,10 @@ class ProjectDataNode(DataNode):
|
|
|
132
133
|
case [query_table, str(version)], [is_quoted, _] if version.isdigit():
|
|
133
134
|
...
|
|
134
135
|
case _:
|
|
135
|
-
raise
|
|
136
|
+
raise EntityNotExistsError(
|
|
137
|
+
f"Table '{query.from_table}' not found in the database. The project database support only single-part names",
|
|
138
|
+
self.project.name,
|
|
139
|
+
)
|
|
136
140
|
|
|
137
141
|
if not is_quoted:
|
|
138
142
|
query_table = query_table.lower()
|
|
@@ -102,9 +102,10 @@ class TablesTable(Table):
|
|
|
102
102
|
row = TablesRow(TABLE_TYPE=TABLES_ROW_TYPE.SYSTEM_VIEW, TABLE_NAME=name)
|
|
103
103
|
data.append(row.to_list())
|
|
104
104
|
|
|
105
|
-
for ds_name
|
|
105
|
+
for ds_name in inf_schema.persist_datanodes_names:
|
|
106
106
|
if databases is not None and ds_name not in databases:
|
|
107
107
|
continue
|
|
108
|
+
ds = inf_schema.get(ds_name)
|
|
108
109
|
|
|
109
110
|
if hasattr(ds, "get_tables_rows"):
|
|
110
111
|
ds_tables = ds.get_tables_rows()
|
|
@@ -131,7 +132,7 @@ class TablesTable(Table):
|
|
|
131
132
|
row.TABLE_SCHEMA = ds_name
|
|
132
133
|
data.append(row.to_list())
|
|
133
134
|
except Exception:
|
|
134
|
-
logger.
|
|
135
|
+
logger.exception(f"Can't get tables from '{ds_name}'")
|
|
135
136
|
|
|
136
137
|
for project_name in inf_schema.get_projects_names():
|
|
137
138
|
if databases is not None and project_name not in databases:
|
|
@@ -1,39 +1,58 @@
|
|
|
1
|
+
from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import ERR
|
|
1
2
|
|
|
2
3
|
|
|
3
4
|
# base exception for unknown error
|
|
4
5
|
class UnknownError(Exception):
|
|
5
|
-
|
|
6
|
-
|
|
6
|
+
mysql_error_code = ERR.ER_UNKNOWN_ERROR
|
|
7
|
+
is_expected = False
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
# base exception for known error
|
|
10
11
|
class ExecutorException(Exception):
|
|
11
|
-
|
|
12
|
+
mysql_error_code = ERR.ER_UNKNOWN_ERROR
|
|
13
|
+
is_expected = False
|
|
12
14
|
|
|
13
15
|
|
|
14
16
|
class NotSupportedYet(ExecutorException):
|
|
15
|
-
|
|
17
|
+
mysql_error_code = ERR.ER_NOT_SUPPORTED_YET
|
|
18
|
+
is_expected = True
|
|
16
19
|
|
|
17
20
|
|
|
18
21
|
class BadDbError(ExecutorException):
|
|
19
|
-
|
|
22
|
+
mysql_error_code = ERR.ER_BAD_DB_ERROR
|
|
23
|
+
is_expected = True
|
|
20
24
|
|
|
21
25
|
|
|
22
26
|
class BadTableError(ExecutorException):
|
|
23
|
-
|
|
27
|
+
mysql_error_code = ERR.ER_BAD_DB_ERROR
|
|
28
|
+
is_expected = True
|
|
24
29
|
|
|
25
30
|
|
|
26
31
|
class KeyColumnDoesNotExist(ExecutorException):
|
|
27
|
-
|
|
32
|
+
mysql_error_code = ERR.ER_KEY_COLUMN_DOES_NOT_EXIST
|
|
33
|
+
is_expected = True
|
|
28
34
|
|
|
29
35
|
|
|
30
36
|
class TableNotExistError(ExecutorException):
|
|
31
|
-
|
|
37
|
+
mysql_error_code = ERR.ER_TABLE_EXISTS_ERROR
|
|
38
|
+
is_expected = True
|
|
32
39
|
|
|
33
40
|
|
|
34
41
|
class WrongArgumentError(ExecutorException):
|
|
35
|
-
|
|
42
|
+
mysql_error_code = ERR.ER_WRONG_ARGUMENTS
|
|
43
|
+
is_expected = True
|
|
36
44
|
|
|
37
45
|
|
|
38
46
|
class LogicError(ExecutorException):
|
|
39
|
-
|
|
47
|
+
mysql_error_code = ERR.ER_WRONG_USAGE
|
|
48
|
+
is_expected = True
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class SqlSyntaxError(ExecutorException):
|
|
52
|
+
err_code = ERR.ER_SYNTAX_ERROR
|
|
53
|
+
is_expected = True
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class WrongCharsetError(ExecutorException):
|
|
57
|
+
err_code = ERR.ER_UNKNOWN_CHARACTER_SET
|
|
58
|
+
is_expected = True
|