MindsDB 25.5.3.0__py3-none-any.whl → 25.5.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +127 -79
- mindsdb/api/a2a/__init__.py +0 -0
- mindsdb/api/a2a/__main__.py +114 -0
- mindsdb/api/a2a/a2a_client.py +439 -0
- mindsdb/api/a2a/agent.py +308 -0
- mindsdb/api/a2a/common/__init__.py +0 -0
- mindsdb/api/a2a/common/client/__init__.py +4 -0
- mindsdb/api/a2a/common/client/card_resolver.py +21 -0
- mindsdb/api/a2a/common/client/client.py +86 -0
- mindsdb/api/a2a/common/server/__init__.py +4 -0
- mindsdb/api/a2a/common/server/server.py +164 -0
- mindsdb/api/a2a/common/server/task_manager.py +287 -0
- mindsdb/api/a2a/common/server/utils.py +28 -0
- mindsdb/api/a2a/common/types.py +365 -0
- mindsdb/api/a2a/constants.py +9 -0
- mindsdb/api/a2a/run_a2a.py +129 -0
- mindsdb/api/a2a/task_manager.py +594 -0
- mindsdb/api/executor/command_executor.py +47 -27
- mindsdb/api/executor/datahub/classes/response.py +5 -2
- mindsdb/api/executor/datahub/datanodes/integration_datanode.py +39 -72
- mindsdb/api/executor/planner/query_planner.py +10 -1
- mindsdb/api/executor/sql_query/result_set.py +185 -52
- mindsdb/api/executor/sql_query/sql_query.py +1 -1
- mindsdb/api/executor/sql_query/steps/apply_predictor_step.py +9 -12
- mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +8 -10
- mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py +5 -44
- mindsdb/api/executor/sql_query/steps/insert_step.py +24 -15
- mindsdb/api/executor/sql_query/steps/join_step.py +1 -1
- mindsdb/api/executor/sql_query/steps/project_step.py +1 -1
- mindsdb/api/executor/sql_query/steps/sql_steps.py +1 -1
- mindsdb/api/executor/sql_query/steps/subselect_step.py +4 -8
- mindsdb/api/executor/sql_query/steps/union_step.py +1 -3
- mindsdb/api/http/initialize.py +99 -83
- mindsdb/api/http/namespaces/analysis.py +3 -3
- mindsdb/api/http/namespaces/file.py +8 -2
- mindsdb/api/http/namespaces/sql.py +13 -27
- mindsdb/api/mcp/start.py +42 -5
- mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py +0 -1
- mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/binary_resultset_row_package.py +52 -19
- mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +8 -10
- mindsdb/api/mysql/mysql_proxy/libs/constants/mysql.py +54 -38
- mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +82 -115
- mindsdb/api/mysql/mysql_proxy/utilities/dump.py +351 -0
- mindsdb/api/postgres/postgres_proxy/executor/executor.py +1 -1
- mindsdb/api/postgres/postgres_proxy/postgres_proxy.py +5 -6
- mindsdb/integrations/handlers/altibase_handler/altibase_handler.py +26 -27
- mindsdb/integrations/handlers/altibase_handler/connection_args.py +13 -13
- mindsdb/integrations/handlers/altibase_handler/tests/test_altibase_handler.py +8 -8
- mindsdb/integrations/handlers/altibase_handler/tests/test_altibase_handler_dsn.py +13 -13
- mindsdb/integrations/handlers/anthropic_handler/__init__.py +2 -2
- mindsdb/integrations/handlers/anthropic_handler/anthropic_handler.py +1 -3
- mindsdb/integrations/handlers/aurora_handler/aurora_handler.py +1 -0
- mindsdb/integrations/handlers/autosklearn_handler/autosklearn_handler.py +1 -1
- mindsdb/integrations/handlers/autosklearn_handler/config.py +0 -1
- mindsdb/integrations/handlers/bigquery_handler/bigquery_handler.py +1 -1
- mindsdb/integrations/handlers/bigquery_handler/tests/test_bigquery_handler.py +1 -1
- mindsdb/integrations/handlers/binance_handler/binance_handler.py +1 -0
- mindsdb/integrations/handlers/binance_handler/binance_tables.py +3 -4
- mindsdb/integrations/handlers/byom_handler/__init__.py +0 -1
- mindsdb/integrations/handlers/ckan_handler/ckan_handler.py +3 -0
- mindsdb/integrations/handlers/clickhouse_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/cloud_spanner_handler/tests/test_cloud_spanner_handler.py +0 -2
- mindsdb/integrations/handlers/cloud_sql_handler/cloud_sql_handler.py +0 -1
- mindsdb/integrations/handlers/cohere_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/cohere_handler/cohere_handler.py +11 -13
- mindsdb/integrations/handlers/confluence_handler/confluence_tables.py +6 -0
- mindsdb/integrations/handlers/databend_handler/connection_args.py +1 -1
- mindsdb/integrations/handlers/databend_handler/databend_handler.py +4 -4
- mindsdb/integrations/handlers/databend_handler/tests/__init__.py +0 -1
- mindsdb/integrations/handlers/databend_handler/tests/test_databend_handler.py +1 -1
- mindsdb/integrations/handlers/derby_handler/connection_args.py +1 -1
- mindsdb/integrations/handlers/derby_handler/derby_handler.py +14 -22
- mindsdb/integrations/handlers/derby_handler/tests/test_derby_handler.py +6 -6
- mindsdb/integrations/handlers/discord_handler/discord_handler.py +5 -5
- mindsdb/integrations/handlers/discord_handler/discord_tables.py +3 -3
- mindsdb/integrations/handlers/discord_handler/tests/test_discord.py +5 -3
- mindsdb/integrations/handlers/dockerhub_handler/dockerhub.py +3 -3
- mindsdb/integrations/handlers/dockerhub_handler/dockerhub_handler.py +2 -2
- mindsdb/integrations/handlers/dockerhub_handler/dockerhub_tables.py +57 -54
- mindsdb/integrations/handlers/dremio_handler/__init__.py +2 -2
- mindsdb/integrations/handlers/druid_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/druid_handler/druid_handler.py +2 -2
- mindsdb/integrations/handlers/edgelessdb_handler/tests/test_edgelessdb_handler.py +9 -9
- mindsdb/integrations/handlers/email_handler/email_client.py +1 -1
- mindsdb/integrations/handlers/email_handler/email_ingestor.py +1 -1
- mindsdb/integrations/handlers/email_handler/email_tables.py +0 -1
- mindsdb/integrations/handlers/email_handler/settings.py +0 -1
- mindsdb/integrations/handlers/eventstoredb_handler/eventstoredb_handler.py +2 -1
- mindsdb/integrations/handlers/firebird_handler/firebird_handler.py +1 -1
- mindsdb/integrations/handlers/flaml_handler/flaml_handler.py +9 -9
- mindsdb/integrations/handlers/frappe_handler/frappe_client.py +5 -5
- mindsdb/integrations/handlers/frappe_handler/frappe_handler.py +6 -5
- mindsdb/integrations/handlers/frappe_handler/frappe_tables.py +2 -2
- mindsdb/integrations/handlers/github_handler/connection_args.py +2 -2
- mindsdb/integrations/handlers/github_handler/github_handler.py +1 -8
- mindsdb/integrations/handlers/github_handler/github_tables.py +13 -24
- mindsdb/integrations/handlers/gitlab_handler/gitlab_handler.py +2 -1
- mindsdb/integrations/handlers/gitlab_handler/gitlab_tables.py +1 -4
- mindsdb/integrations/handlers/gmail_handler/gmail_handler.py +6 -13
- mindsdb/integrations/handlers/google_books_handler/google_books_handler.py +2 -1
- mindsdb/integrations/handlers/google_books_handler/google_books_tables.py +0 -3
- mindsdb/integrations/handlers/google_calendar_handler/google_calendar_handler.py +4 -4
- mindsdb/integrations/handlers/google_calendar_handler/google_calendar_tables.py +2 -6
- mindsdb/integrations/handlers/google_content_shopping_handler/google_content_shopping_handler.py +3 -2
- mindsdb/integrations/handlers/google_content_shopping_handler/google_content_shopping_tables.py +0 -3
- mindsdb/integrations/handlers/google_fit_handler/google_fit_handler.py +10 -12
- mindsdb/integrations/handlers/google_fit_handler/google_fit_tables.py +11 -13
- mindsdb/integrations/handlers/google_search_handler/google_search_handler.py +2 -1
- mindsdb/integrations/handlers/google_search_handler/google_search_tables.py +0 -3
- mindsdb/integrations/handlers/groq_handler/__init__.py +3 -3
- mindsdb/integrations/handlers/hackernews_handler/hn_handler.py +5 -7
- mindsdb/integrations/handlers/hackernews_handler/hn_table.py +6 -7
- mindsdb/integrations/handlers/hive_handler/tests/test_hive_handler.py +1 -1
- mindsdb/integrations/handlers/hsqldb_handler/connection_args.py +6 -6
- mindsdb/integrations/handlers/hsqldb_handler/hsqldb_handler.py +4 -3
- mindsdb/integrations/handlers/huggingface_api_handler/exceptions.py +1 -1
- mindsdb/integrations/handlers/huggingface_api_handler/huggingface_api_handler.py +1 -8
- mindsdb/integrations/handlers/huggingface_handler/huggingface_handler.py +6 -6
- mindsdb/integrations/handlers/huggingface_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/huggingface_handler/requirements_cpu.txt +1 -1
- mindsdb/integrations/handlers/ignite_handler/ignite_handler.py +2 -1
- mindsdb/integrations/handlers/impala_handler/impala_handler.py +9 -12
- mindsdb/integrations/handlers/impala_handler/tests/test_impala_handler.py +11 -11
- mindsdb/integrations/handlers/influxdb_handler/influxdb_handler.py +10 -13
- mindsdb/integrations/handlers/influxdb_handler/influxdb_tables.py +20 -20
- mindsdb/integrations/handlers/informix_handler/__about__.py +8 -8
- mindsdb/integrations/handlers/informix_handler/__init__.py +12 -5
- mindsdb/integrations/handlers/informix_handler/informix_handler.py +99 -133
- mindsdb/integrations/handlers/informix_handler/tests/test_informix_handler.py +13 -11
- mindsdb/integrations/handlers/ingres_handler/__about__.py +0 -1
- mindsdb/integrations/handlers/ingres_handler/ingres_handler.py +1 -0
- mindsdb/integrations/handlers/jira_handler/jira_handler.py +4 -4
- mindsdb/integrations/handlers/jira_handler/jira_tables.py +9 -9
- mindsdb/integrations/handlers/kinetica_handler/__init__.py +0 -1
- mindsdb/integrations/handlers/langchain_handler/langchain_handler.py +4 -4
- mindsdb/integrations/handlers/langchain_handler/tools.py +9 -10
- mindsdb/integrations/handlers/leonardoai_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/lightwood_handler/functions.py +2 -2
- mindsdb/integrations/handlers/lightwood_handler/lightwood_handler.py +0 -1
- mindsdb/integrations/handlers/lightwood_handler/tests/test_lightwood_handler.py +11 -11
- mindsdb/integrations/handlers/llama_index_handler/llama_index_handler.py +4 -4
- mindsdb/integrations/handlers/llama_index_handler/settings.py +10 -9
- mindsdb/integrations/handlers/materialize_handler/tests/test_materialize_handler.py +8 -10
- mindsdb/integrations/handlers/matrixone_handler/matrixone_handler.py +4 -4
- mindsdb/integrations/handlers/matrixone_handler/tests/test_matrixone_handler.py +8 -9
- mindsdb/integrations/handlers/maxdb_handler/connection_args.py +25 -25
- mindsdb/integrations/handlers/maxdb_handler/maxdb_handler.py +1 -0
- mindsdb/integrations/handlers/mediawiki_handler/mediawiki_handler.py +3 -2
- mindsdb/integrations/handlers/mediawiki_handler/mediawiki_tables.py +1 -1
- mindsdb/integrations/handlers/mendeley_handler/__about__.py +1 -1
- mindsdb/integrations/handlers/mendeley_handler/__init__.py +2 -2
- mindsdb/integrations/handlers/mendeley_handler/mendeley_handler.py +48 -56
- mindsdb/integrations/handlers/mendeley_handler/mendeley_tables.py +24 -29
- mindsdb/integrations/handlers/mendeley_handler/tests/test_mendeley_handler.py +19 -17
- mindsdb/integrations/handlers/merlion_handler/merlion_handler.py +5 -4
- mindsdb/integrations/handlers/minds_endpoint_handler/__init__.py +3 -3
- mindsdb/integrations/handlers/mlflow_handler/mlflow_handler.py +58 -36
- mindsdb/integrations/handlers/monetdb_handler/__about__.py +8 -8
- mindsdb/integrations/handlers/monetdb_handler/__init__.py +15 -5
- mindsdb/integrations/handlers/monetdb_handler/connection_args.py +17 -18
- mindsdb/integrations/handlers/monetdb_handler/monetdb_handler.py +40 -57
- mindsdb/integrations/handlers/monetdb_handler/tests/test_monetdb_handler.py +7 -8
- mindsdb/integrations/handlers/monetdb_handler/utils/monet_get_id.py +13 -14
- mindsdb/integrations/handlers/monkeylearn_handler/__about__.py +1 -1
- mindsdb/integrations/handlers/monkeylearn_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/monkeylearn_handler/monkeylearn_handler.py +2 -5
- mindsdb/integrations/handlers/ms_one_drive_handler/ms_graph_api_one_drive_client.py +1 -0
- mindsdb/integrations/handlers/ms_one_drive_handler/ms_one_drive_handler.py +1 -1
- mindsdb/integrations/handlers/ms_teams_handler/ms_graph_api_teams_client.py +23 -23
- mindsdb/integrations/handlers/ms_teams_handler/ms_teams_handler.py +3 -3
- mindsdb/integrations/handlers/ms_teams_handler/ms_teams_tables.py +10 -5
- mindsdb/integrations/handlers/mssql_handler/mssql_handler.py +73 -8
- mindsdb/integrations/handlers/mysql_handler/__about__.py +8 -8
- mindsdb/integrations/handlers/mysql_handler/__init__.py +15 -5
- mindsdb/integrations/handlers/mysql_handler/connection_args.py +43 -47
- mindsdb/integrations/handlers/mysql_handler/mysql_handler.py +101 -34
- mindsdb/integrations/handlers/mysql_handler/settings.py +15 -13
- mindsdb/integrations/handlers/neuralforecast_handler/neuralforecast_handler.py +1 -1
- mindsdb/integrations/handlers/newsapi_handler/newsapi_handler.py +1 -1
- mindsdb/integrations/handlers/newsapi_handler/tests/test_newsapi_handler.py +4 -4
- mindsdb/integrations/handlers/nuo_jdbc_handler/connection_args.py +2 -2
- mindsdb/integrations/handlers/nuo_jdbc_handler/nuo_jdbc_handler.py +28 -36
- mindsdb/integrations/handlers/nuo_jdbc_handler/tests/test_nuo_handler.py +5 -5
- mindsdb/integrations/handlers/oceanbase_handler/oceanbase_handler.py +0 -1
- mindsdb/integrations/handlers/oceanbase_handler/tests/test_oceanbase_handler.py +8 -10
- mindsdb/integrations/handlers/ollama_handler/ollama_handler.py +3 -3
- mindsdb/integrations/handlers/opengauss_handler/tests/test_opengauss_handler.py +1 -2
- mindsdb/integrations/handlers/openstreetmap_handler/__init__.py +7 -7
- mindsdb/integrations/handlers/oracle_handler/connection_args.py +6 -0
- mindsdb/integrations/handlers/oracle_handler/oracle_handler.py +77 -11
- mindsdb/integrations/handlers/orioledb_handler/tests/test_orioledb_handler.py +8 -10
- mindsdb/integrations/handlers/palm_handler/__about__.py +1 -1
- mindsdb/integrations/handlers/palm_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/palm_handler/palm_handler.py +1 -3
- mindsdb/integrations/handlers/paypal_handler/paypal_handler.py +2 -2
- mindsdb/integrations/handlers/paypal_handler/paypal_tables.py +15 -14
- mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py +53 -10
- mindsdb/integrations/handlers/phoenix_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/phoenix_handler/phoenix_handler.py +1 -0
- mindsdb/integrations/handlers/pinot_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/pinot_handler/pinot_handler.py +3 -2
- mindsdb/integrations/handlers/plaid_handler/plaid_handler.py +13 -13
- mindsdb/integrations/handlers/plaid_handler/plaid_tables.py +10 -12
- mindsdb/integrations/handlers/plaid_handler/utils.py +4 -6
- mindsdb/integrations/handlers/planetscale_handler/planetscale_handler.py +1 -4
- mindsdb/integrations/handlers/portkey_handler/__init__.py +2 -2
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +105 -24
- mindsdb/integrations/handlers/postgres_handler/tests/test_postgres_handler.py +11 -6
- mindsdb/integrations/handlers/questdb_handler/questdb_handler.py +1 -2
- mindsdb/integrations/handlers/questdb_handler/tests/test_questdb_handler.py +2 -3
- mindsdb/integrations/handlers/quickbooks_handler/quickbooks_handler.py +6 -8
- mindsdb/integrations/handlers/quickbooks_handler/quickbooks_table.py +10 -10
- mindsdb/integrations/handlers/rag_handler/ingest.py +2 -2
- mindsdb/integrations/handlers/rag_handler/rag_handler.py +1 -1
- mindsdb/integrations/handlers/rag_handler/settings.py +1 -1
- mindsdb/integrations/handlers/reddit_handler/reddit_handler.py +2 -7
- mindsdb/integrations/handlers/reddit_handler/reddit_tables.py +2 -3
- mindsdb/integrations/handlers/replicate_handler/replicate_handler.py +6 -6
- mindsdb/integrations/handlers/rocket_chat_handler/rocket_chat_handler.py +1 -2
- mindsdb/integrations/handlers/rocket_chat_handler/rocket_chat_tables.py +0 -3
- mindsdb/integrations/handlers/rockset_handler/connection_args.py +14 -14
- mindsdb/integrations/handlers/rockset_handler/tests/test_rockset_handler.py +1 -0
- mindsdb/integrations/handlers/scylla_handler/scylla_handler.py +6 -5
- mindsdb/integrations/handlers/sendinblue_handler/sendinblue_handler.py +2 -1
- mindsdb/integrations/handlers/sendinblue_handler/sendinblue_tables.py +16 -16
- mindsdb/integrations/handlers/sentence_transformers_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/sheets_handler/connection_args.py +1 -1
- mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +7 -6
- mindsdb/integrations/handlers/shopify_handler/shopify_tables.py +38 -41
- mindsdb/integrations/handlers/singlestore_handler/__about__.py +1 -1
- mindsdb/integrations/handlers/singlestore_handler/__init__.py +0 -1
- mindsdb/integrations/handlers/singlestore_handler/singlestore_handler.py +1 -0
- mindsdb/integrations/handlers/singlestore_handler/tests/test_singlestore_handler.py +3 -3
- mindsdb/integrations/handlers/slack_handler/__init__.py +3 -3
- mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +100 -6
- mindsdb/integrations/handlers/solr_handler/connection_args.py +7 -7
- mindsdb/integrations/handlers/solr_handler/solr_handler.py +2 -1
- mindsdb/integrations/handlers/solr_handler/tests/test_solr_handler.py +2 -1
- mindsdb/integrations/handlers/sqlany_handler/sqlany_handler.py +3 -2
- mindsdb/integrations/handlers/sqlite_handler/sqlite_handler.py +1 -0
- mindsdb/integrations/handlers/sqreamdb_handler/connection_args.py +1 -1
- mindsdb/integrations/handlers/sqreamdb_handler/sqreamdb_handler.py +15 -20
- mindsdb/integrations/handlers/sqreamdb_handler/tests/test_sqreamdb_handler.py +4 -4
- mindsdb/integrations/handlers/stabilityai_handler/__init__.py +1 -1
- mindsdb/integrations/handlers/starrocks_handler/starrocks_handler.py +0 -1
- mindsdb/integrations/handlers/starrocks_handler/tests/test_starrocks_handler.py +8 -10
- mindsdb/integrations/handlers/statsforecast_handler/statsforecast_handler.py +2 -2
- mindsdb/integrations/handlers/strava_handler/strava_handler.py +4 -8
- mindsdb/integrations/handlers/strava_handler/strava_tables.py +22 -30
- mindsdb/integrations/handlers/stripe_handler/stripe_handler.py +3 -2
- mindsdb/integrations/handlers/stripe_handler/stripe_tables.py +11 -27
- mindsdb/integrations/handlers/supabase_handler/tests/test_supabase_handler.py +1 -1
- mindsdb/integrations/handlers/surrealdb_handler/surrealdb_handler.py +4 -4
- mindsdb/integrations/handlers/tdengine_handler/tdengine_handler.py +25 -27
- mindsdb/integrations/handlers/tdengine_handler/tests/test_tdengine_handler.py +8 -8
- mindsdb/integrations/handlers/tidb_handler/tests/test_tidb_handler.py +1 -2
- mindsdb/integrations/handlers/timegpt_handler/timegpt_handler.py +5 -5
- mindsdb/integrations/handlers/tpot_handler/tpot_handler.py +21 -26
- mindsdb/integrations/handlers/trino_handler/trino_handler.py +14 -14
- mindsdb/integrations/handlers/twitter_handler/twitter_handler.py +2 -4
- mindsdb/integrations/handlers/unify_handler/tests/test_unify_handler.py +7 -8
- mindsdb/integrations/handlers/unify_handler/unify_handler.py +9 -9
- mindsdb/integrations/handlers/vertex_handler/vertex_client.py +1 -1
- mindsdb/integrations/handlers/vertica_handler/tests/test_vertica_handler.py +11 -11
- mindsdb/integrations/handlers/vertica_handler/vertica_handler.py +11 -14
- mindsdb/integrations/handlers/vitess_handler/tests/test_vitess_handler.py +9 -11
- mindsdb/integrations/handlers/vitess_handler/vitess_handler.py +0 -1
- mindsdb/integrations/handlers/web_handler/web_handler.py +1 -0
- mindsdb/integrations/handlers/whatsapp_handler/__init__.py +3 -3
- mindsdb/integrations/handlers/writer_handler/evaluate.py +1 -1
- mindsdb/integrations/handlers/writer_handler/settings.py +0 -1
- mindsdb/integrations/handlers/writer_handler/writer_handler.py +1 -0
- mindsdb/integrations/handlers/youtube_handler/youtube_handler.py +5 -5
- mindsdb/integrations/handlers/youtube_handler/youtube_tables.py +26 -27
- mindsdb/integrations/handlers/yugabyte_handler/tests/test_yugabyte_handler.py +3 -3
- mindsdb/integrations/handlers/yugabyte_handler/yugabyte_handler.py +0 -6
- mindsdb/integrations/libs/response.py +67 -52
- mindsdb/integrations/libs/vectordatabase_handler.py +6 -0
- mindsdb/integrations/utilities/handler_utils.py +15 -3
- mindsdb/integrations/utilities/handlers/api_utilities/__init__.py +0 -1
- mindsdb/integrations/utilities/handlers/auth_utilities/__init__.py +0 -2
- mindsdb/integrations/utilities/utils.py +3 -3
- mindsdb/interfaces/agents/agents_controller.py +164 -1
- mindsdb/interfaces/agents/constants.py +15 -0
- mindsdb/interfaces/agents/langchain_agent.py +16 -4
- mindsdb/interfaces/agents/mindsdb_database_agent.py +101 -2
- mindsdb/interfaces/knowledge_base/controller.py +25 -0
- mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py +13 -10
- mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +434 -0
- mindsdb/interfaces/knowledge_base/preprocessing/models.py +54 -0
- mindsdb/interfaces/query_context/context_controller.py +66 -10
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_kb_tools.py +190 -0
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +92 -0
- mindsdb/interfaces/skills/skill_tool.py +202 -57
- mindsdb/interfaces/skills/sql_agent.py +205 -17
- mindsdb/interfaces/storage/fs.py +1 -0
- mindsdb/interfaces/variables/__init__.py +0 -0
- mindsdb/interfaces/variables/variables_controller.py +97 -0
- mindsdb/migrations/env.py +5 -7
- mindsdb/migrations/migrate.py +47 -7
- mindsdb/migrations/versions/2025-05-21_9f150e4f9a05_checkpoint_1.py +360 -0
- mindsdb/utilities/config.py +331 -219
- mindsdb/utilities/starters.py +13 -0
- {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/METADATA +641 -695
- {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/RECORD +309 -288
- {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/WHEEL +1 -1
- mindsdb/integrations/handlers/monkeylearn_handler/requirements.txt +0 -1
- {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/top_level.txt +0 -0
|
@@ -1,29 +1,70 @@
|
|
|
1
1
|
import copy
|
|
2
|
-
from
|
|
2
|
+
from dataclasses import dataclass, field, MISSING
|
|
3
3
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
import pandas as pd
|
|
6
|
+
from pandas.api import types as pd_types
|
|
7
|
+
import sqlalchemy.types as sqlalchemy_types
|
|
6
8
|
|
|
9
|
+
from mindsdb_sql_parser.ast import TableColumn
|
|
10
|
+
|
|
11
|
+
from mindsdb.utilities import log
|
|
7
12
|
from mindsdb.api.executor.exceptions import WrongArgumentError
|
|
13
|
+
from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
logger = log.getLogger(__name__)
|
|
17
|
+
|
|
8
18
|
|
|
19
|
+
def get_mysql_data_type_from_series(series: pd.Series, do_infer: bool = False) -> MYSQL_DATA_TYPE:
|
|
20
|
+
"""Maps pandas Series data type to corresponding MySQL data type.
|
|
9
21
|
|
|
22
|
+
This function examines the dtype of a pandas Series and returns the appropriate
|
|
23
|
+
MySQL data type enum value. For object dtypes, it can optionally attempt to infer
|
|
24
|
+
a more specific type.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
series (pd.Series): The pandas Series to determine the MySQL type for
|
|
28
|
+
do_infer (bool): If True and series has object dtype, attempt to infer a more specific type
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
MYSQL_DATA_TYPE: The corresponding MySQL data type enum value
|
|
32
|
+
"""
|
|
33
|
+
dtype = series.dtype
|
|
34
|
+
if pd_types.is_object_dtype(dtype) and do_infer is True:
|
|
35
|
+
dtype = series.infer_objects().dtype
|
|
36
|
+
|
|
37
|
+
if pd_types.is_object_dtype(dtype):
|
|
38
|
+
return MYSQL_DATA_TYPE.TEXT
|
|
39
|
+
if pd_types.is_datetime64_dtype(dtype):
|
|
40
|
+
return MYSQL_DATA_TYPE.DATETIME
|
|
41
|
+
if pd_types.is_string_dtype(dtype):
|
|
42
|
+
return MYSQL_DATA_TYPE.TEXT
|
|
43
|
+
if pd_types.is_bool_dtype(dtype):
|
|
44
|
+
return MYSQL_DATA_TYPE.BOOL
|
|
45
|
+
if pd_types.is_integer_dtype(dtype):
|
|
46
|
+
return MYSQL_DATA_TYPE.INT
|
|
47
|
+
if pd_types.is_numeric_dtype(dtype):
|
|
48
|
+
return MYSQL_DATA_TYPE.FLOAT
|
|
49
|
+
return MYSQL_DATA_TYPE.TEXT
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(kw_only=True, slots=True)
|
|
10
53
|
class Column:
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
self.
|
|
22
|
-
|
|
23
|
-
self.
|
|
24
|
-
|
|
25
|
-
self.flags = flags
|
|
26
|
-
self.charset = charset
|
|
54
|
+
name: str = field(default=MISSING)
|
|
55
|
+
alias: str | None = None
|
|
56
|
+
table_name: str | None = None
|
|
57
|
+
table_alias: str | None = None
|
|
58
|
+
type: MYSQL_DATA_TYPE | None = None
|
|
59
|
+
database: str | None = None
|
|
60
|
+
flags: dict = None
|
|
61
|
+
charset: str | None = None
|
|
62
|
+
|
|
63
|
+
def __post_init__(self):
|
|
64
|
+
if self.alias is None:
|
|
65
|
+
self.alias = self.name
|
|
66
|
+
if self.table_alias is None:
|
|
67
|
+
self.table_alias = self.table_name
|
|
27
68
|
|
|
28
69
|
def get_hash_name(self, prefix):
|
|
29
70
|
table_name = self.table_name if self.table_alias is None else self.table_alias
|
|
@@ -32,11 +73,8 @@ class Column:
|
|
|
32
73
|
name = f'{prefix}_{table_name}_{name}'
|
|
33
74
|
return name
|
|
34
75
|
|
|
35
|
-
def __repr__(self):
|
|
36
|
-
return f'{self.__class__.__name__}({self.__dict__})'
|
|
37
|
-
|
|
38
76
|
|
|
39
|
-
def rename_df_columns(df: pd.DataFrame, names:
|
|
77
|
+
def rename_df_columns(df: pd.DataFrame, names: list | None = None) -> None:
|
|
40
78
|
"""Inplace rename of dataframe columns
|
|
41
79
|
|
|
42
80
|
Args:
|
|
@@ -50,7 +88,15 @@ def rename_df_columns(df: pd.DataFrame, names: Optional[List] = None) -> None:
|
|
|
50
88
|
|
|
51
89
|
|
|
52
90
|
class ResultSet:
|
|
53
|
-
def __init__(
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
columns: list[Column] | None = None,
|
|
94
|
+
values: list[list] | None = None,
|
|
95
|
+
df: pd.DataFrame | None = None,
|
|
96
|
+
affected_rows: int | None = None,
|
|
97
|
+
is_prediction: bool = False,
|
|
98
|
+
mysql_types: list[MYSQL_DATA_TYPE] | None = None
|
|
99
|
+
):
|
|
54
100
|
"""
|
|
55
101
|
Args:
|
|
56
102
|
columns: list of Columns
|
|
@@ -62,15 +108,18 @@ class ResultSet:
|
|
|
62
108
|
columns = []
|
|
63
109
|
self._columns = columns
|
|
64
110
|
|
|
65
|
-
if
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
111
|
+
if df is None:
|
|
112
|
+
if values is None:
|
|
113
|
+
df = None
|
|
114
|
+
else:
|
|
115
|
+
df = pd.DataFrame(values)
|
|
69
116
|
self._df = df
|
|
70
117
|
|
|
71
118
|
self.affected_rows = affected_rows
|
|
72
119
|
|
|
73
|
-
self.is_prediction =
|
|
120
|
+
self.is_prediction = is_prediction
|
|
121
|
+
|
|
122
|
+
self.mysql_types = mysql_types
|
|
74
123
|
|
|
75
124
|
def __repr__(self):
|
|
76
125
|
col_names = ', '.join([col.name for col in self._columns])
|
|
@@ -89,43 +138,76 @@ class ResultSet:
|
|
|
89
138
|
|
|
90
139
|
# --- converters ---
|
|
91
140
|
|
|
92
|
-
|
|
93
|
-
|
|
141
|
+
@classmethod
|
|
142
|
+
def from_df(
|
|
143
|
+
cls, df: pd.DataFrame, database=None, table_name=None, table_alias=None,
|
|
144
|
+
is_prediction: bool = False, mysql_types: list[MYSQL_DATA_TYPE] | None = None
|
|
145
|
+
):
|
|
146
|
+
match mysql_types:
|
|
147
|
+
case None:
|
|
148
|
+
mysql_types = [None] * len(df.columns)
|
|
149
|
+
case list() if len(mysql_types) != len(df.columns):
|
|
150
|
+
raise WrongArgumentError(
|
|
151
|
+
f'Mysql types length mismatch: {len(mysql_types)} != {len(df.columns)}'
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
columns = [
|
|
94
155
|
Column(
|
|
95
156
|
name=column_name,
|
|
96
157
|
table_name=table_name,
|
|
97
158
|
table_alias=table_alias,
|
|
98
159
|
database=database,
|
|
99
|
-
type=
|
|
100
|
-
) for column_name,
|
|
101
|
-
in zip(df.columns,
|
|
160
|
+
type=mysql_type
|
|
161
|
+
) for column_name, mysql_type
|
|
162
|
+
in zip(df.columns, mysql_types)
|
|
102
163
|
]
|
|
103
164
|
|
|
104
165
|
rename_df_columns(df)
|
|
105
|
-
|
|
166
|
+
return cls(
|
|
167
|
+
df=df,
|
|
168
|
+
columns=columns,
|
|
169
|
+
is_prediction=is_prediction,
|
|
170
|
+
mysql_types=mysql_types
|
|
171
|
+
)
|
|
106
172
|
|
|
107
|
-
|
|
173
|
+
@classmethod
|
|
174
|
+
def from_df_cols(cls, df: pd.DataFrame, columns_dict: dict[str, Column], strict: bool = True) -> 'ResultSet':
|
|
175
|
+
"""Create ResultSet from dataframe and dictionary of columns
|
|
108
176
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
if col.alias is not None:
|
|
114
|
-
alias_idx[col.alias] = col
|
|
177
|
+
Args:
|
|
178
|
+
df (pd.DataFrame): dataframe
|
|
179
|
+
columns_dict (dict[str, Column]): dictionary of columns
|
|
180
|
+
strict (bool): if True, raise an error if a column is not found in columns_dict
|
|
115
181
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
182
|
+
Returns:
|
|
183
|
+
ResultSet: result set
|
|
184
|
+
|
|
185
|
+
Raises:
|
|
186
|
+
ValueError: if a column is not found in columns_dict and strict is True
|
|
187
|
+
"""
|
|
188
|
+
alias_idx = {
|
|
189
|
+
column.alias: column
|
|
190
|
+
for column in columns_dict.values()
|
|
191
|
+
if column.alias is not None
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
columns = []
|
|
195
|
+
for column_name in df.columns:
|
|
196
|
+
if strict and column_name not in columns_dict:
|
|
197
|
+
raise ValueError(f'Column {column_name} not found in columns_dict')
|
|
198
|
+
column = (
|
|
199
|
+
columns_dict.get(column_name)
|
|
200
|
+
or alias_idx.get(column_name)
|
|
201
|
+
or Column(name=column_name)
|
|
202
|
+
)
|
|
203
|
+
columns.append(column)
|
|
124
204
|
|
|
125
205
|
rename_df_columns(df)
|
|
126
|
-
self._df = df
|
|
127
206
|
|
|
128
|
-
return
|
|
207
|
+
return cls(
|
|
208
|
+
columns=columns,
|
|
209
|
+
df=df
|
|
210
|
+
)
|
|
129
211
|
|
|
130
212
|
def to_df(self):
|
|
131
213
|
columns_names = self.get_column_names()
|
|
@@ -133,7 +215,7 @@ class ResultSet:
|
|
|
133
215
|
rename_df_columns(df, columns_names)
|
|
134
216
|
return df
|
|
135
217
|
|
|
136
|
-
def to_df_cols(self, prefix=''):
|
|
218
|
+
def to_df_cols(self, prefix: str = '') -> tuple[pd.DataFrame, dict[str, Column]]:
|
|
137
219
|
# returns dataframe and dict of columns
|
|
138
220
|
# can be restored to ResultSet by from_df_cols method
|
|
139
221
|
|
|
@@ -262,6 +344,57 @@ class ResultSet:
|
|
|
262
344
|
)
|
|
263
345
|
self.add_raw_df(df)
|
|
264
346
|
|
|
347
|
+
def get_ast_columns(self) -> list[TableColumn]:
|
|
348
|
+
"""Converts ResultSet columns to a list of TableColumn objects with SQLAlchemy types.
|
|
349
|
+
|
|
350
|
+
This method processes each column in the ResultSet, determines its MySQL data type
|
|
351
|
+
(inferring it if necessary), and maps it to the appropriate SQLAlchemy type.
|
|
352
|
+
The resulting TableColumn objects most likely will be used in CREATE TABLE statement.
|
|
353
|
+
|
|
354
|
+
Returns:
|
|
355
|
+
list[TableColumn]: A list of TableColumn objects with properly mapped SQLAlchemy types
|
|
356
|
+
"""
|
|
357
|
+
columns: list[TableColumn] = []
|
|
358
|
+
|
|
359
|
+
type_mapping = {
|
|
360
|
+
MYSQL_DATA_TYPE.TINYINT: sqlalchemy_types.INTEGER,
|
|
361
|
+
MYSQL_DATA_TYPE.SMALLINT: sqlalchemy_types.INTEGER,
|
|
362
|
+
MYSQL_DATA_TYPE.MEDIUMINT: sqlalchemy_types.INTEGER,
|
|
363
|
+
MYSQL_DATA_TYPE.INT: sqlalchemy_types.INTEGER,
|
|
364
|
+
MYSQL_DATA_TYPE.BIGINT: sqlalchemy_types.INTEGER,
|
|
365
|
+
MYSQL_DATA_TYPE.YEAR: sqlalchemy_types.INTEGER,
|
|
366
|
+
MYSQL_DATA_TYPE.BOOL: sqlalchemy_types.BOOLEAN,
|
|
367
|
+
MYSQL_DATA_TYPE.BOOLEAN: sqlalchemy_types.BOOLEAN,
|
|
368
|
+
MYSQL_DATA_TYPE.FLOAT: sqlalchemy_types.FLOAT,
|
|
369
|
+
MYSQL_DATA_TYPE.DOUBLE: sqlalchemy_types.FLOAT,
|
|
370
|
+
MYSQL_DATA_TYPE.TIME: sqlalchemy_types.TIME,
|
|
371
|
+
MYSQL_DATA_TYPE.DATE: sqlalchemy_types.DATE,
|
|
372
|
+
MYSQL_DATA_TYPE.DATETIME: sqlalchemy_types.DATETIME,
|
|
373
|
+
MYSQL_DATA_TYPE.TIMESTAMP: sqlalchemy_types.TIMESTAMP,
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
for i, column in enumerate(self._columns):
|
|
377
|
+
column_type: MYSQL_DATA_TYPE | None = column.type
|
|
378
|
+
|
|
379
|
+
# infer MYSQL_DATA_TYPE if not set
|
|
380
|
+
if isinstance(column_type, MYSQL_DATA_TYPE) is False:
|
|
381
|
+
if column_type is not None:
|
|
382
|
+
logger.warning(f'Unexpected column type: {column_type}')
|
|
383
|
+
if self._df is None:
|
|
384
|
+
column_type = MYSQL_DATA_TYPE.TEXT
|
|
385
|
+
else:
|
|
386
|
+
column_type = get_mysql_data_type_from_series(self._df.iloc[:, i])
|
|
387
|
+
|
|
388
|
+
sqlalchemy_type = type_mapping.get(column_type, sqlalchemy_types.TEXT)
|
|
389
|
+
|
|
390
|
+
columns.append(
|
|
391
|
+
TableColumn(
|
|
392
|
+
name=column.alias,
|
|
393
|
+
type=sqlalchemy_type
|
|
394
|
+
)
|
|
395
|
+
)
|
|
396
|
+
return columns
|
|
397
|
+
|
|
265
398
|
def to_lists(self, json_types=False):
|
|
266
399
|
"""
|
|
267
400
|
:param type_cast: cast numpy types
|
|
@@ -281,7 +414,7 @@ class ResultSet:
|
|
|
281
414
|
return df.to_records(index=False).tolist()
|
|
282
415
|
|
|
283
416
|
# slower but keep timestamp type
|
|
284
|
-
df = self._df.replace({np.nan: None})
|
|
417
|
+
df = self._df.replace({np.nan: None}) # TODO rework
|
|
285
418
|
return df.to_dict('split')['data']
|
|
286
419
|
|
|
287
420
|
def get_column_values(self, col_idx):
|
|
@@ -252,7 +252,7 @@ class SQLQuery:
|
|
|
252
252
|
# return query info
|
|
253
253
|
# columns in upper case
|
|
254
254
|
rec = {k.upper(): v for k, v in self.run_query.get_info().items()}
|
|
255
|
-
self.fetched_data = ResultSet
|
|
255
|
+
self.fetched_data = ResultSet.from_df(pd.DataFrame([rec]))
|
|
256
256
|
self.columns_list = self.fetched_data.columns
|
|
257
257
|
return
|
|
258
258
|
self.run_query.mark_as_run()
|
|
@@ -91,21 +91,18 @@ class ApplyPredictorRowStepCall(ApplyPredictorBaseCall):
|
|
|
91
91
|
|
|
92
92
|
table_name = get_preditor_alias(step, self.context.get('database'))
|
|
93
93
|
|
|
94
|
-
result = ResultSet()
|
|
95
|
-
result.is_prediction = True
|
|
96
94
|
if len(predictions) == 0:
|
|
97
95
|
columns_names = project_datanode.get_table_columns_names(predictor_name)
|
|
98
96
|
predictions = pd.DataFrame([], columns=columns_names)
|
|
99
97
|
|
|
100
|
-
|
|
101
|
-
predictions,
|
|
98
|
+
return ResultSet.from_df(
|
|
99
|
+
df=predictions,
|
|
102
100
|
database=table_name[0],
|
|
103
101
|
table_name=table_name[1],
|
|
104
|
-
table_alias=table_name[2]
|
|
102
|
+
table_alias=table_name[2],
|
|
103
|
+
is_prediction=True
|
|
105
104
|
)
|
|
106
105
|
|
|
107
|
-
return result
|
|
108
|
-
|
|
109
106
|
|
|
110
107
|
class ApplyPredictorStepCall(ApplyPredictorBaseCall):
|
|
111
108
|
|
|
@@ -164,15 +161,14 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
|
|
|
164
161
|
params['force_ts_infer'] = True
|
|
165
162
|
_mdb_forecast_offset = None
|
|
166
163
|
|
|
167
|
-
data.add_column(Column('__mdb_forecast_offset'), _mdb_forecast_offset)
|
|
164
|
+
data.add_column(Column(name='__mdb_forecast_offset'), _mdb_forecast_offset)
|
|
168
165
|
|
|
169
166
|
table_name = get_preditor_alias(step, self.context['database'])
|
|
170
|
-
result = ResultSet()
|
|
171
|
-
result.is_prediction = True
|
|
172
167
|
|
|
173
168
|
project_datanode = self.session.datahub.get(project_name)
|
|
174
169
|
if len(data) == 0:
|
|
175
170
|
columns_names = project_datanode.get_table_columns_names(predictor_name) + ['__mindsdb_row_id']
|
|
171
|
+
result = ResultSet(is_prediction=True)
|
|
176
172
|
for column_name in columns_names:
|
|
177
173
|
result.add_column(Column(
|
|
178
174
|
name=column_name,
|
|
@@ -230,11 +226,12 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
|
|
|
230
226
|
pred_data = self.apply_ts_filter(pred_data, where_data, step, predictor_metadata)
|
|
231
227
|
predictions = pd.DataFrame(pred_data)
|
|
232
228
|
|
|
233
|
-
result.from_df(
|
|
229
|
+
result = ResultSet.from_df(
|
|
234
230
|
predictions,
|
|
235
231
|
database=table_name[0],
|
|
236
232
|
table_name=table_name[1],
|
|
237
|
-
table_alias=table_name[2]
|
|
233
|
+
table_alias=table_name[2],
|
|
234
|
+
is_prediction=True
|
|
238
235
|
)
|
|
239
236
|
|
|
240
237
|
return result
|
|
@@ -7,11 +7,12 @@ from mindsdb_sql_parser.ast import (
|
|
|
7
7
|
BinaryOperation,
|
|
8
8
|
Tuple,
|
|
9
9
|
)
|
|
10
|
-
from mindsdb.api.executor.planner.steps import FetchDataframeStep
|
|
11
|
-
from mindsdb.integrations.utilities.query_traversal import query_traversal
|
|
12
10
|
|
|
11
|
+
from mindsdb.api.executor.planner.steps import FetchDataframeStep
|
|
12
|
+
from mindsdb.api.executor.datahub.classes.response import DataHubResponse
|
|
13
13
|
from mindsdb.api.executor.sql_query.result_set import ResultSet
|
|
14
14
|
from mindsdb.api.executor.exceptions import UnknownError
|
|
15
|
+
from mindsdb.integrations.utilities.query_traversal import query_traversal
|
|
15
16
|
from mindsdb.interfaces.query_context.context_controller import query_context_controller
|
|
16
17
|
|
|
17
18
|
from .base import BaseStepCall
|
|
@@ -89,7 +90,7 @@ class FetchDataframeStepCall(BaseStepCall):
|
|
|
89
90
|
table_alias = (self.context.get('database'), 'result', 'result')
|
|
90
91
|
|
|
91
92
|
# fetch raw_query
|
|
92
|
-
response = dn.query(
|
|
93
|
+
response: DataHubResponse = dn.query(
|
|
93
94
|
native_query=step.raw_query,
|
|
94
95
|
session=self.session
|
|
95
96
|
)
|
|
@@ -105,7 +106,7 @@ class FetchDataframeStepCall(BaseStepCall):
|
|
|
105
106
|
|
|
106
107
|
query, context_callback = query_context_controller.handle_db_context_vars(query, dn, self.session)
|
|
107
108
|
|
|
108
|
-
response = dn.query(
|
|
109
|
+
response: DataHubResponse = dn.query(
|
|
109
110
|
query=query,
|
|
110
111
|
session=self.session
|
|
111
112
|
)
|
|
@@ -114,13 +115,10 @@ class FetchDataframeStepCall(BaseStepCall):
|
|
|
114
115
|
if context_callback:
|
|
115
116
|
context_callback(df, response.columns)
|
|
116
117
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
result.from_df(
|
|
118
|
+
return ResultSet.from_df(
|
|
120
119
|
df,
|
|
121
120
|
table_name=table_alias[1],
|
|
122
121
|
table_alias=table_alias[2],
|
|
123
|
-
database=table_alias[0]
|
|
122
|
+
database=table_alias[0],
|
|
123
|
+
mysql_types=response.mysql_types
|
|
124
124
|
)
|
|
125
|
-
|
|
126
|
-
return result
|
|
@@ -8,7 +8,7 @@ from mindsdb.integrations.utilities.query_traversal import query_traversal
|
|
|
8
8
|
from mindsdb.interfaces.query_context.context_controller import RunningQuery
|
|
9
9
|
from mindsdb.api.executor.sql_query.result_set import ResultSet
|
|
10
10
|
from mindsdb.utilities import log
|
|
11
|
-
from mindsdb.utilities.config import
|
|
11
|
+
from mindsdb.utilities.config import config
|
|
12
12
|
from mindsdb.utilities.partitioning import get_max_thread_count, split_data_frame
|
|
13
13
|
from mindsdb.api.executor.sql_query.steps.fetch_dataframe import get_table_alias, get_fill_param_fnc
|
|
14
14
|
from mindsdb.utilities.context_executor import ContextThreadPoolExecutor
|
|
@@ -64,8 +64,6 @@ class FetchDataframePartitionCall(BaseStepCall):
|
|
|
64
64
|
self.current_step_num = step.step_num
|
|
65
65
|
self.substeps = step.steps
|
|
66
66
|
|
|
67
|
-
config = Config()
|
|
68
|
-
|
|
69
67
|
# ml task queue enabled?
|
|
70
68
|
use_threads, thread_count = False, None
|
|
71
69
|
if config['ml_task_queue']['type'] == 'redis':
|
|
@@ -95,21 +93,8 @@ class FetchDataframePartitionCall(BaseStepCall):
|
|
|
95
93
|
"""
|
|
96
94
|
|
|
97
95
|
results = []
|
|
98
|
-
while True:
|
|
99
|
-
|
|
100
|
-
# fetch batch
|
|
101
|
-
query2 = run_query.get_partition_query(self.current_step_num, query)
|
|
102
|
-
response = self.dn.query(
|
|
103
|
-
query=query2,
|
|
104
|
-
session=self.session
|
|
105
|
-
)
|
|
106
|
-
df = response.data_frame
|
|
107
|
-
|
|
108
|
-
if df is None or len(df) == 0:
|
|
109
|
-
break
|
|
110
96
|
|
|
111
|
-
|
|
112
|
-
max_track_value = run_query.get_max_track_value(df)
|
|
97
|
+
for df in run_query.get_partitions(self.dn, self, query):
|
|
113
98
|
try:
|
|
114
99
|
sub_data = self.exec_sub_steps(df)
|
|
115
100
|
results.append(sub_data)
|
|
@@ -119,8 +104,6 @@ class FetchDataframePartitionCall(BaseStepCall):
|
|
|
119
104
|
else:
|
|
120
105
|
raise e
|
|
121
106
|
|
|
122
|
-
run_query.set_progress(df, max_track_value)
|
|
123
|
-
|
|
124
107
|
return self.concat_results(results)
|
|
125
108
|
|
|
126
109
|
def concat_results(self, results: List[ResultSet]) -> ResultSet:
|
|
@@ -135,7 +118,7 @@ class FetchDataframePartitionCall(BaseStepCall):
|
|
|
135
118
|
|
|
136
119
|
data = ResultSet()
|
|
137
120
|
if len(df_list) > 0:
|
|
138
|
-
data.from_df_cols(pd.concat(df_list), col_names)
|
|
121
|
+
data = ResultSet.from_df_cols(pd.concat(df_list), col_names)
|
|
139
122
|
|
|
140
123
|
return data
|
|
141
124
|
|
|
@@ -147,10 +130,7 @@ class FetchDataframePartitionCall(BaseStepCall):
|
|
|
147
130
|
- substep are executed using result of previos step (like it is all fetched data is available)
|
|
148
131
|
- the final result is returned and used outside to concatenate with results of other's batches
|
|
149
132
|
"""
|
|
150
|
-
|
|
151
|
-
input_data = ResultSet()
|
|
152
|
-
|
|
153
|
-
input_data.from_df(
|
|
133
|
+
input_data = ResultSet.from_df(
|
|
154
134
|
df,
|
|
155
135
|
table_name=self.table_alias[1],
|
|
156
136
|
table_alias=self.table_alias[2],
|
|
@@ -191,22 +171,7 @@ class FetchDataframePartitionCall(BaseStepCall):
|
|
|
191
171
|
|
|
192
172
|
with ContextThreadPoolExecutor(max_workers=thread_count) as executor:
|
|
193
173
|
|
|
194
|
-
|
|
195
|
-
# fetch batch
|
|
196
|
-
query2 = run_query.get_partition_query(self.current_step_num, query)
|
|
197
|
-
response = self.dn.query(
|
|
198
|
-
query=query2,
|
|
199
|
-
session=self.session
|
|
200
|
-
)
|
|
201
|
-
df = response.data_frame
|
|
202
|
-
|
|
203
|
-
if df is None or len(df) == 0:
|
|
204
|
-
# TODO detect circles: data handler ignores condition and output is repeated
|
|
205
|
-
|
|
206
|
-
# exit & stop workers
|
|
207
|
-
break
|
|
208
|
-
|
|
209
|
-
max_track_value = run_query.get_max_track_value(df)
|
|
174
|
+
for df in run_query.get_partitions(self.dn, self, query):
|
|
210
175
|
|
|
211
176
|
# split into chunks and send to workers
|
|
212
177
|
futures = []
|
|
@@ -225,9 +190,5 @@ class FetchDataframePartitionCall(BaseStepCall):
|
|
|
225
190
|
if self.sql_query.stop_event is not None and self.sql_query.stop_event.is_set():
|
|
226
191
|
executor.shutdown()
|
|
227
192
|
raise RuntimeError('Query is interrupted')
|
|
228
|
-
# TODO
|
|
229
|
-
# 1. get next batch without updating track_value:
|
|
230
|
-
# it allows to keep queue_in filled with data between fetching batches
|
|
231
|
-
run_query.set_progress(df, max_track_value)
|
|
232
193
|
|
|
233
194
|
return self.concat_results(results)
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
from mindsdb_sql_parser.ast import (
|
|
2
2
|
Identifier,
|
|
3
3
|
)
|
|
4
|
+
|
|
4
5
|
from mindsdb.api.executor.planner.steps import (
|
|
5
6
|
SaveToTable,
|
|
6
7
|
InsertToTable,
|
|
7
8
|
CreateTableStep
|
|
8
9
|
)
|
|
9
|
-
|
|
10
10
|
from mindsdb.api.executor.sql_query.result_set import ResultSet, Column
|
|
11
11
|
from mindsdb.api.executor.exceptions import (
|
|
12
12
|
NotSupportedYet,
|
|
13
13
|
LogicError
|
|
14
14
|
)
|
|
15
|
+
from mindsdb.integrations.libs.response import INF_SCHEMA_COLUMNS_NAMES
|
|
15
16
|
|
|
16
17
|
from .base import BaseStepCall
|
|
17
18
|
|
|
@@ -30,12 +31,32 @@ class InsertToTableCall(BaseStepCall):
|
|
|
30
31
|
if step.is_replace:
|
|
31
32
|
is_replace = True
|
|
32
33
|
|
|
34
|
+
if len(step.table.parts) > 1:
|
|
35
|
+
integration_name = step.table.parts[0]
|
|
36
|
+
table_name = Identifier(parts=step.table.parts[1:])
|
|
37
|
+
else:
|
|
38
|
+
integration_name = self.context['database']
|
|
39
|
+
table_name = step.table
|
|
40
|
+
|
|
41
|
+
dn = self.session.datahub.get(integration_name)
|
|
42
|
+
|
|
43
|
+
if hasattr(dn, 'create_table') is False:
|
|
44
|
+
raise NotSupportedYet(f"Creating table in '{integration_name}' is not supported")
|
|
45
|
+
|
|
33
46
|
if step.dataframe is not None:
|
|
34
47
|
data = self.steps_data[step.dataframe.result.step_num]
|
|
35
48
|
elif step.query is not None:
|
|
36
49
|
data = ResultSet()
|
|
37
|
-
|
|
38
|
-
|
|
50
|
+
if step.query.columns is None:
|
|
51
|
+
# Is query like: INSERT INTO table VALUES (...)
|
|
52
|
+
table_columns_df = dn.get_table_columns_df(str(table_name))
|
|
53
|
+
columns_names = table_columns_df[INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME].to_list()
|
|
54
|
+
for column_name in columns_names:
|
|
55
|
+
data.add_column(Column(name=column_name))
|
|
56
|
+
else:
|
|
57
|
+
# Is query like: INSERT INTO table (column_name, ...) VALUES (...)
|
|
58
|
+
for col in step.query.columns:
|
|
59
|
+
data.add_column(Column(name=col.name))
|
|
39
60
|
|
|
40
61
|
records = []
|
|
41
62
|
for row in step.query.values:
|
|
@@ -53,18 +74,6 @@ class InsertToTableCall(BaseStepCall):
|
|
|
53
74
|
else:
|
|
54
75
|
raise LogicError(f'Data not found for insert: {step}')
|
|
55
76
|
|
|
56
|
-
if len(step.table.parts) > 1:
|
|
57
|
-
integration_name = step.table.parts[0]
|
|
58
|
-
table_name = Identifier(parts=step.table.parts[1:])
|
|
59
|
-
else:
|
|
60
|
-
integration_name = self.context['database']
|
|
61
|
-
table_name = step.table
|
|
62
|
-
|
|
63
|
-
dn = self.session.datahub.get(integration_name)
|
|
64
|
-
|
|
65
|
-
if hasattr(dn, 'create_table') is False:
|
|
66
|
-
raise NotSupportedYet(f"Creating table in '{integration_name}' is not supported")
|
|
67
|
-
|
|
68
77
|
# del 'service' columns
|
|
69
78
|
for col in data.find_columns('__mindsdb_row_id'):
|
|
70
79
|
data.del_column(col)
|
|
@@ -101,7 +101,7 @@ class JoinStepCall(BaseStepCall):
|
|
|
101
101
|
resp_df.replace({np.nan: None}, inplace=True)
|
|
102
102
|
|
|
103
103
|
names_a.update(names_b)
|
|
104
|
-
data = ResultSet
|
|
104
|
+
data = ResultSet.from_df_cols(df=resp_df, columns_dict=names_a)
|
|
105
105
|
|
|
106
106
|
for col in data.find_columns('__mindsdb_row_id'):
|
|
107
107
|
data.del_column(col)
|
|
@@ -55,7 +55,7 @@ class SubSelectStepCall(BaseStepCall):
|
|
|
55
55
|
|
|
56
56
|
for col_name in query_cols:
|
|
57
57
|
if col_name not in result_cols:
|
|
58
|
-
result.add_column(Column(col_name))
|
|
58
|
+
result.add_column(Column(name=col_name))
|
|
59
59
|
|
|
60
60
|
# inject previous step values
|
|
61
61
|
if isinstance(query, Select):
|
|
@@ -69,12 +69,10 @@ class SubSelectStepCall(BaseStepCall):
|
|
|
69
69
|
df = result.to_df()
|
|
70
70
|
res = query_df(df, query, session=self.session)
|
|
71
71
|
|
|
72
|
-
result2 = ResultSet()
|
|
73
72
|
# get database from first column
|
|
74
73
|
database = result.columns[0].database
|
|
75
|
-
result2.from_df(res, database, table_name)
|
|
76
74
|
|
|
77
|
-
return
|
|
75
|
+
return ResultSet.from_df(res, database, table_name)
|
|
78
76
|
|
|
79
77
|
|
|
80
78
|
class QueryStepCall(BaseStepCall):
|
|
@@ -86,7 +84,7 @@ class QueryStepCall(BaseStepCall):
|
|
|
86
84
|
|
|
87
85
|
if step.from_table is not None:
|
|
88
86
|
if isinstance(step.from_table, pd.DataFrame):
|
|
89
|
-
result_set = ResultSet
|
|
87
|
+
result_set = ResultSet.from_df(step.from_table)
|
|
90
88
|
else:
|
|
91
89
|
result_set = self.steps_data[step.from_table.step_num]
|
|
92
90
|
else:
|
|
@@ -208,6 +206,4 @@ class QueryStepCall(BaseStepCall):
|
|
|
208
206
|
query.from_table = Identifier('df_table')
|
|
209
207
|
res = query_df(df, query, session=self.session)
|
|
210
208
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
return data
|
|
209
|
+
return ResultSet.from_df_cols(df=res, columns_dict=col_names, strict=False)
|