MindsDB 25.5.3.0__py3-none-any.whl → 25.5.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of MindsDB might be problematic. Click here for more details.

Files changed (310) hide show
  1. mindsdb/__about__.py +1 -1
  2. mindsdb/__main__.py +127 -79
  3. mindsdb/api/a2a/__init__.py +0 -0
  4. mindsdb/api/a2a/__main__.py +114 -0
  5. mindsdb/api/a2a/a2a_client.py +439 -0
  6. mindsdb/api/a2a/agent.py +308 -0
  7. mindsdb/api/a2a/common/__init__.py +0 -0
  8. mindsdb/api/a2a/common/client/__init__.py +4 -0
  9. mindsdb/api/a2a/common/client/card_resolver.py +21 -0
  10. mindsdb/api/a2a/common/client/client.py +86 -0
  11. mindsdb/api/a2a/common/server/__init__.py +4 -0
  12. mindsdb/api/a2a/common/server/server.py +164 -0
  13. mindsdb/api/a2a/common/server/task_manager.py +287 -0
  14. mindsdb/api/a2a/common/server/utils.py +28 -0
  15. mindsdb/api/a2a/common/types.py +365 -0
  16. mindsdb/api/a2a/constants.py +9 -0
  17. mindsdb/api/a2a/run_a2a.py +129 -0
  18. mindsdb/api/a2a/task_manager.py +594 -0
  19. mindsdb/api/executor/command_executor.py +47 -27
  20. mindsdb/api/executor/datahub/classes/response.py +5 -2
  21. mindsdb/api/executor/datahub/datanodes/integration_datanode.py +39 -72
  22. mindsdb/api/executor/planner/query_planner.py +10 -1
  23. mindsdb/api/executor/sql_query/result_set.py +185 -52
  24. mindsdb/api/executor/sql_query/sql_query.py +1 -1
  25. mindsdb/api/executor/sql_query/steps/apply_predictor_step.py +9 -12
  26. mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +8 -10
  27. mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py +5 -44
  28. mindsdb/api/executor/sql_query/steps/insert_step.py +24 -15
  29. mindsdb/api/executor/sql_query/steps/join_step.py +1 -1
  30. mindsdb/api/executor/sql_query/steps/project_step.py +1 -1
  31. mindsdb/api/executor/sql_query/steps/sql_steps.py +1 -1
  32. mindsdb/api/executor/sql_query/steps/subselect_step.py +4 -8
  33. mindsdb/api/executor/sql_query/steps/union_step.py +1 -3
  34. mindsdb/api/http/initialize.py +99 -83
  35. mindsdb/api/http/namespaces/analysis.py +3 -3
  36. mindsdb/api/http/namespaces/file.py +8 -2
  37. mindsdb/api/http/namespaces/sql.py +13 -27
  38. mindsdb/api/mcp/start.py +42 -5
  39. mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py +0 -1
  40. mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/binary_resultset_row_package.py +52 -19
  41. mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +8 -10
  42. mindsdb/api/mysql/mysql_proxy/libs/constants/mysql.py +54 -38
  43. mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +82 -115
  44. mindsdb/api/mysql/mysql_proxy/utilities/dump.py +351 -0
  45. mindsdb/api/postgres/postgres_proxy/executor/executor.py +1 -1
  46. mindsdb/api/postgres/postgres_proxy/postgres_proxy.py +5 -6
  47. mindsdb/integrations/handlers/altibase_handler/altibase_handler.py +26 -27
  48. mindsdb/integrations/handlers/altibase_handler/connection_args.py +13 -13
  49. mindsdb/integrations/handlers/altibase_handler/tests/test_altibase_handler.py +8 -8
  50. mindsdb/integrations/handlers/altibase_handler/tests/test_altibase_handler_dsn.py +13 -13
  51. mindsdb/integrations/handlers/anthropic_handler/__init__.py +2 -2
  52. mindsdb/integrations/handlers/anthropic_handler/anthropic_handler.py +1 -3
  53. mindsdb/integrations/handlers/aurora_handler/aurora_handler.py +1 -0
  54. mindsdb/integrations/handlers/autosklearn_handler/autosklearn_handler.py +1 -1
  55. mindsdb/integrations/handlers/autosklearn_handler/config.py +0 -1
  56. mindsdb/integrations/handlers/bigquery_handler/bigquery_handler.py +1 -1
  57. mindsdb/integrations/handlers/bigquery_handler/tests/test_bigquery_handler.py +1 -1
  58. mindsdb/integrations/handlers/binance_handler/binance_handler.py +1 -0
  59. mindsdb/integrations/handlers/binance_handler/binance_tables.py +3 -4
  60. mindsdb/integrations/handlers/byom_handler/__init__.py +0 -1
  61. mindsdb/integrations/handlers/ckan_handler/ckan_handler.py +3 -0
  62. mindsdb/integrations/handlers/clickhouse_handler/__init__.py +1 -1
  63. mindsdb/integrations/handlers/cloud_spanner_handler/tests/test_cloud_spanner_handler.py +0 -2
  64. mindsdb/integrations/handlers/cloud_sql_handler/cloud_sql_handler.py +0 -1
  65. mindsdb/integrations/handlers/cohere_handler/__init__.py +1 -1
  66. mindsdb/integrations/handlers/cohere_handler/cohere_handler.py +11 -13
  67. mindsdb/integrations/handlers/confluence_handler/confluence_tables.py +6 -0
  68. mindsdb/integrations/handlers/databend_handler/connection_args.py +1 -1
  69. mindsdb/integrations/handlers/databend_handler/databend_handler.py +4 -4
  70. mindsdb/integrations/handlers/databend_handler/tests/__init__.py +0 -1
  71. mindsdb/integrations/handlers/databend_handler/tests/test_databend_handler.py +1 -1
  72. mindsdb/integrations/handlers/derby_handler/connection_args.py +1 -1
  73. mindsdb/integrations/handlers/derby_handler/derby_handler.py +14 -22
  74. mindsdb/integrations/handlers/derby_handler/tests/test_derby_handler.py +6 -6
  75. mindsdb/integrations/handlers/discord_handler/discord_handler.py +5 -5
  76. mindsdb/integrations/handlers/discord_handler/discord_tables.py +3 -3
  77. mindsdb/integrations/handlers/discord_handler/tests/test_discord.py +5 -3
  78. mindsdb/integrations/handlers/dockerhub_handler/dockerhub.py +3 -3
  79. mindsdb/integrations/handlers/dockerhub_handler/dockerhub_handler.py +2 -2
  80. mindsdb/integrations/handlers/dockerhub_handler/dockerhub_tables.py +57 -54
  81. mindsdb/integrations/handlers/dremio_handler/__init__.py +2 -2
  82. mindsdb/integrations/handlers/druid_handler/__init__.py +1 -1
  83. mindsdb/integrations/handlers/druid_handler/druid_handler.py +2 -2
  84. mindsdb/integrations/handlers/edgelessdb_handler/tests/test_edgelessdb_handler.py +9 -9
  85. mindsdb/integrations/handlers/email_handler/email_client.py +1 -1
  86. mindsdb/integrations/handlers/email_handler/email_ingestor.py +1 -1
  87. mindsdb/integrations/handlers/email_handler/email_tables.py +0 -1
  88. mindsdb/integrations/handlers/email_handler/settings.py +0 -1
  89. mindsdb/integrations/handlers/eventstoredb_handler/eventstoredb_handler.py +2 -1
  90. mindsdb/integrations/handlers/firebird_handler/firebird_handler.py +1 -1
  91. mindsdb/integrations/handlers/flaml_handler/flaml_handler.py +9 -9
  92. mindsdb/integrations/handlers/frappe_handler/frappe_client.py +5 -5
  93. mindsdb/integrations/handlers/frappe_handler/frappe_handler.py +6 -5
  94. mindsdb/integrations/handlers/frappe_handler/frappe_tables.py +2 -2
  95. mindsdb/integrations/handlers/github_handler/connection_args.py +2 -2
  96. mindsdb/integrations/handlers/github_handler/github_handler.py +1 -8
  97. mindsdb/integrations/handlers/github_handler/github_tables.py +13 -24
  98. mindsdb/integrations/handlers/gitlab_handler/gitlab_handler.py +2 -1
  99. mindsdb/integrations/handlers/gitlab_handler/gitlab_tables.py +1 -4
  100. mindsdb/integrations/handlers/gmail_handler/gmail_handler.py +6 -13
  101. mindsdb/integrations/handlers/google_books_handler/google_books_handler.py +2 -1
  102. mindsdb/integrations/handlers/google_books_handler/google_books_tables.py +0 -3
  103. mindsdb/integrations/handlers/google_calendar_handler/google_calendar_handler.py +4 -4
  104. mindsdb/integrations/handlers/google_calendar_handler/google_calendar_tables.py +2 -6
  105. mindsdb/integrations/handlers/google_content_shopping_handler/google_content_shopping_handler.py +3 -2
  106. mindsdb/integrations/handlers/google_content_shopping_handler/google_content_shopping_tables.py +0 -3
  107. mindsdb/integrations/handlers/google_fit_handler/google_fit_handler.py +10 -12
  108. mindsdb/integrations/handlers/google_fit_handler/google_fit_tables.py +11 -13
  109. mindsdb/integrations/handlers/google_search_handler/google_search_handler.py +2 -1
  110. mindsdb/integrations/handlers/google_search_handler/google_search_tables.py +0 -3
  111. mindsdb/integrations/handlers/groq_handler/__init__.py +3 -3
  112. mindsdb/integrations/handlers/hackernews_handler/hn_handler.py +5 -7
  113. mindsdb/integrations/handlers/hackernews_handler/hn_table.py +6 -7
  114. mindsdb/integrations/handlers/hive_handler/tests/test_hive_handler.py +1 -1
  115. mindsdb/integrations/handlers/hsqldb_handler/connection_args.py +6 -6
  116. mindsdb/integrations/handlers/hsqldb_handler/hsqldb_handler.py +4 -3
  117. mindsdb/integrations/handlers/huggingface_api_handler/exceptions.py +1 -1
  118. mindsdb/integrations/handlers/huggingface_api_handler/huggingface_api_handler.py +1 -8
  119. mindsdb/integrations/handlers/huggingface_handler/huggingface_handler.py +6 -6
  120. mindsdb/integrations/handlers/huggingface_handler/requirements.txt +1 -1
  121. mindsdb/integrations/handlers/huggingface_handler/requirements_cpu.txt +1 -1
  122. mindsdb/integrations/handlers/ignite_handler/ignite_handler.py +2 -1
  123. mindsdb/integrations/handlers/impala_handler/impala_handler.py +9 -12
  124. mindsdb/integrations/handlers/impala_handler/tests/test_impala_handler.py +11 -11
  125. mindsdb/integrations/handlers/influxdb_handler/influxdb_handler.py +10 -13
  126. mindsdb/integrations/handlers/influxdb_handler/influxdb_tables.py +20 -20
  127. mindsdb/integrations/handlers/informix_handler/__about__.py +8 -8
  128. mindsdb/integrations/handlers/informix_handler/__init__.py +12 -5
  129. mindsdb/integrations/handlers/informix_handler/informix_handler.py +99 -133
  130. mindsdb/integrations/handlers/informix_handler/tests/test_informix_handler.py +13 -11
  131. mindsdb/integrations/handlers/ingres_handler/__about__.py +0 -1
  132. mindsdb/integrations/handlers/ingres_handler/ingres_handler.py +1 -0
  133. mindsdb/integrations/handlers/jira_handler/jira_handler.py +4 -4
  134. mindsdb/integrations/handlers/jira_handler/jira_tables.py +9 -9
  135. mindsdb/integrations/handlers/kinetica_handler/__init__.py +0 -1
  136. mindsdb/integrations/handlers/langchain_handler/langchain_handler.py +4 -4
  137. mindsdb/integrations/handlers/langchain_handler/tools.py +9 -10
  138. mindsdb/integrations/handlers/leonardoai_handler/__init__.py +1 -1
  139. mindsdb/integrations/handlers/lightwood_handler/functions.py +2 -2
  140. mindsdb/integrations/handlers/lightwood_handler/lightwood_handler.py +0 -1
  141. mindsdb/integrations/handlers/lightwood_handler/tests/test_lightwood_handler.py +11 -11
  142. mindsdb/integrations/handlers/llama_index_handler/llama_index_handler.py +4 -4
  143. mindsdb/integrations/handlers/llama_index_handler/settings.py +10 -9
  144. mindsdb/integrations/handlers/materialize_handler/tests/test_materialize_handler.py +8 -10
  145. mindsdb/integrations/handlers/matrixone_handler/matrixone_handler.py +4 -4
  146. mindsdb/integrations/handlers/matrixone_handler/tests/test_matrixone_handler.py +8 -9
  147. mindsdb/integrations/handlers/maxdb_handler/connection_args.py +25 -25
  148. mindsdb/integrations/handlers/maxdb_handler/maxdb_handler.py +1 -0
  149. mindsdb/integrations/handlers/mediawiki_handler/mediawiki_handler.py +3 -2
  150. mindsdb/integrations/handlers/mediawiki_handler/mediawiki_tables.py +1 -1
  151. mindsdb/integrations/handlers/mendeley_handler/__about__.py +1 -1
  152. mindsdb/integrations/handlers/mendeley_handler/__init__.py +2 -2
  153. mindsdb/integrations/handlers/mendeley_handler/mendeley_handler.py +48 -56
  154. mindsdb/integrations/handlers/mendeley_handler/mendeley_tables.py +24 -29
  155. mindsdb/integrations/handlers/mendeley_handler/tests/test_mendeley_handler.py +19 -17
  156. mindsdb/integrations/handlers/merlion_handler/merlion_handler.py +5 -4
  157. mindsdb/integrations/handlers/minds_endpoint_handler/__init__.py +3 -3
  158. mindsdb/integrations/handlers/mlflow_handler/mlflow_handler.py +58 -36
  159. mindsdb/integrations/handlers/monetdb_handler/__about__.py +8 -8
  160. mindsdb/integrations/handlers/monetdb_handler/__init__.py +15 -5
  161. mindsdb/integrations/handlers/monetdb_handler/connection_args.py +17 -18
  162. mindsdb/integrations/handlers/monetdb_handler/monetdb_handler.py +40 -57
  163. mindsdb/integrations/handlers/monetdb_handler/tests/test_monetdb_handler.py +7 -8
  164. mindsdb/integrations/handlers/monetdb_handler/utils/monet_get_id.py +13 -14
  165. mindsdb/integrations/handlers/monkeylearn_handler/__about__.py +1 -1
  166. mindsdb/integrations/handlers/monkeylearn_handler/__init__.py +1 -1
  167. mindsdb/integrations/handlers/monkeylearn_handler/monkeylearn_handler.py +2 -5
  168. mindsdb/integrations/handlers/ms_one_drive_handler/ms_graph_api_one_drive_client.py +1 -0
  169. mindsdb/integrations/handlers/ms_one_drive_handler/ms_one_drive_handler.py +1 -1
  170. mindsdb/integrations/handlers/ms_teams_handler/ms_graph_api_teams_client.py +23 -23
  171. mindsdb/integrations/handlers/ms_teams_handler/ms_teams_handler.py +3 -3
  172. mindsdb/integrations/handlers/ms_teams_handler/ms_teams_tables.py +10 -5
  173. mindsdb/integrations/handlers/mssql_handler/mssql_handler.py +73 -8
  174. mindsdb/integrations/handlers/mysql_handler/__about__.py +8 -8
  175. mindsdb/integrations/handlers/mysql_handler/__init__.py +15 -5
  176. mindsdb/integrations/handlers/mysql_handler/connection_args.py +43 -47
  177. mindsdb/integrations/handlers/mysql_handler/mysql_handler.py +101 -34
  178. mindsdb/integrations/handlers/mysql_handler/settings.py +15 -13
  179. mindsdb/integrations/handlers/neuralforecast_handler/neuralforecast_handler.py +1 -1
  180. mindsdb/integrations/handlers/newsapi_handler/newsapi_handler.py +1 -1
  181. mindsdb/integrations/handlers/newsapi_handler/tests/test_newsapi_handler.py +4 -4
  182. mindsdb/integrations/handlers/nuo_jdbc_handler/connection_args.py +2 -2
  183. mindsdb/integrations/handlers/nuo_jdbc_handler/nuo_jdbc_handler.py +28 -36
  184. mindsdb/integrations/handlers/nuo_jdbc_handler/tests/test_nuo_handler.py +5 -5
  185. mindsdb/integrations/handlers/oceanbase_handler/oceanbase_handler.py +0 -1
  186. mindsdb/integrations/handlers/oceanbase_handler/tests/test_oceanbase_handler.py +8 -10
  187. mindsdb/integrations/handlers/ollama_handler/ollama_handler.py +3 -3
  188. mindsdb/integrations/handlers/opengauss_handler/tests/test_opengauss_handler.py +1 -2
  189. mindsdb/integrations/handlers/openstreetmap_handler/__init__.py +7 -7
  190. mindsdb/integrations/handlers/oracle_handler/connection_args.py +6 -0
  191. mindsdb/integrations/handlers/oracle_handler/oracle_handler.py +77 -11
  192. mindsdb/integrations/handlers/orioledb_handler/tests/test_orioledb_handler.py +8 -10
  193. mindsdb/integrations/handlers/palm_handler/__about__.py +1 -1
  194. mindsdb/integrations/handlers/palm_handler/__init__.py +1 -1
  195. mindsdb/integrations/handlers/palm_handler/palm_handler.py +1 -3
  196. mindsdb/integrations/handlers/paypal_handler/paypal_handler.py +2 -2
  197. mindsdb/integrations/handlers/paypal_handler/paypal_tables.py +15 -14
  198. mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py +53 -10
  199. mindsdb/integrations/handlers/phoenix_handler/__init__.py +1 -1
  200. mindsdb/integrations/handlers/phoenix_handler/phoenix_handler.py +1 -0
  201. mindsdb/integrations/handlers/pinot_handler/__init__.py +1 -1
  202. mindsdb/integrations/handlers/pinot_handler/pinot_handler.py +3 -2
  203. mindsdb/integrations/handlers/plaid_handler/plaid_handler.py +13 -13
  204. mindsdb/integrations/handlers/plaid_handler/plaid_tables.py +10 -12
  205. mindsdb/integrations/handlers/plaid_handler/utils.py +4 -6
  206. mindsdb/integrations/handlers/planetscale_handler/planetscale_handler.py +1 -4
  207. mindsdb/integrations/handlers/portkey_handler/__init__.py +2 -2
  208. mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +105 -24
  209. mindsdb/integrations/handlers/postgres_handler/tests/test_postgres_handler.py +11 -6
  210. mindsdb/integrations/handlers/questdb_handler/questdb_handler.py +1 -2
  211. mindsdb/integrations/handlers/questdb_handler/tests/test_questdb_handler.py +2 -3
  212. mindsdb/integrations/handlers/quickbooks_handler/quickbooks_handler.py +6 -8
  213. mindsdb/integrations/handlers/quickbooks_handler/quickbooks_table.py +10 -10
  214. mindsdb/integrations/handlers/rag_handler/ingest.py +2 -2
  215. mindsdb/integrations/handlers/rag_handler/rag_handler.py +1 -1
  216. mindsdb/integrations/handlers/rag_handler/settings.py +1 -1
  217. mindsdb/integrations/handlers/reddit_handler/reddit_handler.py +2 -7
  218. mindsdb/integrations/handlers/reddit_handler/reddit_tables.py +2 -3
  219. mindsdb/integrations/handlers/replicate_handler/replicate_handler.py +6 -6
  220. mindsdb/integrations/handlers/rocket_chat_handler/rocket_chat_handler.py +1 -2
  221. mindsdb/integrations/handlers/rocket_chat_handler/rocket_chat_tables.py +0 -3
  222. mindsdb/integrations/handlers/rockset_handler/connection_args.py +14 -14
  223. mindsdb/integrations/handlers/rockset_handler/tests/test_rockset_handler.py +1 -0
  224. mindsdb/integrations/handlers/scylla_handler/scylla_handler.py +6 -5
  225. mindsdb/integrations/handlers/sendinblue_handler/sendinblue_handler.py +2 -1
  226. mindsdb/integrations/handlers/sendinblue_handler/sendinblue_tables.py +16 -16
  227. mindsdb/integrations/handlers/sentence_transformers_handler/__init__.py +1 -1
  228. mindsdb/integrations/handlers/sheets_handler/connection_args.py +1 -1
  229. mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +7 -6
  230. mindsdb/integrations/handlers/shopify_handler/shopify_tables.py +38 -41
  231. mindsdb/integrations/handlers/singlestore_handler/__about__.py +1 -1
  232. mindsdb/integrations/handlers/singlestore_handler/__init__.py +0 -1
  233. mindsdb/integrations/handlers/singlestore_handler/singlestore_handler.py +1 -0
  234. mindsdb/integrations/handlers/singlestore_handler/tests/test_singlestore_handler.py +3 -3
  235. mindsdb/integrations/handlers/slack_handler/__init__.py +3 -3
  236. mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +100 -6
  237. mindsdb/integrations/handlers/solr_handler/connection_args.py +7 -7
  238. mindsdb/integrations/handlers/solr_handler/solr_handler.py +2 -1
  239. mindsdb/integrations/handlers/solr_handler/tests/test_solr_handler.py +2 -1
  240. mindsdb/integrations/handlers/sqlany_handler/sqlany_handler.py +3 -2
  241. mindsdb/integrations/handlers/sqlite_handler/sqlite_handler.py +1 -0
  242. mindsdb/integrations/handlers/sqreamdb_handler/connection_args.py +1 -1
  243. mindsdb/integrations/handlers/sqreamdb_handler/sqreamdb_handler.py +15 -20
  244. mindsdb/integrations/handlers/sqreamdb_handler/tests/test_sqreamdb_handler.py +4 -4
  245. mindsdb/integrations/handlers/stabilityai_handler/__init__.py +1 -1
  246. mindsdb/integrations/handlers/starrocks_handler/starrocks_handler.py +0 -1
  247. mindsdb/integrations/handlers/starrocks_handler/tests/test_starrocks_handler.py +8 -10
  248. mindsdb/integrations/handlers/statsforecast_handler/statsforecast_handler.py +2 -2
  249. mindsdb/integrations/handlers/strava_handler/strava_handler.py +4 -8
  250. mindsdb/integrations/handlers/strava_handler/strava_tables.py +22 -30
  251. mindsdb/integrations/handlers/stripe_handler/stripe_handler.py +3 -2
  252. mindsdb/integrations/handlers/stripe_handler/stripe_tables.py +11 -27
  253. mindsdb/integrations/handlers/supabase_handler/tests/test_supabase_handler.py +1 -1
  254. mindsdb/integrations/handlers/surrealdb_handler/surrealdb_handler.py +4 -4
  255. mindsdb/integrations/handlers/tdengine_handler/tdengine_handler.py +25 -27
  256. mindsdb/integrations/handlers/tdengine_handler/tests/test_tdengine_handler.py +8 -8
  257. mindsdb/integrations/handlers/tidb_handler/tests/test_tidb_handler.py +1 -2
  258. mindsdb/integrations/handlers/timegpt_handler/timegpt_handler.py +5 -5
  259. mindsdb/integrations/handlers/tpot_handler/tpot_handler.py +21 -26
  260. mindsdb/integrations/handlers/trino_handler/trino_handler.py +14 -14
  261. mindsdb/integrations/handlers/twitter_handler/twitter_handler.py +2 -4
  262. mindsdb/integrations/handlers/unify_handler/tests/test_unify_handler.py +7 -8
  263. mindsdb/integrations/handlers/unify_handler/unify_handler.py +9 -9
  264. mindsdb/integrations/handlers/vertex_handler/vertex_client.py +1 -1
  265. mindsdb/integrations/handlers/vertica_handler/tests/test_vertica_handler.py +11 -11
  266. mindsdb/integrations/handlers/vertica_handler/vertica_handler.py +11 -14
  267. mindsdb/integrations/handlers/vitess_handler/tests/test_vitess_handler.py +9 -11
  268. mindsdb/integrations/handlers/vitess_handler/vitess_handler.py +0 -1
  269. mindsdb/integrations/handlers/web_handler/web_handler.py +1 -0
  270. mindsdb/integrations/handlers/whatsapp_handler/__init__.py +3 -3
  271. mindsdb/integrations/handlers/writer_handler/evaluate.py +1 -1
  272. mindsdb/integrations/handlers/writer_handler/settings.py +0 -1
  273. mindsdb/integrations/handlers/writer_handler/writer_handler.py +1 -0
  274. mindsdb/integrations/handlers/youtube_handler/youtube_handler.py +5 -5
  275. mindsdb/integrations/handlers/youtube_handler/youtube_tables.py +26 -27
  276. mindsdb/integrations/handlers/yugabyte_handler/tests/test_yugabyte_handler.py +3 -3
  277. mindsdb/integrations/handlers/yugabyte_handler/yugabyte_handler.py +0 -6
  278. mindsdb/integrations/libs/response.py +67 -52
  279. mindsdb/integrations/libs/vectordatabase_handler.py +6 -0
  280. mindsdb/integrations/utilities/handler_utils.py +15 -3
  281. mindsdb/integrations/utilities/handlers/api_utilities/__init__.py +0 -1
  282. mindsdb/integrations/utilities/handlers/auth_utilities/__init__.py +0 -2
  283. mindsdb/integrations/utilities/utils.py +3 -3
  284. mindsdb/interfaces/agents/agents_controller.py +164 -1
  285. mindsdb/interfaces/agents/constants.py +15 -0
  286. mindsdb/interfaces/agents/langchain_agent.py +16 -4
  287. mindsdb/interfaces/agents/mindsdb_database_agent.py +101 -2
  288. mindsdb/interfaces/knowledge_base/controller.py +25 -0
  289. mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py +13 -10
  290. mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +434 -0
  291. mindsdb/interfaces/knowledge_base/preprocessing/models.py +54 -0
  292. mindsdb/interfaces/query_context/context_controller.py +66 -10
  293. mindsdb/interfaces/skills/custom/text2sql/mindsdb_kb_tools.py +190 -0
  294. mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +92 -0
  295. mindsdb/interfaces/skills/skill_tool.py +202 -57
  296. mindsdb/interfaces/skills/sql_agent.py +205 -17
  297. mindsdb/interfaces/storage/fs.py +1 -0
  298. mindsdb/interfaces/variables/__init__.py +0 -0
  299. mindsdb/interfaces/variables/variables_controller.py +97 -0
  300. mindsdb/migrations/env.py +5 -7
  301. mindsdb/migrations/migrate.py +47 -7
  302. mindsdb/migrations/versions/2025-05-21_9f150e4f9a05_checkpoint_1.py +360 -0
  303. mindsdb/utilities/config.py +331 -219
  304. mindsdb/utilities/starters.py +13 -0
  305. {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/METADATA +641 -695
  306. {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/RECORD +309 -288
  307. {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/WHEEL +1 -1
  308. mindsdb/integrations/handlers/monkeylearn_handler/requirements.txt +0 -1
  309. {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/licenses/LICENSE +0 -0
  310. {mindsdb-25.5.3.0.dist-info → mindsdb-25.5.4.0.dist-info}/top_level.txt +0 -0
@@ -1,29 +1,70 @@
1
1
  import copy
2
- from typing import List, Optional
2
+ from dataclasses import dataclass, field, MISSING
3
3
 
4
4
  import numpy as np
5
5
  import pandas as pd
6
+ from pandas.api import types as pd_types
7
+ import sqlalchemy.types as sqlalchemy_types
6
8
 
9
+ from mindsdb_sql_parser.ast import TableColumn
10
+
11
+ from mindsdb.utilities import log
7
12
  from mindsdb.api.executor.exceptions import WrongArgumentError
13
+ from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
14
+
15
+
16
+ logger = log.getLogger(__name__)
17
+
8
18
 
19
+ def get_mysql_data_type_from_series(series: pd.Series, do_infer: bool = False) -> MYSQL_DATA_TYPE:
20
+ """Maps pandas Series data type to corresponding MySQL data type.
9
21
 
22
+ This function examines the dtype of a pandas Series and returns the appropriate
23
+ MySQL data type enum value. For object dtypes, it can optionally attempt to infer
24
+ a more specific type.
25
+
26
+ Args:
27
+ series (pd.Series): The pandas Series to determine the MySQL type for
28
+ do_infer (bool): If True and series has object dtype, attempt to infer a more specific type
29
+
30
+ Returns:
31
+ MYSQL_DATA_TYPE: The corresponding MySQL data type enum value
32
+ """
33
+ dtype = series.dtype
34
+ if pd_types.is_object_dtype(dtype) and do_infer is True:
35
+ dtype = series.infer_objects().dtype
36
+
37
+ if pd_types.is_object_dtype(dtype):
38
+ return MYSQL_DATA_TYPE.TEXT
39
+ if pd_types.is_datetime64_dtype(dtype):
40
+ return MYSQL_DATA_TYPE.DATETIME
41
+ if pd_types.is_string_dtype(dtype):
42
+ return MYSQL_DATA_TYPE.TEXT
43
+ if pd_types.is_bool_dtype(dtype):
44
+ return MYSQL_DATA_TYPE.BOOL
45
+ if pd_types.is_integer_dtype(dtype):
46
+ return MYSQL_DATA_TYPE.INT
47
+ if pd_types.is_numeric_dtype(dtype):
48
+ return MYSQL_DATA_TYPE.FLOAT
49
+ return MYSQL_DATA_TYPE.TEXT
50
+
51
+
52
+ @dataclass(kw_only=True, slots=True)
10
53
  class Column:
11
- def __init__(self, name=None, alias=None,
12
- table_name=None, table_alias=None,
13
- type=None, database=None, flags=None,
14
- charset=None):
15
- if alias is None:
16
- alias = name
17
- if table_alias is None:
18
- table_alias = table_name
19
- self.name = name
20
- self.alias = alias
21
- self.table_name = table_name
22
- self.table_alias = table_alias
23
- self.type = type
24
- self.database = database
25
- self.flags = flags
26
- self.charset = charset
54
+ name: str = field(default=MISSING)
55
+ alias: str | None = None
56
+ table_name: str | None = None
57
+ table_alias: str | None = None
58
+ type: MYSQL_DATA_TYPE | None = None
59
+ database: str | None = None
60
+ flags: dict = None
61
+ charset: str | None = None
62
+
63
+ def __post_init__(self):
64
+ if self.alias is None:
65
+ self.alias = self.name
66
+ if self.table_alias is None:
67
+ self.table_alias = self.table_name
27
68
 
28
69
  def get_hash_name(self, prefix):
29
70
  table_name = self.table_name if self.table_alias is None else self.table_alias
@@ -32,11 +73,8 @@ class Column:
32
73
  name = f'{prefix}_{table_name}_{name}'
33
74
  return name
34
75
 
35
- def __repr__(self):
36
- return f'{self.__class__.__name__}({self.__dict__})'
37
-
38
76
 
39
- def rename_df_columns(df: pd.DataFrame, names: Optional[List] = None) -> None:
77
+ def rename_df_columns(df: pd.DataFrame, names: list | None = None) -> None:
40
78
  """Inplace rename of dataframe columns
41
79
 
42
80
  Args:
@@ -50,7 +88,15 @@ def rename_df_columns(df: pd.DataFrame, names: Optional[List] = None) -> None:
50
88
 
51
89
 
52
90
  class ResultSet:
53
- def __init__(self, columns=None, values: List[List] = None, df: pd.DataFrame = None, affected_rows: int = None):
91
+ def __init__(
92
+ self,
93
+ columns: list[Column] | None = None,
94
+ values: list[list] | None = None,
95
+ df: pd.DataFrame | None = None,
96
+ affected_rows: int | None = None,
97
+ is_prediction: bool = False,
98
+ mysql_types: list[MYSQL_DATA_TYPE] | None = None
99
+ ):
54
100
  """
55
101
  Args:
56
102
  columns: list of Columns
@@ -62,15 +108,18 @@ class ResultSet:
62
108
  columns = []
63
109
  self._columns = columns
64
110
 
65
- if values is None:
66
- df = None
67
- elif df is None:
68
- df = pd.DataFrame(values)
111
+ if df is None:
112
+ if values is None:
113
+ df = None
114
+ else:
115
+ df = pd.DataFrame(values)
69
116
  self._df = df
70
117
 
71
118
  self.affected_rows = affected_rows
72
119
 
73
- self.is_prediction = False
120
+ self.is_prediction = is_prediction
121
+
122
+ self.mysql_types = mysql_types
74
123
 
75
124
  def __repr__(self):
76
125
  col_names = ', '.join([col.name for col in self._columns])
@@ -89,43 +138,76 @@ class ResultSet:
89
138
 
90
139
  # --- converters ---
91
140
 
92
- def from_df(self, df, database=None, table_name=None, table_alias=None):
93
- self._columns = [
141
+ @classmethod
142
+ def from_df(
143
+ cls, df: pd.DataFrame, database=None, table_name=None, table_alias=None,
144
+ is_prediction: bool = False, mysql_types: list[MYSQL_DATA_TYPE] | None = None
145
+ ):
146
+ match mysql_types:
147
+ case None:
148
+ mysql_types = [None] * len(df.columns)
149
+ case list() if len(mysql_types) != len(df.columns):
150
+ raise WrongArgumentError(
151
+ f'Mysql types length mismatch: {len(mysql_types)} != {len(df.columns)}'
152
+ )
153
+
154
+ columns = [
94
155
  Column(
95
156
  name=column_name,
96
157
  table_name=table_name,
97
158
  table_alias=table_alias,
98
159
  database=database,
99
- type=column_dtype
100
- ) for column_name, column_dtype
101
- in zip(df.columns, df.dtypes)
160
+ type=mysql_type
161
+ ) for column_name, mysql_type
162
+ in zip(df.columns, mysql_types)
102
163
  ]
103
164
 
104
165
  rename_df_columns(df)
105
- self._df = df
166
+ return cls(
167
+ df=df,
168
+ columns=columns,
169
+ is_prediction=is_prediction,
170
+ mysql_types=mysql_types
171
+ )
106
172
 
107
- return self
173
+ @classmethod
174
+ def from_df_cols(cls, df: pd.DataFrame, columns_dict: dict[str, Column], strict: bool = True) -> 'ResultSet':
175
+ """Create ResultSet from dataframe and dictionary of columns
108
176
 
109
- def from_df_cols(self, df, col_names, strict=True):
110
- # find column by alias
111
- alias_idx = {}
112
- for col in col_names.values():
113
- if col.alias is not None:
114
- alias_idx[col.alias] = col
177
+ Args:
178
+ df (pd.DataFrame): dataframe
179
+ columns_dict (dict[str, Column]): dictionary of columns
180
+ strict (bool): if True, raise an error if a column is not found in columns_dict
115
181
 
116
- for col in df.columns:
117
- if col in col_names or strict:
118
- column = col_names[col]
119
- elif col in alias_idx:
120
- column = alias_idx[col]
121
- else:
122
- column = Column(col)
123
- self._columns.append(column)
182
+ Returns:
183
+ ResultSet: result set
184
+
185
+ Raises:
186
+ ValueError: if a column is not found in columns_dict and strict is True
187
+ """
188
+ alias_idx = {
189
+ column.alias: column
190
+ for column in columns_dict.values()
191
+ if column.alias is not None
192
+ }
193
+
194
+ columns = []
195
+ for column_name in df.columns:
196
+ if strict and column_name not in columns_dict:
197
+ raise ValueError(f'Column {column_name} not found in columns_dict')
198
+ column = (
199
+ columns_dict.get(column_name)
200
+ or alias_idx.get(column_name)
201
+ or Column(name=column_name)
202
+ )
203
+ columns.append(column)
124
204
 
125
205
  rename_df_columns(df)
126
- self._df = df
127
206
 
128
- return self
207
+ return cls(
208
+ columns=columns,
209
+ df=df
210
+ )
129
211
 
130
212
  def to_df(self):
131
213
  columns_names = self.get_column_names()
@@ -133,7 +215,7 @@ class ResultSet:
133
215
  rename_df_columns(df, columns_names)
134
216
  return df
135
217
 
136
- def to_df_cols(self, prefix=''):
218
+ def to_df_cols(self, prefix: str = '') -> tuple[pd.DataFrame, dict[str, Column]]:
137
219
  # returns dataframe and dict of columns
138
220
  # can be restored to ResultSet by from_df_cols method
139
221
 
@@ -262,6 +344,57 @@ class ResultSet:
262
344
  )
263
345
  self.add_raw_df(df)
264
346
 
347
+ def get_ast_columns(self) -> list[TableColumn]:
348
+ """Converts ResultSet columns to a list of TableColumn objects with SQLAlchemy types.
349
+
350
+ This method processes each column in the ResultSet, determines its MySQL data type
351
+ (inferring it if necessary), and maps it to the appropriate SQLAlchemy type.
352
+ The resulting TableColumn objects most likely will be used in CREATE TABLE statement.
353
+
354
+ Returns:
355
+ list[TableColumn]: A list of TableColumn objects with properly mapped SQLAlchemy types
356
+ """
357
+ columns: list[TableColumn] = []
358
+
359
+ type_mapping = {
360
+ MYSQL_DATA_TYPE.TINYINT: sqlalchemy_types.INTEGER,
361
+ MYSQL_DATA_TYPE.SMALLINT: sqlalchemy_types.INTEGER,
362
+ MYSQL_DATA_TYPE.MEDIUMINT: sqlalchemy_types.INTEGER,
363
+ MYSQL_DATA_TYPE.INT: sqlalchemy_types.INTEGER,
364
+ MYSQL_DATA_TYPE.BIGINT: sqlalchemy_types.INTEGER,
365
+ MYSQL_DATA_TYPE.YEAR: sqlalchemy_types.INTEGER,
366
+ MYSQL_DATA_TYPE.BOOL: sqlalchemy_types.BOOLEAN,
367
+ MYSQL_DATA_TYPE.BOOLEAN: sqlalchemy_types.BOOLEAN,
368
+ MYSQL_DATA_TYPE.FLOAT: sqlalchemy_types.FLOAT,
369
+ MYSQL_DATA_TYPE.DOUBLE: sqlalchemy_types.FLOAT,
370
+ MYSQL_DATA_TYPE.TIME: sqlalchemy_types.TIME,
371
+ MYSQL_DATA_TYPE.DATE: sqlalchemy_types.DATE,
372
+ MYSQL_DATA_TYPE.DATETIME: sqlalchemy_types.DATETIME,
373
+ MYSQL_DATA_TYPE.TIMESTAMP: sqlalchemy_types.TIMESTAMP,
374
+ }
375
+
376
+ for i, column in enumerate(self._columns):
377
+ column_type: MYSQL_DATA_TYPE | None = column.type
378
+
379
+ # infer MYSQL_DATA_TYPE if not set
380
+ if isinstance(column_type, MYSQL_DATA_TYPE) is False:
381
+ if column_type is not None:
382
+ logger.warning(f'Unexpected column type: {column_type}')
383
+ if self._df is None:
384
+ column_type = MYSQL_DATA_TYPE.TEXT
385
+ else:
386
+ column_type = get_mysql_data_type_from_series(self._df.iloc[:, i])
387
+
388
+ sqlalchemy_type = type_mapping.get(column_type, sqlalchemy_types.TEXT)
389
+
390
+ columns.append(
391
+ TableColumn(
392
+ name=column.alias,
393
+ type=sqlalchemy_type
394
+ )
395
+ )
396
+ return columns
397
+
265
398
  def to_lists(self, json_types=False):
266
399
  """
267
400
  :param type_cast: cast numpy types
@@ -281,7 +414,7 @@ class ResultSet:
281
414
  return df.to_records(index=False).tolist()
282
415
 
283
416
  # slower but keep timestamp type
284
- df = self._df.replace({np.nan: None})
417
+ df = self._df.replace({np.nan: None}) # TODO rework
285
418
  return df.to_dict('split')['data']
286
419
 
287
420
  def get_column_values(self, col_idx):
@@ -252,7 +252,7 @@ class SQLQuery:
252
252
  # return query info
253
253
  # columns in upper case
254
254
  rec = {k.upper(): v for k, v in self.run_query.get_info().items()}
255
- self.fetched_data = ResultSet().from_df(pd.DataFrame([rec]))
255
+ self.fetched_data = ResultSet.from_df(pd.DataFrame([rec]))
256
256
  self.columns_list = self.fetched_data.columns
257
257
  return
258
258
  self.run_query.mark_as_run()
@@ -91,21 +91,18 @@ class ApplyPredictorRowStepCall(ApplyPredictorBaseCall):
91
91
 
92
92
  table_name = get_preditor_alias(step, self.context.get('database'))
93
93
 
94
- result = ResultSet()
95
- result.is_prediction = True
96
94
  if len(predictions) == 0:
97
95
  columns_names = project_datanode.get_table_columns_names(predictor_name)
98
96
  predictions = pd.DataFrame([], columns=columns_names)
99
97
 
100
- result.from_df(
101
- predictions,
98
+ return ResultSet.from_df(
99
+ df=predictions,
102
100
  database=table_name[0],
103
101
  table_name=table_name[1],
104
- table_alias=table_name[2]
102
+ table_alias=table_name[2],
103
+ is_prediction=True
105
104
  )
106
105
 
107
- return result
108
-
109
106
 
110
107
  class ApplyPredictorStepCall(ApplyPredictorBaseCall):
111
108
 
@@ -164,15 +161,14 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
164
161
  params['force_ts_infer'] = True
165
162
  _mdb_forecast_offset = None
166
163
 
167
- data.add_column(Column('__mdb_forecast_offset'), _mdb_forecast_offset)
164
+ data.add_column(Column(name='__mdb_forecast_offset'), _mdb_forecast_offset)
168
165
 
169
166
  table_name = get_preditor_alias(step, self.context['database'])
170
- result = ResultSet()
171
- result.is_prediction = True
172
167
 
173
168
  project_datanode = self.session.datahub.get(project_name)
174
169
  if len(data) == 0:
175
170
  columns_names = project_datanode.get_table_columns_names(predictor_name) + ['__mindsdb_row_id']
171
+ result = ResultSet(is_prediction=True)
176
172
  for column_name in columns_names:
177
173
  result.add_column(Column(
178
174
  name=column_name,
@@ -230,11 +226,12 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
230
226
  pred_data = self.apply_ts_filter(pred_data, where_data, step, predictor_metadata)
231
227
  predictions = pd.DataFrame(pred_data)
232
228
 
233
- result.from_df(
229
+ result = ResultSet.from_df(
234
230
  predictions,
235
231
  database=table_name[0],
236
232
  table_name=table_name[1],
237
- table_alias=table_name[2]
233
+ table_alias=table_name[2],
234
+ is_prediction=True
238
235
  )
239
236
 
240
237
  return result
@@ -7,11 +7,12 @@ from mindsdb_sql_parser.ast import (
7
7
  BinaryOperation,
8
8
  Tuple,
9
9
  )
10
- from mindsdb.api.executor.planner.steps import FetchDataframeStep
11
- from mindsdb.integrations.utilities.query_traversal import query_traversal
12
10
 
11
+ from mindsdb.api.executor.planner.steps import FetchDataframeStep
12
+ from mindsdb.api.executor.datahub.classes.response import DataHubResponse
13
13
  from mindsdb.api.executor.sql_query.result_set import ResultSet
14
14
  from mindsdb.api.executor.exceptions import UnknownError
15
+ from mindsdb.integrations.utilities.query_traversal import query_traversal
15
16
  from mindsdb.interfaces.query_context.context_controller import query_context_controller
16
17
 
17
18
  from .base import BaseStepCall
@@ -89,7 +90,7 @@ class FetchDataframeStepCall(BaseStepCall):
89
90
  table_alias = (self.context.get('database'), 'result', 'result')
90
91
 
91
92
  # fetch raw_query
92
- response = dn.query(
93
+ response: DataHubResponse = dn.query(
93
94
  native_query=step.raw_query,
94
95
  session=self.session
95
96
  )
@@ -105,7 +106,7 @@ class FetchDataframeStepCall(BaseStepCall):
105
106
 
106
107
  query, context_callback = query_context_controller.handle_db_context_vars(query, dn, self.session)
107
108
 
108
- response = dn.query(
109
+ response: DataHubResponse = dn.query(
109
110
  query=query,
110
111
  session=self.session
111
112
  )
@@ -114,13 +115,10 @@ class FetchDataframeStepCall(BaseStepCall):
114
115
  if context_callback:
115
116
  context_callback(df, response.columns)
116
117
 
117
- result = ResultSet()
118
-
119
- result.from_df(
118
+ return ResultSet.from_df(
120
119
  df,
121
120
  table_name=table_alias[1],
122
121
  table_alias=table_alias[2],
123
- database=table_alias[0]
122
+ database=table_alias[0],
123
+ mysql_types=response.mysql_types
124
124
  )
125
-
126
- return result
@@ -8,7 +8,7 @@ from mindsdb.integrations.utilities.query_traversal import query_traversal
8
8
  from mindsdb.interfaces.query_context.context_controller import RunningQuery
9
9
  from mindsdb.api.executor.sql_query.result_set import ResultSet
10
10
  from mindsdb.utilities import log
11
- from mindsdb.utilities.config import Config
11
+ from mindsdb.utilities.config import config
12
12
  from mindsdb.utilities.partitioning import get_max_thread_count, split_data_frame
13
13
  from mindsdb.api.executor.sql_query.steps.fetch_dataframe import get_table_alias, get_fill_param_fnc
14
14
  from mindsdb.utilities.context_executor import ContextThreadPoolExecutor
@@ -64,8 +64,6 @@ class FetchDataframePartitionCall(BaseStepCall):
64
64
  self.current_step_num = step.step_num
65
65
  self.substeps = step.steps
66
66
 
67
- config = Config()
68
-
69
67
  # ml task queue enabled?
70
68
  use_threads, thread_count = False, None
71
69
  if config['ml_task_queue']['type'] == 'redis':
@@ -95,21 +93,8 @@ class FetchDataframePartitionCall(BaseStepCall):
95
93
  """
96
94
 
97
95
  results = []
98
- while True:
99
-
100
- # fetch batch
101
- query2 = run_query.get_partition_query(self.current_step_num, query)
102
- response = self.dn.query(
103
- query=query2,
104
- session=self.session
105
- )
106
- df = response.data_frame
107
-
108
- if df is None or len(df) == 0:
109
- break
110
96
 
111
- # executing of sub steps can modify dataframe columns, lets memorise max tracking value
112
- max_track_value = run_query.get_max_track_value(df)
97
+ for df in run_query.get_partitions(self.dn, self, query):
113
98
  try:
114
99
  sub_data = self.exec_sub_steps(df)
115
100
  results.append(sub_data)
@@ -119,8 +104,6 @@ class FetchDataframePartitionCall(BaseStepCall):
119
104
  else:
120
105
  raise e
121
106
 
122
- run_query.set_progress(df, max_track_value)
123
-
124
107
  return self.concat_results(results)
125
108
 
126
109
  def concat_results(self, results: List[ResultSet]) -> ResultSet:
@@ -135,7 +118,7 @@ class FetchDataframePartitionCall(BaseStepCall):
135
118
 
136
119
  data = ResultSet()
137
120
  if len(df_list) > 0:
138
- data.from_df_cols(pd.concat(df_list), col_names)
121
+ data = ResultSet.from_df_cols(pd.concat(df_list), col_names)
139
122
 
140
123
  return data
141
124
 
@@ -147,10 +130,7 @@ class FetchDataframePartitionCall(BaseStepCall):
147
130
  - substep are executed using result of previos step (like it is all fetched data is available)
148
131
  - the final result is returned and used outside to concatenate with results of other's batches
149
132
  """
150
-
151
- input_data = ResultSet()
152
-
153
- input_data.from_df(
133
+ input_data = ResultSet.from_df(
154
134
  df,
155
135
  table_name=self.table_alias[1],
156
136
  table_alias=self.table_alias[2],
@@ -191,22 +171,7 @@ class FetchDataframePartitionCall(BaseStepCall):
191
171
 
192
172
  with ContextThreadPoolExecutor(max_workers=thread_count) as executor:
193
173
 
194
- while True:
195
- # fetch batch
196
- query2 = run_query.get_partition_query(self.current_step_num, query)
197
- response = self.dn.query(
198
- query=query2,
199
- session=self.session
200
- )
201
- df = response.data_frame
202
-
203
- if df is None or len(df) == 0:
204
- # TODO detect circles: data handler ignores condition and output is repeated
205
-
206
- # exit & stop workers
207
- break
208
-
209
- max_track_value = run_query.get_max_track_value(df)
174
+ for df in run_query.get_partitions(self.dn, self, query):
210
175
 
211
176
  # split into chunks and send to workers
212
177
  futures = []
@@ -225,9 +190,5 @@ class FetchDataframePartitionCall(BaseStepCall):
225
190
  if self.sql_query.stop_event is not None and self.sql_query.stop_event.is_set():
226
191
  executor.shutdown()
227
192
  raise RuntimeError('Query is interrupted')
228
- # TODO
229
- # 1. get next batch without updating track_value:
230
- # it allows to keep queue_in filled with data between fetching batches
231
- run_query.set_progress(df, max_track_value)
232
193
 
233
194
  return self.concat_results(results)
@@ -1,17 +1,18 @@
1
1
  from mindsdb_sql_parser.ast import (
2
2
  Identifier,
3
3
  )
4
+
4
5
  from mindsdb.api.executor.planner.steps import (
5
6
  SaveToTable,
6
7
  InsertToTable,
7
8
  CreateTableStep
8
9
  )
9
-
10
10
  from mindsdb.api.executor.sql_query.result_set import ResultSet, Column
11
11
  from mindsdb.api.executor.exceptions import (
12
12
  NotSupportedYet,
13
13
  LogicError
14
14
  )
15
+ from mindsdb.integrations.libs.response import INF_SCHEMA_COLUMNS_NAMES
15
16
 
16
17
  from .base import BaseStepCall
17
18
 
@@ -30,12 +31,32 @@ class InsertToTableCall(BaseStepCall):
30
31
  if step.is_replace:
31
32
  is_replace = True
32
33
 
34
+ if len(step.table.parts) > 1:
35
+ integration_name = step.table.parts[0]
36
+ table_name = Identifier(parts=step.table.parts[1:])
37
+ else:
38
+ integration_name = self.context['database']
39
+ table_name = step.table
40
+
41
+ dn = self.session.datahub.get(integration_name)
42
+
43
+ if hasattr(dn, 'create_table') is False:
44
+ raise NotSupportedYet(f"Creating table in '{integration_name}' is not supported")
45
+
33
46
  if step.dataframe is not None:
34
47
  data = self.steps_data[step.dataframe.result.step_num]
35
48
  elif step.query is not None:
36
49
  data = ResultSet()
37
- for col in step.query.columns:
38
- data.add_column(Column(col.name))
50
+ if step.query.columns is None:
51
+ # Is query like: INSERT INTO table VALUES (...)
52
+ table_columns_df = dn.get_table_columns_df(str(table_name))
53
+ columns_names = table_columns_df[INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME].to_list()
54
+ for column_name in columns_names:
55
+ data.add_column(Column(name=column_name))
56
+ else:
57
+ # Is query like: INSERT INTO table (column_name, ...) VALUES (...)
58
+ for col in step.query.columns:
59
+ data.add_column(Column(name=col.name))
39
60
 
40
61
  records = []
41
62
  for row in step.query.values:
@@ -53,18 +74,6 @@ class InsertToTableCall(BaseStepCall):
53
74
  else:
54
75
  raise LogicError(f'Data not found for insert: {step}')
55
76
 
56
- if len(step.table.parts) > 1:
57
- integration_name = step.table.parts[0]
58
- table_name = Identifier(parts=step.table.parts[1:])
59
- else:
60
- integration_name = self.context['database']
61
- table_name = step.table
62
-
63
- dn = self.session.datahub.get(integration_name)
64
-
65
- if hasattr(dn, 'create_table') is False:
66
- raise NotSupportedYet(f"Creating table in '{integration_name}' is not supported")
67
-
68
77
  # del 'service' columns
69
78
  for col in data.find_columns('__mindsdb_row_id'):
70
79
  data.del_column(col)
@@ -101,7 +101,7 @@ class JoinStepCall(BaseStepCall):
101
101
  resp_df.replace({np.nan: None}, inplace=True)
102
102
 
103
103
  names_a.update(names_b)
104
- data = ResultSet().from_df_cols(resp_df, col_names=names_a)
104
+ data = ResultSet.from_df_cols(df=resp_df, columns_dict=names_a)
105
105
 
106
106
  for col in data.find_columns('__mindsdb_row_id'):
107
107
  data.del_column(col)
@@ -83,4 +83,4 @@ class ProjectStepCall(BaseStepCall):
83
83
 
84
84
  res = query_df(df, query, session=self.session)
85
85
 
86
- return ResultSet().from_df_cols(res, col_names, strict=False)
86
+ return ResultSet.from_df_cols(df=res, columns_dict=col_names, strict=False)
@@ -38,4 +38,4 @@ class DataStepCall(BaseStepCall):
38
38
  def call(self, step):
39
39
  # create resultset
40
40
  df = pd.DataFrame(step.data)
41
- return ResultSet().from_df(df, database='', table_name='')
41
+ return ResultSet.from_df(df, database='', table_name='')
@@ -55,7 +55,7 @@ class SubSelectStepCall(BaseStepCall):
55
55
 
56
56
  for col_name in query_cols:
57
57
  if col_name not in result_cols:
58
- result.add_column(Column(col_name))
58
+ result.add_column(Column(name=col_name))
59
59
 
60
60
  # inject previous step values
61
61
  if isinstance(query, Select):
@@ -69,12 +69,10 @@ class SubSelectStepCall(BaseStepCall):
69
69
  df = result.to_df()
70
70
  res = query_df(df, query, session=self.session)
71
71
 
72
- result2 = ResultSet()
73
72
  # get database from first column
74
73
  database = result.columns[0].database
75
- result2.from_df(res, database, table_name)
76
74
 
77
- return result2
75
+ return ResultSet.from_df(res, database, table_name)
78
76
 
79
77
 
80
78
  class QueryStepCall(BaseStepCall):
@@ -86,7 +84,7 @@ class QueryStepCall(BaseStepCall):
86
84
 
87
85
  if step.from_table is not None:
88
86
  if isinstance(step.from_table, pd.DataFrame):
89
- result_set = ResultSet().from_df(step.from_table)
87
+ result_set = ResultSet.from_df(step.from_table)
90
88
  else:
91
89
  result_set = self.steps_data[step.from_table.step_num]
92
90
  else:
@@ -208,6 +206,4 @@ class QueryStepCall(BaseStepCall):
208
206
  query.from_table = Identifier('df_table')
209
207
  res = query_df(df, query, session=self.session)
210
208
 
211
- data = ResultSet().from_df_cols(res, col_names, strict=False)
212
-
213
- return data
209
+ return ResultSet.from_df_cols(df=res, columns_dict=col_names, strict=False)