mage-ai 0.9.74__py3-none-any.whl → 0.9.79__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (420) hide show
  1. mage_ai/ai/llm_pipeline_wizard.py +6 -4
  2. mage_ai/ai/openai_client.py +7 -5
  3. mage_ai/api/policies/PipelineSchedulePolicy.py +1 -0
  4. mage_ai/api/presenters/PipelineSchedulePresenter.py +11 -2
  5. mage_ai/api/resources/GitFileResource.py +8 -0
  6. mage_ai/api/resources/PipelineScheduleResource.py +20 -14
  7. mage_ai/api/resources/PipelineTriggerResource.py +3 -1
  8. mage_ai/api/resources/SessionResource.py +2 -2
  9. mage_ai/api/resources/SyncResource.py +1 -1
  10. mage_ai/api/resources/UserResource.py +1 -1
  11. mage_ai/cli/main.py +8 -1
  12. mage_ai/data_cleaner/analysis/charts.py +1 -1
  13. mage_ai/data_cleaner/cleaning_rules/reformat_values.py +1 -1
  14. mage_ai/data_integrations/destinations/constants.py +3 -0
  15. mage_ai/data_integrations/sources/constants.py +2 -0
  16. mage_ai/data_preparation/executors/block_executor.py +8 -3
  17. mage_ai/data_preparation/executors/pipeline_executor.py +35 -19
  18. mage_ai/data_preparation/git/utils.py +2 -2
  19. mage_ai/data_preparation/logging/logger_manager.py +31 -2
  20. mage_ai/data_preparation/models/block/__init__.py +33 -27
  21. mage_ai/data_preparation/models/block/dbt/dbt_adapter.py +20 -8
  22. mage_ai/data_preparation/models/block/dynamic/constants.py +0 -1
  23. mage_ai/data_preparation/models/block/dynamic/counter.py +1 -3
  24. mage_ai/data_preparation/models/block/outputs.py +7 -1
  25. mage_ai/data_preparation/models/block/r/__init__.py +16 -5
  26. mage_ai/data_preparation/models/block/sql/__init__.py +2 -0
  27. mage_ai/data_preparation/models/block/sql/mssql.py +8 -0
  28. mage_ai/data_preparation/models/block/sql/utils/shared.py +6 -2
  29. mage_ai/data_preparation/models/constants.py +4 -1
  30. mage_ai/data_preparation/models/pipeline.py +11 -2
  31. mage_ai/data_preparation/models/project/__init__.py +3 -1
  32. mage_ai/data_preparation/models/triggers/__init__.py +1 -1
  33. mage_ai/data_preparation/storage/local_storage.py +4 -1
  34. mage_ai/data_preparation/templates/constants.py +7 -0
  35. mage_ai/data_preparation/templates/data_exporters/streaming/elasticsearch.yaml +3 -0
  36. mage_ai/data_preparation/templates/data_loaders/airtable.py +28 -0
  37. mage_ai/data_preparation/templates/data_loaders/streaming/nats.yaml +6 -3
  38. mage_ai/data_preparation/templates/repo/io_config.yaml +2 -0
  39. mage_ai/io/airtable.py +104 -0
  40. mage_ai/io/base.py +30 -1
  41. mage_ai/io/bigquery.py +36 -0
  42. mage_ai/io/config.py +6 -0
  43. mage_ai/io/mssql.py +21 -9
  44. mage_ai/io/mysql.py +6 -1
  45. mage_ai/io/oracledb.py +2 -4
  46. mage_ai/io/postgres.py +41 -19
  47. mage_ai/io/qdrant.py +1 -1
  48. mage_ai/io/redshift.py +13 -0
  49. mage_ai/io/sql.py +1 -0
  50. mage_ai/io/utils.py +18 -0
  51. mage_ai/orchestration/db/__init__.py +23 -3
  52. mage_ai/orchestration/db/migrations/versions/39d36f1dab73_create_genericjob.py +47 -0
  53. mage_ai/orchestration/db/models/oauth.py +2 -1
  54. mage_ai/orchestration/db/models/schedules.py +108 -6
  55. mage_ai/orchestration/db/models/schedules_project_platform.py +1 -1
  56. mage_ai/orchestration/db/models/secrets.py +11 -1
  57. mage_ai/orchestration/job_manager.py +19 -0
  58. mage_ai/orchestration/metrics/pipeline_run.py +1 -1
  59. mage_ai/orchestration/notification/sender.py +2 -2
  60. mage_ai/orchestration/pipeline_scheduler_original.py +150 -6
  61. mage_ai/orchestration/pipeline_scheduler_project_platform.py +4 -5
  62. mage_ai/orchestration/queue/config.py +11 -1
  63. mage_ai/orchestration/queue/process_queue.py +4 -0
  64. mage_ai/orchestration/utils/distributed_lock.py +8 -1
  65. mage_ai/orchestration/utils/resources.py +56 -2
  66. mage_ai/sample_datasets/salary_survey.csv +52 -52
  67. mage_ai/server/api/base.py +41 -0
  68. mage_ai/server/api/constants.py +1 -0
  69. mage_ai/server/api/triggers.py +9 -0
  70. mage_ai/server/constants.py +1 -1
  71. mage_ai/server/frontend_dist/404.html +3 -3
  72. mage_ai/server/frontend_dist/_next/static/TUo4RceCdMufBTBTq8CAq/_buildManifest.js +1 -0
  73. mage_ai/server/frontend_dist/_next/static/chunks/{1187-839336d276186105.js → 1187-4560c3895e1d7099.js} +1 -1
  74. mage_ai/server/frontend_dist/_next/static/chunks/{1598-0adca9dce3ba4c60.js → 1598-cbf3f5a6078fc3f5.js} +1 -1
  75. mage_ai/server/frontend_dist/_next/static/chunks/2717-638a944d24d5abde.js +1 -0
  76. mage_ai/server/frontend_dist/_next/static/chunks/3548-36f746b1824004f2.js +1 -0
  77. mage_ai/server/frontend_dist/_next/static/chunks/{3763-39a5174f6a3924db.js → 3763-aabe2703076636b0.js} +1 -1
  78. mage_ai/server/frontend_dist/_next/static/chunks/3782-3e2acb5ed45b582b.js +1 -0
  79. mage_ai/server/frontend_dist/_next/static/chunks/449-5e2253c6aba42557.js +1 -0
  80. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/5627-d5e559859dd0e1e0.js → frontend_dist/_next/static/chunks/5627-10e76bafa5a26f5f.js} +1 -1
  81. mage_ai/server/frontend_dist/_next/static/chunks/{5699-e49718dfc9eb2854.js → 5699-e99379e332bd0b41.js} +1 -1
  82. mage_ai/server/frontend_dist/_next/static/chunks/{7966-163da2621b8c987c.js → 7966-a5a7db345ce81263.js} +1 -1
  83. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-b697b35dfc4e6e26.js +2 -0
  84. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/compute-ed67fa8e81662e8b.js → frontend_dist/_next/static/chunks/pages/compute-9e2dea78024e3bb4.js} +1 -1
  85. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/files-e0ecd7ced09a63b2.js → frontend_dist/_next/static/chunks/pages/files-e08c7fe76f968f9c.js} +1 -1
  86. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/{[...slug]-c7a729477ecda50e.js → [...slug]-30c3807057a4e65b.js} +1 -1
  87. mage_ai/server/frontend_dist/_next/static/chunks/pages/{global-data-products-fd6ae6a358a60a0c.js → global-data-products-8dcb3b31af9e0e39.js} +1 -1
  88. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/global-hooks/[...slug]-8e50243797a7fe59.js → frontend_dist/_next/static/chunks/pages/global-hooks/[...slug]-85a64b64d27214b6.js} +1 -1
  89. mage_ai/server/frontend_dist/_next/static/chunks/pages/{global-hooks-d0c003446332dc0d.js → global-hooks-4ff959d51b8a9502.js} +1 -1
  90. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{files-a69ed8e9f814490c.js → files-d08a460641d0efaa.js} +1 -1
  91. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{overview-1aad7093c6d39257.js → overview-aae747f487e08d51.js} +1 -1
  92. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{pipeline-runs-528d30e0d13b0cc7.js → pipeline-runs-09a842d64a6ada62.js} +1 -1
  93. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{settings-fb9201d9cf63031d.js → settings-2e98e57d9376a458.js} +1 -1
  94. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users/{[user]-000f5a980a07da39.js → [user]-7be6e41ad66089bb.js} +1 -1
  95. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/manage/users/new-e4e613f6e817a733.js → frontend_dist/_next/static/chunks/pages/manage/users/new-4c088833063bfa07.js} +1 -1
  96. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users-5db54821a3059c69.js +1 -0
  97. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/manage-34d718b8a4066c23.js → frontend_dist/_next/static/chunks/pages/manage-868fcd8cbeb265f0.js} +1 -1
  98. mage_ai/server/frontend_dist/_next/static/chunks/pages/{oauth-3bfd1b8d7f036726.js → oauth-6ceceb62191dfe8a.js} +1 -1
  99. mage_ai/server/frontend_dist/_next/static/chunks/pages/overview-f65416f6dbe30ad3.js +1 -0
  100. mage_ai/server/frontend_dist/_next/static/chunks/pages/{pipeline-runs-5f8c100e648efa8a.js → pipeline-runs-2d0136b51b57de93.js} +1 -1
  101. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-688c652f3296bb9c.js → [...slug]-1ad5238742e25b4c.js} +1 -1
  102. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-bd11e87d026bfbf9.js +1 -0
  103. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{dashboard-1236e36d39b1637d.js → dashboard-0f4f47f721b0723f.js} +1 -1
  104. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-5ae8efe9e0530212.js +1 -0
  105. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-fe91dfb0091f6bc6.js +1 -0
  106. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-0d68d4bf6290fefb.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-cf794b2d22a80f31.js} +1 -1
  107. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-9254358d58f07714.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-a964caef91bed9e1.js} +1 -1
  108. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{monitors-821001e690caebe2.js → monitors-80bebb4401eefe25.js} +1 -1
  109. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-2eae7cb017027682.js +1 -0
  110. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs-2d4b2a0800a66b33.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-776b2e5b0b6ceba8.js} +1 -1
  111. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-03d9bca3bc5e6088.js +1 -0
  112. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/syncs-707ed8ca942ca802.js +1 -0
  113. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/{[...slug]-259143ed3cf59e31.js → [...slug]-8429f17d4146e1ec.js} +1 -1
  114. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers-193045d9836d8d80.js +1 -0
  115. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-d25d07db166cbb04.js +1 -0
  116. mage_ai/server/frontend_dist/_next/static/chunks/pages/platform/global-hooks/{[...slug]-5eeec927e4202b63.js → [...slug]-6834ae87bd668cb2.js} +1 -1
  117. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/platform/global-hooks-fbe9ad995d46d837.js → frontend_dist/_next/static/chunks/pages/platform/global-hooks-b3f7309a23e592b2.js} +1 -1
  118. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-fc659962d4015cb3.js → profile-f8b7374385e1f1bf.js} +1 -1
  119. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-8de68502a9afa299.js +1 -0
  120. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-a4f88c334414402b.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-50fb6a34f3913f1f.js} +1 -1
  121. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-4deb9579ef99a3c6.js → [...slug]-2e5c098c21ea32b7.js} +1 -1
  122. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e0cda2f2bfce8d61.js → permissions-54e4b15b9585bfc4.js} +1 -1
  123. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-040f83d75d0f6537.js +1 -0
  124. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-910257d16c604ebd.js → [...slug]-95088f43034e3c95.js} +1 -1
  125. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{roles-4f7a0756806cee34.js → roles-e9149e1fcf218f42.js} +1 -1
  126. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{sync-data-208d6f955204d704.js → sync-data-75b67ae4a00818ef.js} +1 -1
  127. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-c89dc67e5a1706a8.js → [...slug]-557dda05ca6c6124.js} +1 -1
  128. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users-fa61dc6c1370e6a5.js +1 -0
  129. mage_ai/server/frontend_dist/_next/static/chunks/pages/{sign-in-054b33312d3193c3.js → sign-in-593c40985d63fcf7.js} +1 -1
  130. mage_ai/server/frontend_dist/_next/static/chunks/pages/templates/{[...slug]-b6ed6a5d818bfd20.js → [...slug]-252c4b6b818345d5.js} +1 -1
  131. mage_ai/server/frontend_dist/_next/static/chunks/pages/{templates-852357bc983af2ea.js → templates-ca528bc607753ab8.js} +1 -1
  132. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/terminal-1f9c56d671bbc67d.js → frontend_dist/_next/static/chunks/pages/terminal-287362c1defcc96b.js} +1 -1
  133. mage_ai/server/frontend_dist/_next/static/chunks/pages/test-2f83af8c9f1378fe.js +1 -0
  134. mage_ai/server/frontend_dist/_next/static/chunks/pages/triggers-d9de73fb799efed8.js +1 -0
  135. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/version-control-ae3469b992a341d6.js → frontend_dist/_next/static/chunks/pages/version-control-573f0225d7a703ed.js} +1 -1
  136. mage_ai/server/frontend_dist/block-layout.html +3 -3
  137. mage_ai/server/frontend_dist/compute.html +6 -6
  138. mage_ai/server/frontend_dist/files.html +6 -6
  139. mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
  140. mage_ai/server/frontend_dist/global-data-products.html +6 -6
  141. mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
  142. mage_ai/server/frontend_dist/global-hooks.html +6 -6
  143. mage_ai/server/frontend_dist/index.html +3 -3
  144. mage_ai/server/frontend_dist/manage/files.html +6 -6
  145. mage_ai/server/frontend_dist/manage/overview.html +6 -6
  146. mage_ai/server/frontend_dist/manage/pipeline-runs.html +6 -6
  147. mage_ai/server/frontend_dist/manage/settings.html +6 -6
  148. mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
  149. mage_ai/server/frontend_dist/manage/users/new.html +6 -6
  150. mage_ai/server/frontend_dist/manage/users.html +6 -6
  151. mage_ai/server/frontend_dist/manage.html +6 -6
  152. mage_ai/server/frontend_dist/oauth.html +5 -5
  153. mage_ai/server/frontend_dist/overview.html +6 -6
  154. mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
  155. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  156. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
  157. mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
  158. mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
  159. mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
  160. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  161. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  162. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
  163. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
  164. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
  165. mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
  166. mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
  167. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  168. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
  169. mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
  170. mage_ai/server/frontend_dist/pipelines.html +6 -6
  171. mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
  172. mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
  173. mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
  174. mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
  175. mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
  176. mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
  177. mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
  178. mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
  179. mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
  180. mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
  181. mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
  182. mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
  183. mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
  184. mage_ai/server/frontend_dist/settings.html +3 -3
  185. mage_ai/server/frontend_dist/sign-in.html +7 -7
  186. mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
  187. mage_ai/server/frontend_dist/templates.html +6 -6
  188. mage_ai/server/frontend_dist/terminal.html +6 -6
  189. mage_ai/server/frontend_dist/test.html +3 -3
  190. mage_ai/server/frontend_dist/triggers.html +6 -6
  191. mage_ai/server/frontend_dist/v2/canvas.html +2 -2
  192. mage_ai/server/frontend_dist/v2.html +2 -2
  193. mage_ai/server/frontend_dist/version-control.html +6 -6
  194. mage_ai/server/frontend_dist_base_path_template/404.html +3 -3
  195. mage_ai/server/frontend_dist_base_path_template/_next/static/2QL-FT4lFR0a9bDZ7lNd9/_buildManifest.js +1 -0
  196. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{1187-839336d276186105.js → 1187-4560c3895e1d7099.js} +1 -1
  197. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{1598-0adca9dce3ba4c60.js → 1598-cbf3f5a6078fc3f5.js} +1 -1
  198. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2717-638a944d24d5abde.js +1 -0
  199. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3548-36f746b1824004f2.js +1 -0
  200. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3763-39a5174f6a3924db.js → 3763-aabe2703076636b0.js} +1 -1
  201. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-3e2acb5ed45b582b.js +1 -0
  202. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/449-5e2253c6aba42557.js +1 -0
  203. mage_ai/server/{frontend_dist/_next/static/chunks/5627-d5e559859dd0e1e0.js → frontend_dist_base_path_template/_next/static/chunks/5627-10e76bafa5a26f5f.js} +1 -1
  204. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{5699-e49718dfc9eb2854.js → 5699-e99379e332bd0b41.js} +1 -1
  205. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{7966-163da2621b8c987c.js → 7966-a5a7db345ce81263.js} +1 -1
  206. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-f205accb03b9ff43.js +2 -0
  207. mage_ai/server/{frontend_dist/_next/static/chunks/pages/compute-ed67fa8e81662e8b.js → frontend_dist_base_path_template/_next/static/chunks/pages/compute-9e2dea78024e3bb4.js} +1 -1
  208. mage_ai/server/{frontend_dist/_next/static/chunks/pages/files-e0ecd7ced09a63b2.js → frontend_dist_base_path_template/_next/static/chunks/pages/files-e08c7fe76f968f9c.js} +1 -1
  209. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/{[...slug]-c7a729477ecda50e.js → [...slug]-30c3807057a4e65b.js} +1 -1
  210. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{global-data-products-fd6ae6a358a60a0c.js → global-data-products-8dcb3b31af9e0e39.js} +1 -1
  211. mage_ai/server/{frontend_dist/_next/static/chunks/pages/global-hooks/[...slug]-8e50243797a7fe59.js → frontend_dist_base_path_template/_next/static/chunks/pages/global-hooks/[...slug]-85a64b64d27214b6.js} +1 -1
  212. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{global-hooks-d0c003446332dc0d.js → global-hooks-4ff959d51b8a9502.js} +1 -1
  213. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/{files-a69ed8e9f814490c.js → files-d08a460641d0efaa.js} +1 -1
  214. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/{overview-1aad7093c6d39257.js → overview-aae747f487e08d51.js} +1 -1
  215. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/{pipeline-runs-528d30e0d13b0cc7.js → pipeline-runs-09a842d64a6ada62.js} +1 -1
  216. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/{settings-fb9201d9cf63031d.js → settings-2e98e57d9376a458.js} +1 -1
  217. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/users/{[user]-000f5a980a07da39.js → [user]-7be6e41ad66089bb.js} +1 -1
  218. mage_ai/server/{frontend_dist/_next/static/chunks/pages/manage/users/new-e4e613f6e817a733.js → frontend_dist_base_path_template/_next/static/chunks/pages/manage/users/new-4c088833063bfa07.js} +1 -1
  219. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/users-5db54821a3059c69.js +1 -0
  220. mage_ai/server/{frontend_dist/_next/static/chunks/pages/manage-34d718b8a4066c23.js → frontend_dist_base_path_template/_next/static/chunks/pages/manage-868fcd8cbeb265f0.js} +1 -1
  221. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{oauth-3bfd1b8d7f036726.js → oauth-6ceceb62191dfe8a.js} +1 -1
  222. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/overview-f65416f6dbe30ad3.js +1 -0
  223. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{pipeline-runs-5f8c100e648efa8a.js → pipeline-runs-2d0136b51b57de93.js} +1 -1
  224. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-688c652f3296bb9c.js → [...slug]-1ad5238742e25b4c.js} +1 -1
  225. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-bd11e87d026bfbf9.js +1 -0
  226. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{dashboard-1236e36d39b1637d.js → dashboard-0f4f47f721b0723f.js} +1 -1
  227. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-5ae8efe9e0530212.js +1 -0
  228. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-fe91dfb0091f6bc6.js +1 -0
  229. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-0d68d4bf6290fefb.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-cf794b2d22a80f31.js} +1 -1
  230. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-9254358d58f07714.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-a964caef91bed9e1.js} +1 -1
  231. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{monitors-821001e690caebe2.js → monitors-80bebb4401eefe25.js} +1 -1
  232. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-2eae7cb017027682.js +1 -0
  233. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-2d4b2a0800a66b33.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs-776b2e5b0b6ceba8.js} +1 -1
  234. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-03d9bca3bc5e6088.js +1 -0
  235. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/syncs-707ed8ca942ca802.js +1 -0
  236. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/{[...slug]-259143ed3cf59e31.js → [...slug]-8429f17d4146e1ec.js} +1 -1
  237. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers-193045d9836d8d80.js +1 -0
  238. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines-d25d07db166cbb04.js +1 -0
  239. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/platform/global-hooks/{[...slug]-5eeec927e4202b63.js → [...slug]-6834ae87bd668cb2.js} +1 -1
  240. mage_ai/server/{frontend_dist/_next/static/chunks/pages/platform/global-hooks-fbe9ad995d46d837.js → frontend_dist_base_path_template/_next/static/chunks/pages/platform/global-hooks-b3f7309a23e592b2.js} +1 -1
  241. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-fc659962d4015cb3.js → profile-f8b7374385e1f1bf.js} +1 -1
  242. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-8de68502a9afa299.js +1 -0
  243. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-a4f88c334414402b.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-50fb6a34f3913f1f.js} +1 -1
  244. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-4deb9579ef99a3c6.js → [...slug]-2e5c098c21ea32b7.js} +1 -1
  245. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e0cda2f2bfce8d61.js → permissions-54e4b15b9585bfc4.js} +1 -1
  246. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-040f83d75d0f6537.js +1 -0
  247. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-910257d16c604ebd.js → [...slug]-95088f43034e3c95.js} +1 -1
  248. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{roles-4f7a0756806cee34.js → roles-e9149e1fcf218f42.js} +1 -1
  249. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{sync-data-208d6f955204d704.js → sync-data-75b67ae4a00818ef.js} +1 -1
  250. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-c89dc67e5a1706a8.js → [...slug]-557dda05ca6c6124.js} +1 -1
  251. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users-fa61dc6c1370e6a5.js +1 -0
  252. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{sign-in-054b33312d3193c3.js → sign-in-593c40985d63fcf7.js} +1 -1
  253. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/templates/{[...slug]-b6ed6a5d818bfd20.js → [...slug]-252c4b6b818345d5.js} +1 -1
  254. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{templates-852357bc983af2ea.js → templates-ca528bc607753ab8.js} +1 -1
  255. mage_ai/server/{frontend_dist/_next/static/chunks/pages/terminal-1f9c56d671bbc67d.js → frontend_dist_base_path_template/_next/static/chunks/pages/terminal-287362c1defcc96b.js} +1 -1
  256. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/test-2f83af8c9f1378fe.js +1 -0
  257. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/triggers-d9de73fb799efed8.js +1 -0
  258. mage_ai/server/{frontend_dist/_next/static/chunks/pages/version-control-ae3469b992a341d6.js → frontend_dist_base_path_template/_next/static/chunks/pages/version-control-573f0225d7a703ed.js} +1 -1
  259. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-12ad70eb5c31aa92.js → webpack-5f4be622608d9267.js} +1 -1
  260. mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
  261. mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
  262. mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
  263. mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
  264. mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
  265. mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
  266. mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
  267. mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
  268. mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
  269. mage_ai/server/frontend_dist_base_path_template/manage/overview.html +6 -6
  270. mage_ai/server/frontend_dist_base_path_template/manage/pipeline-runs.html +6 -6
  271. mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
  272. mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
  273. mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
  274. mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
  275. mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
  276. mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
  277. mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
  278. mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
  279. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  280. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
  281. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
  282. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
  283. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
  284. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  285. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  286. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
  287. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
  288. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
  289. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
  290. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
  291. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  292. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
  293. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
  294. mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
  295. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
  296. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
  297. mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
  298. mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
  299. mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
  300. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
  301. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
  302. mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
  303. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
  304. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
  305. mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
  306. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
  307. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
  308. mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
  309. mage_ai/server/frontend_dist_base_path_template/sign-in.html +7 -7
  310. mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
  311. mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
  312. mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
  313. mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
  314. mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
  315. mage_ai/server/frontend_dist_base_path_template/v2/canvas.html +2 -2
  316. mage_ai/server/frontend_dist_base_path_template/v2.html +2 -2
  317. mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
  318. mage_ai/server/scheduler_manager.py +2 -0
  319. mage_ai/server/server.py +13 -0
  320. mage_ai/server/terminal_server.py +3 -0
  321. mage_ai/server/utils/output_display.py +5 -3
  322. mage_ai/services/aws/events/events.py +2 -2
  323. mage_ai/services/gcp/cloud_run/cloud_run.py +1 -1
  324. mage_ai/services/teams/config.py +13 -2
  325. mage_ai/services/teams/teams.py +13 -11
  326. mage_ai/settings/server.py +12 -1
  327. mage_ai/shared/constants.py +3 -1
  328. mage_ai/shared/croniter.py +1398 -0
  329. mage_ai/shared/enum.py +2 -5
  330. mage_ai/shared/environments.py +27 -3
  331. mage_ai/streaming/sinks/elasticsearch.py +15 -5
  332. mage_ai/streaming/sinks/kafka.py +21 -3
  333. mage_ai/streaming/sources/kafka.py +64 -7
  334. mage_ai/streaming/sources/kafka_oauth.py +182 -0
  335. mage_ai/tests/api/endpoints/test_blocks.py +1 -101
  336. mage_ai/tests/api/endpoints/test_configuration_options.py +1 -48
  337. mage_ai/tests/api/endpoints/test_configuration_options_project_platform.py +68 -0
  338. mage_ai/tests/api/endpoints/test_custom_designs.py +1 -106
  339. mage_ai/tests/api/endpoints/test_custom_designs_project_platform.py +132 -0
  340. mage_ai/tests/api/endpoints/test_dbt_blocks.py +111 -0
  341. mage_ai/tests/api/endpoints/test_file_contents.py +0 -48
  342. mage_ai/tests/api/endpoints/test_file_contents_with_project_platform.py +66 -0
  343. mage_ai/tests/api/endpoints/test_pipelines.py +0 -134
  344. mage_ai/tests/api/endpoints/test_pipelines_with_project_platform.py +143 -0
  345. mage_ai/tests/data_preparation/executors/test_block_executor.py +3 -3
  346. mage_ai/tests/data_preparation/logging/test_logger_manager.py +24 -5
  347. mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +1 -3
  348. mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +3 -3
  349. mage_ai/tests/data_preparation/models/test_block.py +7 -0
  350. mage_ai/tests/data_preparation/models/test_pipeline.py +55 -0
  351. mage_ai/tests/data_preparation/models/test_variable.py +2 -0
  352. mage_ai/tests/data_preparation/test_repo_manager.py +0 -63
  353. mage_ai/tests/data_preparation/test_repo_manager_project_platform.py +67 -0
  354. mage_ai/tests/data_preparation/test_variable_manager.py +0 -51
  355. mage_ai/tests/data_preparation/test_variable_manager_project_platform.py +41 -0
  356. mage_ai/tests/io/create_table/test_postgresql.py +28 -0
  357. mage_ai/tests/orchestration/db/models/test_schedules.py +1 -1
  358. mage_ai/tests/orchestration/notification/test_config.py +3 -3
  359. mage_ai/tests/orchestration/notification/test_sender.py +5 -1
  360. mage_ai/tests/orchestration/utils/__init__.py +0 -0
  361. mage_ai/tests/orchestration/utils/test_resources.py +235 -0
  362. mage_ai/tests/shared/test_croniter.py +2541 -0
  363. mage_ai/tests/streaming/sinks/test_kafka.py +130 -0
  364. mage_ai/tests/streaming/sources/test_kafka.py +125 -3
  365. mage_ai/tests/streaming/sources/test_kafka_oauth.py +208 -0
  366. mage_ai/tests/streaming/sources/test_kafka_raw_value.py +105 -0
  367. mage_ai/usage_statistics/logger.py +99 -15
  368. mage_ai-0.9.79.dist-info/METADATA +358 -0
  369. {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info}/RECORD +377 -359
  370. {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info}/WHEEL +1 -1
  371. mage_ai/server/frontend_dist/_next/static/chunks/1557-1ad0c64c2d08e569.js +0 -1
  372. mage_ai/server/frontend_dist/_next/static/chunks/2717-d65056b6b5e124eb.js +0 -1
  373. mage_ai/server/frontend_dist/_next/static/chunks/3548-b2c5edfb710886a6.js +0 -1
  374. mage_ai/server/frontend_dist/_next/static/chunks/3782-4b3091e550f809a2.js +0 -1
  375. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-5bdff745074fb350.js +0 -2
  376. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users-a5e9d77ed5b50205.js +0 -1
  377. mage_ai/server/frontend_dist/_next/static/chunks/pages/overview-69af3253ad0d0d89.js +0 -1
  378. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-fe112809feb25b05.js +0 -1
  379. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e9ca358209cdf93d.js +0 -1
  380. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-a29f1615d2e7d330.js +0 -1
  381. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-6d382ae5bad9745c.js +0 -1
  382. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-cd49372ae1702963.js +0 -1
  383. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/syncs-135be8974f7f5f2b.js +0 -1
  384. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers-3af13e89adff4d6c.js +0 -1
  385. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-b578b075a8d857e3.js +0 -1
  386. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-058d283ee178c038.js +0 -1
  387. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-7b02bb70462144cb.js +0 -1
  388. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users-5212c01a9dc558da.js +0 -1
  389. mage_ai/server/frontend_dist/_next/static/chunks/pages/test-86b12cc12d4a625f.js +0 -1
  390. mage_ai/server/frontend_dist/_next/static/chunks/pages/triggers-2481c40b18d5b6d4.js +0 -1
  391. mage_ai/server/frontend_dist/_next/static/pLWT6Sqd09xYpufCVIqnz/_buildManifest.js +0 -1
  392. mage_ai/server/frontend_dist_base_path_template/_next/static/JQewSAObpbhO0wrdAM6Ng/_buildManifest.js +0 -1
  393. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-1ad0c64c2d08e569.js +0 -1
  394. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2717-d65056b6b5e124eb.js +0 -1
  395. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3548-b2c5edfb710886a6.js +0 -1
  396. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-4b3091e550f809a2.js +0 -1
  397. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-90de19bc03f1484b.js +0 -2
  398. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/users-a5e9d77ed5b50205.js +0 -1
  399. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/overview-69af3253ad0d0d89.js +0 -1
  400. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-fe112809feb25b05.js +0 -1
  401. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e9ca358209cdf93d.js +0 -1
  402. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-a29f1615d2e7d330.js +0 -1
  403. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-6d382ae5bad9745c.js +0 -1
  404. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-cd49372ae1702963.js +0 -1
  405. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/syncs-135be8974f7f5f2b.js +0 -1
  406. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers-3af13e89adff4d6c.js +0 -1
  407. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines-b578b075a8d857e3.js +0 -1
  408. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-058d283ee178c038.js +0 -1
  409. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-7b02bb70462144cb.js +0 -1
  410. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users-5212c01a9dc558da.js +0 -1
  411. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/test-86b12cc12d4a625f.js +0 -1
  412. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/triggers-2481c40b18d5b6d4.js +0 -1
  413. mage_ai-0.9.74.dist-info/METADATA +0 -544
  414. /mage_ai/server/frontend_dist/_next/static/{pLWT6Sqd09xYpufCVIqnz → TUo4RceCdMufBTBTq8CAq}/_ssgManifest.js +0 -0
  415. /mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-5bdff745074fb350.js.LICENSE.txt → _app-b697b35dfc4e6e26.js.LICENSE.txt} +0 -0
  416. /mage_ai/server/frontend_dist_base_path_template/_next/static/{JQewSAObpbhO0wrdAM6Ng → 2QL-FT4lFR0a9bDZ7lNd9}/_ssgManifest.js +0 -0
  417. /mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{_app-90de19bc03f1484b.js.LICENSE.txt → _app-f205accb03b9ff43.js.LICENSE.txt} +0 -0
  418. {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info}/entry_points.txt +0 -0
  419. {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info/licenses}/LICENSE +0 -0
  420. {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info}/top_level.txt +0 -0
mage_ai/io/config.py CHANGED
@@ -16,6 +16,8 @@ class ConfigKey(StrEnum):
16
16
  List of configuration settings for use with data IO clients.
17
17
  """
18
18
 
19
+ AIRTABLE_ACCESS_TOKEN = "AIRTABLE_ACCESS_TOKEN"
20
+
19
21
  ALGOLIA_APP_ID = 'ALGOLIA_APP_ID'
20
22
  ALGOLIA_API_KEY = 'ALGOLIA_API_KEY'
21
23
  ALGOLIA_INDEX_NAME = 'ALGOLIA_INDEX_NAME'
@@ -64,6 +66,7 @@ class ConfigKey(StrEnum):
64
66
  MONGODB_PORT = 'MONGODB_PORT'
65
67
  MONGODB_USER = 'MONGODB_USER'
66
68
 
69
+ MSSQL_AUTHENTICATION = 'MSSQL_AUTHENTICATION'
67
70
  MSSQL_DATABASE = 'MSSQL_DATABASE'
68
71
  MSSQL_DRIVER = 'MSSQL_DRIVER'
69
72
  MSSQL_HOST = 'MSSQL_HOST'
@@ -78,6 +81,7 @@ class ConfigKey(StrEnum):
78
81
  MYSQL_PASSWORD = 'MYSQL_PASSWORD'
79
82
  MYSQL_PORT = 'MYSQL_PORT'
80
83
  MYSQL_USER = 'MYSQL_USER'
84
+ MYSQL_ALLOW_LOCAL_INFILE = 'MYSQL_ALLOW_LOCAL_INFILE'
81
85
 
82
86
  ORACLEDB_USER = 'ORACLEDB_USER'
83
87
  ORACLEDB_PASSWORD = 'ORACLEDB_PASSWORD'
@@ -338,6 +342,7 @@ class VerboseConfigKey(StrEnum):
338
342
  Config key headers for the verbose configuration file format.
339
343
  """
340
344
 
345
+ AIRTABLE = 'Airtable'
341
346
  ALGOLIA = 'Algolia'
342
347
  AWS = 'AWS'
343
348
  BIGQUERY = 'BigQuery'
@@ -356,6 +361,7 @@ class VerboseConfigKey(StrEnum):
356
361
 
357
362
  class ConfigFileLoader(BaseConfigLoader):
358
363
  KEY_MAP = {
364
+ ConfigKey.AIRTABLE_ACCESS_TOKEN: VerboseConfigKey.AIRTABLE,
359
365
  ConfigKey.ALGOLIA_APP_ID: (
360
366
  VerboseConfigKey.ALGOLIA, 'app_id'),
361
367
  ConfigKey.ALGOLIA_API_KEY: (
mage_ai/io/mssql.py CHANGED
@@ -14,6 +14,7 @@ from mage_ai.io.config import BaseConfigLoader, ConfigKey
14
14
  from mage_ai.io.constants import UNIQUE_CONFLICT_METHOD_UPDATE
15
15
  from mage_ai.io.export_utils import PandasTypes
16
16
  from mage_ai.io.sql import BaseSQL
17
+ from mage_ai.shared.hash import extract
17
18
  from mage_ai.shared.parsers import encode_complex
18
19
 
19
20
  MERGE_TABLE_SQL = '''MERGE {table_name} AS t
@@ -37,12 +38,14 @@ class MSSQL(BaseSQL):
37
38
  host: str,
38
39
  password: str,
39
40
  user: str,
41
+ authentication: str = None,
40
42
  schema: str = None,
41
43
  port: int = 1433,
42
44
  verbose: bool = True,
43
45
  **kwargs,
44
46
  ) -> None:
45
47
  super().__init__(
48
+ authentication=authentication,
46
49
  database=database,
47
50
  server=host,
48
51
  user=user,
@@ -60,15 +63,19 @@ class MSSQL(BaseSQL):
60
63
  database = self.settings['database']
61
64
  username = self.settings['user']
62
65
  password = self.settings['password']
63
- return (
64
- f'DRIVER={{{driver}}};'
65
- f'SERVER={server};'
66
- f'DATABASE={database};'
67
- f'UID={username};'
68
- f'PWD={password};'
69
- 'ENCRYPT=yes;'
70
- 'TrustServerCertificate=yes;'
71
- )
66
+ conn_opts = [
67
+ f'DRIVER={{{driver}}}',
68
+ f'SERVER={server}',
69
+ f'DATABASE={database}',
70
+ f'UID={username}',
71
+ f'PWD={password}',
72
+ 'ENCRYPT=yes',
73
+ 'TrustServerCertificate=yes',
74
+ ]
75
+ if self.settings.get('authentication'):
76
+ authentication = self.settings.get('authentication')
77
+ conn_opts.append(f'Authentication={authentication}')
78
+ return ';'.join(conn_opts)
72
79
 
73
80
  def default_schema(self) -> str:
74
81
  return self.settings.get('schema') or 'dbo'
@@ -197,9 +204,11 @@ class MSSQL(BaseSQL):
197
204
  TrustServerCertificate='yes',
198
205
  ),
199
206
  )
207
+ conn_kwargs = extract(kwargs, ['pool_size', 'max_overflow'])
200
208
  engine = create_engine(
201
209
  connection_url,
202
210
  fast_executemany=True,
211
+ **conn_kwargs,
203
212
  )
204
213
 
205
214
  unique_conflict_method = kwargs.get('unique_conflict_method')
@@ -242,6 +251,7 @@ class MSSQL(BaseSQL):
242
251
  method=merge_table,
243
252
  )
244
253
  return
254
+ sql_kwargs = extract(kwargs, ['chunksize', 'method'])
245
255
 
246
256
  df.to_sql(
247
257
  table_name,
@@ -249,6 +259,7 @@ class MSSQL(BaseSQL):
249
259
  schema=schema_name,
250
260
  if_exists=if_exists or ExportWritePolicy.REPLACE,
251
261
  index=False,
262
+ **sql_kwargs,
252
263
  )
253
264
 
254
265
  def get_type(self, column: Series, dtype: str) -> str:
@@ -313,6 +324,7 @@ class MSSQL(BaseSQL):
313
324
  @classmethod
314
325
  def with_config(cls, config: BaseConfigLoader) -> 'MSSQL':
315
326
  return cls(
327
+ authentication=config[ConfigKey.MSSQL_AUTHENTICATION],
316
328
  database=config[ConfigKey.MSSQL_DATABASE],
317
329
  schema=config[ConfigKey.MSSQL_SCHEMA],
318
330
  driver=config[ConfigKey.MSSQL_DRIVER],
mage_ai/io/mysql.py CHANGED
@@ -25,6 +25,7 @@ class MySQL(BaseSQL):
25
25
  password: str,
26
26
  user: str,
27
27
  port: int = 3306,
28
+ allow_local_infile: bool = False,
28
29
  verbose: bool = True,
29
30
  **kwargs,
30
31
  ) -> None:
@@ -35,18 +36,22 @@ class MySQL(BaseSQL):
35
36
  port=port or 3306,
36
37
  user=user,
37
38
  verbose=verbose,
39
+ allow_local_infile=allow_local_infile,
38
40
  **kwargs,
39
41
  )
40
42
 
41
43
  @classmethod
42
44
  def with_config(cls, config: BaseConfigLoader) -> 'MySQL':
43
- return cls(
45
+ conn_kwargs = dict(
44
46
  database=config[ConfigKey.MYSQL_DATABASE],
45
47
  host=config[ConfigKey.MYSQL_HOST],
46
48
  password=config[ConfigKey.MYSQL_PASSWORD],
47
49
  port=config[ConfigKey.MYSQL_PORT],
48
50
  user=config[ConfigKey.MYSQL_USER],
49
51
  )
52
+ if config[ConfigKey.MYSQL_ALLOW_LOCAL_INFILE] is not None:
53
+ conn_kwargs['allow_local_infile'] = config[ConfigKey.MYSQL_ALLOW_LOCAL_INFILE]
54
+ return cls(**conn_kwargs)
50
55
 
51
56
  def build_create_table_command(
52
57
  self,
mage_ai/io/oracledb.py CHANGED
@@ -244,10 +244,8 @@ FETCH FIRST {limit} ROWS ONLY
244
244
  # Remove extraneous surrounding double quotes
245
245
  # that get added while performing conversion to string.
246
246
  df_[col] = df_[col].apply(lambda x: x.strip('"') if x and isinstance(x, str) else x)
247
- df_.replace({np.NaN: None}, inplace=True)
248
- values = []
249
- for i in range(0, len(df_)):
250
- values.append(tuple(df_.fillna('').values[i]))
247
+ df_.fillna('', inplace=True)
248
+ values = list(df_.itertuples(index=False, name=None))
251
249
 
252
250
  # Create values placeholder
253
251
  colmn_names = df.columns.tolist()
mage_ai/io/postgres.py CHANGED
@@ -99,6 +99,8 @@ class Postgres(BaseSQL):
99
99
  password = self.settings['password']
100
100
  port = self.settings['port']
101
101
  user = self.settings['user']
102
+ keepalives = self.settings.get('keepalives', 1)
103
+ keepalives_idle = self.settings.get('keepalives_idle', 300)
102
104
  if self.settings['connection_method'] == 'ssh_tunnel':
103
105
  ssh_setting = dict(ssh_username=self.settings['ssh_username'])
104
106
  if self.settings['ssh_pkey'] is not None:
@@ -128,19 +130,35 @@ class Postgres(BaseSQL):
128
130
  host = '127.0.0.1'
129
131
  port = self.ssh_tunnel.local_bind_port
130
132
 
131
- connect_opts = dict(
132
- database=database,
133
- host=host,
134
- password=password,
135
- port=port,
136
- user=user,
137
- keepalives=1,
138
- keepalives_idle=300,
133
+ connect_opts = self.settings.copy()
134
+ # See recognized keyword parameters for psycopg2.connect()
135
+ # https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS
136
+ unrecognized_keys = [
137
+ 'dbname',
138
+ 'schema',
139
+ 'connection_method',
140
+ 'ssh_host',
141
+ 'ssh_port',
142
+ 'ssh_username',
143
+ 'ssh_password',
144
+ 'ssh_pkey',
145
+ "verbose"
146
+ ]
147
+ for key in unrecognized_keys:
148
+ connect_opts.pop(key, None)
149
+
150
+ connect_opts.update(
151
+ {
152
+ 'database': database,
153
+ 'host': host,
154
+ 'password': password,
155
+ 'port': port,
156
+ 'user': user,
157
+ 'keepalives': keepalives,
158
+ 'keepalives_idle': keepalives_idle,
159
+ }
139
160
  )
140
161
 
141
- if self.settings.get('connect_timeout'):
142
- connect_opts['connect_timeout'] = self.settings['connect_timeout']
143
-
144
162
  try:
145
163
  self._ctx = connect(**connect_opts)
146
164
  except Exception:
@@ -277,6 +295,14 @@ class Postgres(BaseSQL):
277
295
 
278
296
  return 'text'
279
297
 
298
+ @staticmethod
299
+ def _clean_array_value(val: str) -> str:
300
+ if val is None or type(val) is not str or len(val) < 2:
301
+ return val
302
+ if val[0] == '[' and val[-1] == ']':
303
+ return str(val).replace('[', '{').replace(']', '}')
304
+ return val
305
+
280
306
  def upload_dataframe(
281
307
  self,
282
308
  cursor: _psycopg.cursor,
@@ -298,30 +324,26 @@ class Postgres(BaseSQL):
298
324
  # Use COPY command
299
325
  use_insert_command = False
300
326
 
301
- def clean_array_value(val):
302
- if val is None or type(val) is not str or len(val) < 2:
303
- return val
304
- if val[0] == '[' and val[-1] == ']':
305
- return '{' + val[1:-1] + '}'
306
- return val
307
-
308
327
  def serialize_obj(val):
309
328
  if type(val) is dict or type(val) is np.ndarray:
310
329
  return simplejson.dumps(
311
330
  val,
312
331
  default=encode_complex,
332
+ ensure_ascii=False,
313
333
  ignore_nan=True,
314
334
  )
315
335
  elif type(val) is list and len(val) >= 1 and type(val[0]) is dict:
316
336
  return simplejson.dumps(
317
337
  val,
318
338
  default=encode_complex,
339
+ ensure_ascii=False,
319
340
  ignore_nan=True,
320
341
  )
321
342
  elif not use_insert_command and type(val) is list:
322
- return clean_array_value(simplejson.dumps(
343
+ return self._clean_array_value(simplejson.dumps(
323
344
  val,
324
345
  default=encode_complex,
346
+ ensure_ascii=False,
325
347
  ignore_nan=True,
326
348
  ))
327
349
  return val
mage_ai/io/qdrant.py CHANGED
@@ -107,7 +107,7 @@ class Qdrant(BaseIO):
107
107
  Save data into Qdrant.
108
108
  Args:
109
109
  df (DataFrame): Data to export.
110
- document_column (str): Column name containinng documents to export.
110
+ document_column (str): Column name containing documents to export.
111
111
  id_column (str): Column name of the id. Default will use index in df.
112
112
  vector_column (str): Column name of the vector. Will use default
113
113
  encoder to auto generate query vector to auto generate query vector.
mage_ai/io/redshift.py CHANGED
@@ -5,6 +5,7 @@ from typing import Dict, Union
5
5
  from pandas import DataFrame
6
6
  from redshift_connector import connect
7
7
 
8
+ from mage_ai.data_preparation.models.block.sql.utils.shared import split_query_string
8
9
  from mage_ai.io.base import QUERY_ROW_LIMIT, ExportWritePolicy
9
10
  from mage_ai.io.config import BaseConfigLoader, ConfigKey
10
11
  from mage_ai.io.export_utils import clean_df_for_export, infer_dtypes
@@ -67,6 +68,18 @@ class Redshift(BaseSQL):
67
68
  with self.conn.cursor() as cur:
68
69
  cur.execute(query_string, **kwargs)
69
70
 
71
+ def execute_query_raw(self, query: str, **kwargs) -> None:
72
+ """
73
+ Overwrite execute query to process multiple queries in one string.
74
+ """
75
+ results = []
76
+ with self.conn.cursor() as cursor:
77
+ for query_string in split_query_string(query):
78
+ result = cursor.execute(query_string)
79
+ results.append(result)
80
+ self.conn.commit()
81
+ return results
82
+
70
83
  def load(
71
84
  self,
72
85
  query_string: str,
mage_ai/io/sql.py CHANGED
@@ -301,6 +301,7 @@ class BaseSQL(BaseSQLConnection):
301
301
  if_exists=if_exists,
302
302
  unique_conflict_method=unique_conflict_method,
303
303
  unique_constraints=unique_constraints,
304
+ **kwargs,
304
305
  )
305
306
  return
306
307
 
mage_ai/io/utils.py CHANGED
@@ -29,3 +29,21 @@ def escape_quotes(line: str, single: bool = True, double: bool = True) -> str:
29
29
  if double:
30
30
  new_line = new_line.replace('\"', '\\"')
31
31
  return new_line
32
+
33
+
34
+ def map_json_to_airtable(data_types):
35
+ # Extract the non-null type (ignoring 'null')
36
+ data_type = next((t for t in data_types if t != 'null'), 'string')
37
+
38
+ # Mapping from JSON types to Airtable types
39
+ type_mapping = {
40
+ 'string': 'multilineText',
41
+ 'integer': 'number',
42
+ 'boolean': 'checkbox',
43
+ 'array': 'multipleSelects',
44
+ 'object': 'singleCollaborator',
45
+ 'number': 'number',
46
+ 'date-time': 'dateTime'
47
+ }
48
+
49
+ return type_mapping.get(data_type, 'str')
@@ -1,5 +1,6 @@
1
1
  import logging
2
2
  import os
3
+ from functools import wraps
3
4
  from urllib.parse import parse_qs, quote_plus, urlparse
4
5
 
5
6
  import sqlalchemy
@@ -13,7 +14,7 @@ from mage_ai.orchestration.db.setup import get_postgres_connection_url
13
14
  from mage_ai.orchestration.db.utils import get_user_info_from_db_connection_url
14
15
  from mage_ai.settings import OTEL_EXPORTER_OTLP_ENDPOINT
15
16
  from mage_ai.settings.repo import get_variables_dir
16
- from mage_ai.shared.environments import is_debug, is_test
17
+ from mage_ai.shared.environments import is_debug, is_test_mage
17
18
 
18
19
  DB_RETRY_COUNT = 2
19
20
  TEST_DB = 'test.db'
@@ -28,7 +29,7 @@ db_kwargs = dict(
28
29
  if OTEL_EXPORTER_OTLP_ENDPOINT:
29
30
  from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
30
31
 
31
- if is_test():
32
+ if is_test_mage():
32
33
  db_connection_url = f'sqlite:///{TEST_DB}'
33
34
  db_kwargs['connect_args']['check_same_thread'] = False
34
35
  elif not db_connection_url:
@@ -37,7 +38,7 @@ elif not db_connection_url:
37
38
  if pg_db_connection_url:
38
39
  db_connection_url = pg_db_connection_url
39
40
  else:
40
- if is_test():
41
+ if is_test_mage():
41
42
  db_connection_url = f'sqlite:///{TEST_DB}'
42
43
  elif os.path.exists(os.path.join('mage_ai', 'orchestration', 'db')):
43
44
  # For local dev environment
@@ -169,6 +170,25 @@ def safe_db_query(func):
169
170
  return func_with_rollback
170
171
 
171
172
 
173
+ def safe_db_query_async(func):
174
+ @wraps(func)
175
+ async def func_with_rollback_async(*args, **kwargs):
176
+ retry_count = 0
177
+ while True:
178
+ try:
179
+ return await func(*args, **kwargs)
180
+ except (
181
+ sqlalchemy.exc.OperationalError,
182
+ sqlalchemy.exc.PendingRollbackError,
183
+ sqlalchemy.exc.InternalError,
184
+ ) as e:
185
+ db_connection.session.rollback()
186
+ if retry_count >= DB_RETRY_COUNT:
187
+ raise e
188
+ retry_count += 1
189
+ return func_with_rollback_async
190
+
191
+
172
192
  logging.basicConfig()
173
193
 
174
194
  if is_debug() and not os.getenv('DISABLE_DATABASE_TERMINAL_OUTPUT'):
@@ -0,0 +1,47 @@
1
+ """Create GenericJob
2
+
3
+ Revision ID: 39d36f1dab73
4
+ Revises: 0227396a216c
5
+ Create Date: 2025-08-27 20:45:29.756869
6
+
7
+ """
8
+ from alembic import op
9
+ import sqlalchemy as sa
10
+
11
+
12
+ # revision identifiers, used by Alembic.
13
+ revision = '39d36f1dab73'
14
+ down_revision = '0227396a216c'
15
+ branch_labels = None
16
+ depends_on = None
17
+
18
+
19
+ def upgrade() -> None:
20
+ # ### commands auto generated by Alembic - please adjust! ###
21
+ op.create_table('generic_job',
22
+ sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
23
+ sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
24
+ sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
25
+ sa.Column('job_type', sa.Enum('CANCEL_PIPELINE_RUN', name='jobtype'), nullable=False),
26
+ sa.Column('status', sa.Enum('INITIAL', 'QUEUED', 'RUNNING', 'COMPLETED', 'FAILED', name='jobstatus'), nullable=True),
27
+ sa.Column('payload', sa.JSON(), nullable=True),
28
+ sa.Column('extra_metadata', sa.JSON(), nullable=True),
29
+ sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
30
+ sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True),
31
+ sa.Column('repo_path', sa.String(length=255), nullable=True),
32
+ sa.PrimaryKeyConstraint('id')
33
+ )
34
+ with op.batch_alter_table('generic_job', schema=None) as batch_op:
35
+ batch_op.create_index(batch_op.f('ix_generic_job_job_type'), ['job_type'], unique=False)
36
+ batch_op.create_index(batch_op.f('ix_generic_job_status'), ['status'], unique=False)
37
+ # ### end Alembic commands ###
38
+
39
+
40
+ def downgrade() -> None:
41
+ # ### commands auto generated by Alembic - please adjust! ###
42
+ with op.batch_alter_table('generic_job', schema=None) as batch_op:
43
+ batch_op.drop_index(batch_op.f('ix_generic_job_status'))
44
+ batch_op.drop_index(batch_op.f('ix_generic_job_job_type'))
45
+
46
+ op.drop_table('generic_job')
47
+ # ### end Alembic commands ###
@@ -882,4 +882,5 @@ class Oauth2AccessToken(BaseModel):
882
882
  def is_valid(self) -> bool:
883
883
  return self.token and \
884
884
  self.expires and \
885
- self.expires >= datetime.utcnow().replace(tzinfo=self.expires.tzinfo)
885
+ self.expires >= datetime.utcnow().replace(tzinfo=self.expires.tzinfo) and \
886
+ self.user is not None
@@ -1,4 +1,3 @@
1
- import asyncio
2
1
  import collections
3
2
  import traceback
4
3
  import uuid
@@ -9,7 +8,6 @@ from typing import DefaultDict, Dict, List
9
8
 
10
9
  import dateutil.parser
11
10
  import pytz
12
- from croniter import croniter
13
11
  from dateutil.relativedelta import relativedelta
14
12
  from sqlalchemy import (
15
13
  JSON,
@@ -71,6 +69,7 @@ from mage_ai.server.kernel_output_parser import DataType
71
69
  from mage_ai.settings.platform import project_platform_activated
72
70
  from mage_ai.settings.repo import get_repo_path
73
71
  from mage_ai.shared.constants import ENV_PROD
72
+ from mage_ai.shared.croniter import croniter
74
73
  from mage_ai.shared.dates import compare
75
74
  from mage_ai.shared.enum import StrEnum
76
75
  from mage_ai.shared.hash import ignore_keys, index_by, merge_dict
@@ -1400,10 +1399,8 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
1400
1399
 
1401
1400
  from mage_ai.usage_statistics.logger import UsageStatisticLogger
1402
1401
 
1403
- asyncio.run(
1404
- UsageStatisticLogger().pipeline_runs_impression(
1405
- lambda: self.query.filter(self.status == self.PipelineRunStatus.COMPLETED).count(),
1406
- )
1402
+ UsageStatisticLogger().pipeline_runs_impression_sync(
1403
+ lambda: self.query.filter(self.status == self.PipelineRunStatus.COMPLETED).count(),
1407
1404
  )
1408
1405
 
1409
1406
  @safe_db_query
@@ -1807,6 +1804,111 @@ class BlockRun(BlockRunProjectPlatformMixin, BaseModel):
1807
1804
  )
1808
1805
 
1809
1806
 
1807
+ class GenericJob(BaseModel):
1808
+ class JobStatus(StrEnum):
1809
+ INITIAL = 'initial'
1810
+ QUEUED = 'queued'
1811
+ RUNNING = 'running'
1812
+ COMPLETED = 'completed'
1813
+ FAILED = 'failed'
1814
+
1815
+ class JobType(StrEnum):
1816
+ CANCEL_PIPELINE_RUN = 'cancel_pipeline_run'
1817
+
1818
+ job_type = Column(Enum(JobType), nullable=False, index=True)
1819
+ status = Column(Enum(JobStatus), default=JobStatus.INITIAL, index=True)
1820
+ payload = Column(JSON) # Job-specific data (e.g., pipeline_run_id)
1821
+ extra_metadata = Column(JSON) # Additional metadata for the job
1822
+ started_at = Column(DateTime(timezone=True))
1823
+ completed_at = Column(DateTime(timezone=True))
1824
+ repo_path = Column(String(255))
1825
+
1826
+ @classproperty
1827
+ def repo_query(cls):
1828
+ return cls.query.filter(
1829
+ or_(
1830
+ GenericJob.repo_path == get_repo_path(),
1831
+ GenericJob.repo_path.is_(None),
1832
+ )
1833
+ )
1834
+
1835
+ @classmethod
1836
+ @safe_db_query
1837
+ def enqueue_cancel_pipeline_run(
1838
+ cls,
1839
+ pipeline_run_id: int,
1840
+ cancelled_block_run_ids: List,
1841
+ repo_path: str = None,
1842
+ ) -> 'GenericJob':
1843
+ """Enqueue a job to cancel a pipeline run."""
1844
+ if repo_path is None:
1845
+ repo_path = get_repo_path()
1846
+
1847
+ job = cls.create(
1848
+ job_type=cls.JobType.CANCEL_PIPELINE_RUN,
1849
+ payload=dict(
1850
+ pipeline_run_id=pipeline_run_id,
1851
+ cancelled_block_run_ids=cancelled_block_run_ids,
1852
+ ),
1853
+ repo_path=repo_path,
1854
+ status=cls.JobStatus.INITIAL
1855
+ )
1856
+ return job
1857
+
1858
+ @classmethod
1859
+ @safe_db_query
1860
+ def get_jobs_with_initial_status(
1861
+ cls,
1862
+ limit: int = 10,
1863
+ repo_path: str = None,
1864
+ ) -> List['GenericJob']:
1865
+ """Get up to N jobs with INITIAL status for processing."""
1866
+ query = cls.repo_query.filter(
1867
+ cls.status == cls.JobStatus.INITIAL,
1868
+ )
1869
+
1870
+ if repo_path:
1871
+ query = query.filter(cls.repo_path == repo_path)
1872
+
1873
+ jobs = query.order_by(cls.created_at.asc()).limit(limit).all()
1874
+
1875
+ return jobs
1876
+
1877
+ @safe_db_query
1878
+ def mark_completed(self):
1879
+ """Mark the job as completed successfully."""
1880
+ self.update(
1881
+ status=self.JobStatus.COMPLETED,
1882
+ completed_at=datetime.now(tz=pytz.UTC)
1883
+ )
1884
+
1885
+ @safe_db_query
1886
+ def mark_failed(self, metadata: dict = None):
1887
+ """Mark the job as failed."""
1888
+ self.update(
1889
+ status=self.JobStatus.FAILED,
1890
+ completed_at=datetime.now(tz=pytz.UTC),
1891
+ extra_metadata=metadata or {}
1892
+ )
1893
+
1894
+ @safe_db_query
1895
+ def mark_queued(self):
1896
+ """Mark the job as queued."""
1897
+ self.update(
1898
+ status=self.JobStatus.QUEUED,
1899
+ )
1900
+
1901
+ @safe_db_query
1902
+ def mark_running(self):
1903
+ """Mark the job as queued."""
1904
+ self.update(
1905
+ status=self.JobStatus.RUNNING,
1906
+ )
1907
+
1908
+ def __repr__(self):
1909
+ return f'GenericJob(id={self.id}, type={self.job_type}, status={self.status})'
1910
+
1911
+
1810
1912
  class EventMatcher(BaseModel):
1811
1913
  class EventType(StrEnum):
1812
1914
  AWS_EVENT = 'aws_event'
@@ -5,7 +5,6 @@ from typing import Dict, List
5
5
 
6
6
  import dateutil.parser
7
7
  import pytz
8
- from croniter import croniter
9
8
  from dateutil.relativedelta import relativedelta
10
9
  from sqlalchemy import or_
11
10
  from sqlalchemy.sql import func
@@ -28,6 +27,7 @@ from mage_ai.settings.platform.utils import (
28
27
  get_pipeline_from_platform_async,
29
28
  )
30
29
  from mage_ai.shared.constants import ENV_PROD
30
+ from mage_ai.shared.croniter import croniter
31
31
  from mage_ai.shared.dates import compare
32
32
  from mage_ai.shared.hash import merge_dict
33
33
  from mage_ai.shared.utils import clean_name
@@ -3,7 +3,8 @@ from typing import Optional
3
3
  from sqlalchemy import Column, String, Text, UniqueConstraint, or_
4
4
 
5
5
  from mage_ai.orchestration.db import safe_db_query
6
- from mage_ai.orchestration.db.models.base import BaseModel
6
+ from mage_ai.orchestration.db.models.base import BaseModel, classproperty
7
+ from mage_ai.settings.repo import get_repo_path
7
8
 
8
9
 
9
10
  class Secret(BaseModel):
@@ -15,6 +16,15 @@ class Secret(BaseModel):
15
16
  key_uuid = Column(String(255), nullable=True)
16
17
  __table_args__ = (UniqueConstraint('name', 'key_uuid', name='name_key_uuid_uc'),)
17
18
 
19
+ @classproperty
20
+ def repo_query(cls):
21
+ return cls.query.filter(
22
+ or_(
23
+ Secret.repo_name == get_repo_path(),
24
+ Secret.repo_name.is_(None),
25
+ )
26
+ )
27
+
18
28
  @classmethod
19
29
  @safe_db_query
20
30
  def get_secret(cls, name: str, key_uuid: str) -> Optional['Secret']: