mage-ai 0.9.70__py3-none-any.whl → 0.9.71__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mage-ai might be problematic. Click here for more details.

Files changed (568) hide show
  1. mage_ai/ai/utils/xgboost.py +222 -0
  2. mage_ai/api/errors.py +37 -25
  3. mage_ai/api/operations/base.py +13 -1
  4. mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
  5. mage_ai/api/policies/BlockOutputPolicy.py +40 -17
  6. mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
  7. mage_ai/api/policies/KernelPolicy.py +55 -32
  8. mage_ai/api/policies/KernelProcessPolicy.py +56 -0
  9. mage_ai/api/policies/OutputPolicy.py +73 -41
  10. mage_ai/api/policies/PipelinePolicy.py +206 -138
  11. mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
  12. mage_ai/api/presenters/BlockPresenter.py +1 -1
  13. mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
  14. mage_ai/api/presenters/KernelPresenter.py +5 -26
  15. mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
  16. mage_ai/api/presenters/PipelinePresenter.py +18 -5
  17. mage_ai/api/presenters/StatusPresenter.py +2 -0
  18. mage_ai/api/presenters/SyncPresenter.py +25 -0
  19. mage_ai/api/resources/AutocompleteItemResource.py +1 -1
  20. mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
  21. mage_ai/api/resources/BlockOutputResource.py +42 -9
  22. mage_ai/api/resources/BlockResource.py +4 -3
  23. mage_ai/api/resources/BlockRunResource.py +27 -22
  24. mage_ai/api/resources/ClusterResource.py +4 -1
  25. mage_ai/api/resources/CustomTemplateResource.py +34 -14
  26. mage_ai/api/resources/DataProviderResource.py +1 -1
  27. mage_ai/api/resources/ExecutionStateResource.py +3 -1
  28. mage_ai/api/resources/FileContentResource.py +8 -2
  29. mage_ai/api/resources/FileResource.py +10 -4
  30. mage_ai/api/resources/FileVersionResource.py +3 -1
  31. mage_ai/api/resources/GitBranchResource.py +46 -9
  32. mage_ai/api/resources/GlobalDataProductResource.py +44 -7
  33. mage_ai/api/resources/GlobalHookResource.py +4 -1
  34. mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
  35. mage_ai/api/resources/IntegrationSourceResource.py +8 -4
  36. mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
  37. mage_ai/api/resources/KernelProcessResource.py +44 -0
  38. mage_ai/api/resources/KernelResource.py +25 -3
  39. mage_ai/api/resources/OutputResource.py +33 -11
  40. mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
  41. mage_ai/api/resources/PipelineInteractionResource.py +31 -15
  42. mage_ai/api/resources/PipelineResource.py +250 -123
  43. mage_ai/api/resources/PipelineRunResource.py +11 -3
  44. mage_ai/api/resources/PipelineScheduleResource.py +7 -2
  45. mage_ai/api/resources/PipelineTriggerResource.py +6 -1
  46. mage_ai/api/resources/ProjectResource.py +18 -7
  47. mage_ai/api/resources/SecretResource.py +1 -1
  48. mage_ai/api/resources/SeedResource.py +8 -1
  49. mage_ai/api/resources/StatusResource.py +21 -6
  50. mage_ai/api/resources/SyncResource.py +6 -8
  51. mage_ai/api/resources/VariableResource.py +46 -26
  52. mage_ai/api/resources/VersionControlProjectResource.py +9 -2
  53. mage_ai/api/resources/WidgetResource.py +1 -1
  54. mage_ai/api/resources/WorkspaceResource.py +1 -1
  55. mage_ai/api/views.py +47 -40
  56. mage_ai/authentication/permissions/seed.py +16 -2
  57. mage_ai/authentication/providers/oidc.py +21 -1
  58. mage_ai/autocomplete/utils.py +13 -9
  59. mage_ai/cache/base.py +1 -1
  60. mage_ai/cache/block.py +18 -12
  61. mage_ai/cache/block_action_object/__init__.py +32 -4
  62. mage_ai/cache/file.py +22 -19
  63. mage_ai/cache/pipeline.py +18 -12
  64. mage_ai/cli/main.py +1 -0
  65. mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
  66. mage_ai/cluster_manager/config.py +2 -2
  67. mage_ai/cluster_manager/manage.py +1 -1
  68. mage_ai/cluster_manager/workspace/base.py +1 -1
  69. mage_ai/command_center/applications/factory.py +10 -7
  70. mage_ai/command_center/files/factory.py +17 -15
  71. mage_ai/command_center/utils.py +25 -13
  72. mage_ai/data/__init__.py +0 -0
  73. mage_ai/data/constants.py +45 -0
  74. mage_ai/data/models/__init__.py +0 -0
  75. mage_ai/data/models/base.py +119 -0
  76. mage_ai/data/models/constants.py +1 -0
  77. mage_ai/data/models/generator.py +115 -0
  78. mage_ai/data/models/manager.py +168 -0
  79. mage_ai/data/models/pyarrow/__init__.py +0 -0
  80. mage_ai/data/models/pyarrow/record_batch.py +55 -0
  81. mage_ai/data/models/pyarrow/shared.py +21 -0
  82. mage_ai/data/models/pyarrow/table.py +8 -0
  83. mage_ai/data/models/reader.py +103 -0
  84. mage_ai/data/models/utils.py +59 -0
  85. mage_ai/data/models/writer.py +91 -0
  86. mage_ai/data/tabular/__init__.py +0 -0
  87. mage_ai/data/tabular/constants.py +23 -0
  88. mage_ai/data/tabular/mocks.py +19 -0
  89. mage_ai/data/tabular/models.py +126 -0
  90. mage_ai/data/tabular/reader.py +602 -0
  91. mage_ai/data/tabular/utils.py +102 -0
  92. mage_ai/data/tabular/writer.py +266 -0
  93. mage_ai/data/variables/__init__.py +0 -0
  94. mage_ai/data/variables/wrapper.py +54 -0
  95. mage_ai/data_cleaner/analysis/charts.py +61 -39
  96. mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
  97. mage_ai/data_cleaner/estimators/encoders.py +5 -2
  98. mage_ai/data_integrations/utils/scheduler.py +16 -11
  99. mage_ai/data_preparation/decorators.py +1 -0
  100. mage_ai/data_preparation/executors/block_executor.py +237 -155
  101. mage_ai/data_preparation/executors/streaming_pipeline_executor.py +1 -1
  102. mage_ai/data_preparation/git/__init__.py +27 -7
  103. mage_ai/data_preparation/git/api.py +7 -1
  104. mage_ai/data_preparation/git/utils.py +22 -16
  105. mage_ai/data_preparation/logging/logger_manager.py +4 -3
  106. mage_ai/data_preparation/models/block/__init__.py +1542 -878
  107. mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
  108. mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
  109. mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
  110. mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
  111. mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
  112. mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
  113. mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
  114. mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
  115. mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
  116. mage_ai/data_preparation/models/block/dynamic/utils.py +291 -168
  117. mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
  118. mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
  119. mage_ai/data_preparation/models/block/extension/utils.py +10 -1
  120. mage_ai/data_preparation/models/block/global_data_product/__init__.py +10 -1
  121. mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
  122. mage_ai/data_preparation/models/block/outputs.py +722 -0
  123. mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
  124. mage_ai/data_preparation/models/block/r/__init__.py +56 -38
  125. mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
  126. mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
  127. mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
  128. mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
  129. mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
  130. mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
  131. mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
  132. mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
  133. mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
  134. mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
  135. mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
  136. mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
  137. mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
  138. mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
  139. mage_ai/data_preparation/models/block/utils.py +127 -70
  140. mage_ai/data_preparation/models/constants.py +19 -14
  141. mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
  142. mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
  143. mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
  144. mage_ai/data_preparation/models/file.py +41 -28
  145. mage_ai/data_preparation/models/global_data_product/__init__.py +88 -58
  146. mage_ai/data_preparation/models/global_hooks/models.py +1 -0
  147. mage_ai/data_preparation/models/interfaces.py +29 -0
  148. mage_ai/data_preparation/models/pipeline.py +365 -180
  149. mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
  150. mage_ai/data_preparation/models/pipelines/seed.py +1 -1
  151. mage_ai/data_preparation/models/project/__init__.py +66 -18
  152. mage_ai/data_preparation/models/project/constants.py +2 -0
  153. mage_ai/data_preparation/models/triggers/__init__.py +120 -24
  154. mage_ai/data_preparation/models/utils.py +467 -17
  155. mage_ai/data_preparation/models/variable.py +1028 -137
  156. mage_ai/data_preparation/models/variables/__init__.py +0 -0
  157. mage_ai/data_preparation/models/variables/cache.py +149 -0
  158. mage_ai/data_preparation/models/variables/constants.py +72 -0
  159. mage_ai/data_preparation/models/variables/summarizer.py +336 -0
  160. mage_ai/data_preparation/models/variables/utils.py +77 -0
  161. mage_ai/data_preparation/models/widget/__init__.py +63 -41
  162. mage_ai/data_preparation/models/widget/charts.py +40 -27
  163. mage_ai/data_preparation/models/widget/constants.py +2 -0
  164. mage_ai/data_preparation/models/widget/utils.py +3 -3
  165. mage_ai/data_preparation/preferences.py +3 -3
  166. mage_ai/data_preparation/repo_manager.py +55 -21
  167. mage_ai/data_preparation/storage/base_storage.py +2 -2
  168. mage_ai/data_preparation/storage/gcs_storage.py +7 -4
  169. mage_ai/data_preparation/storage/local_storage.py +6 -3
  170. mage_ai/data_preparation/storage/s3_storage.py +5 -2
  171. mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
  172. mage_ai/data_preparation/variable_manager.py +281 -76
  173. mage_ai/io/base.py +3 -2
  174. mage_ai/io/bigquery.py +1 -0
  175. mage_ai/io/redshift.py +7 -5
  176. mage_ai/kernels/__init__.py +0 -0
  177. mage_ai/kernels/models.py +188 -0
  178. mage_ai/kernels/utils.py +169 -0
  179. mage_ai/orchestration/concurrency.py +6 -2
  180. mage_ai/orchestration/db/__init__.py +1 -0
  181. mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
  182. mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
  183. mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
  184. mage_ai/orchestration/db/models/oauth.py +2 -9
  185. mage_ai/orchestration/db/models/projects.py +10 -0
  186. mage_ai/orchestration/db/models/schedules.py +204 -187
  187. mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
  188. mage_ai/orchestration/db/models/utils.py +46 -5
  189. mage_ai/orchestration/metrics/pipeline_run.py +8 -9
  190. mage_ai/orchestration/notification/sender.py +1 -0
  191. mage_ai/orchestration/pipeline_scheduler_original.py +32 -8
  192. mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
  193. mage_ai/orchestration/run_status_checker.py +11 -4
  194. mage_ai/orchestration/triggers/api.py +12 -1
  195. mage_ai/presenters/charts/data_sources/base.py +4 -2
  196. mage_ai/presenters/charts/data_sources/block.py +15 -9
  197. mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
  198. mage_ai/presenters/charts/data_sources/constants.py +1 -0
  199. mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
  200. mage_ai/presenters/interactions/models.py +11 -7
  201. mage_ai/presenters/pages/loaders/pipelines.py +5 -3
  202. mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
  203. mage_ai/presenters/utils.py +2 -0
  204. mage_ai/server/api/blocks.py +2 -1
  205. mage_ai/server/api/downloads.py +5 -1
  206. mage_ai/server/api/triggers.py +3 -1
  207. mage_ai/server/constants.py +1 -1
  208. mage_ai/server/frontend_dist/404.html +5 -5
  209. mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
  210. mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
  211. mage_ai/server/frontend_dist/_next/static/chunks/{1557-b3502f3f1aa92ac7.js → 1557-25a7d985d5564fd3.js} +1 -1
  212. mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
  213. mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
  214. mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
  215. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/3763-61b542dafdbf5754.js → frontend_dist/_next/static/chunks/3763-40780c6d1e4b261d.js} +1 -1
  216. mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
  217. mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
  218. mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
  219. mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
  220. mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
  221. mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
  222. mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
  223. mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
  224. mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
  225. mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
  226. mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
  227. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
  228. mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
  229. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
  230. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
  231. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/overview-597b74828bf105db.js → frontend_dist/_next/static/chunks/pages/overview-9f1ac4ec003884f3.js} +1 -1
  232. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-7181b086c93784d2.js → [...slug]-7e737f6fc7e83e9b.js} +1 -1
  233. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
  234. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
  235. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
  236. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
  237. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
  238. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
  239. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
  240. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
  241. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js → frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js} +1 -1
  242. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
  243. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
  244. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
  245. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
  246. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
  247. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
  248. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
  249. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
  250. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
  251. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
  252. mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
  253. mage_ai/server/frontend_dist/block-layout.html +3 -3
  254. mage_ai/server/frontend_dist/compute.html +6 -6
  255. mage_ai/server/frontend_dist/files.html +6 -6
  256. mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
  257. mage_ai/server/frontend_dist/global-data-products.html +6 -6
  258. mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
  259. mage_ai/server/frontend_dist/global-hooks.html +6 -6
  260. mage_ai/server/frontend_dist/index.html +3 -3
  261. mage_ai/server/frontend_dist/manage/files.html +6 -6
  262. mage_ai/server/frontend_dist/manage/settings.html +6 -6
  263. mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
  264. mage_ai/server/frontend_dist/manage/users/new.html +6 -6
  265. mage_ai/server/frontend_dist/manage/users.html +6 -6
  266. mage_ai/server/frontend_dist/manage.html +6 -6
  267. mage_ai/server/frontend_dist/oauth.html +5 -5
  268. mage_ai/server/frontend_dist/overview.html +6 -6
  269. mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
  270. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  271. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
  272. mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
  273. mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
  274. mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
  275. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  276. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  277. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
  278. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
  279. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
  280. mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
  281. mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
  282. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  283. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
  284. mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
  285. mage_ai/server/frontend_dist/pipelines.html +6 -6
  286. mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
  287. mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
  288. mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
  289. mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
  290. mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
  291. mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
  292. mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
  293. mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
  294. mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
  295. mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
  296. mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
  297. mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
  298. mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
  299. mage_ai/server/frontend_dist/settings.html +3 -3
  300. mage_ai/server/frontend_dist/sign-in.html +12 -12
  301. mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
  302. mage_ai/server/frontend_dist/templates.html +6 -6
  303. mage_ai/server/frontend_dist/terminal.html +6 -6
  304. mage_ai/server/frontend_dist/test.html +3 -3
  305. mage_ai/server/frontend_dist/triggers.html +6 -6
  306. mage_ai/server/frontend_dist/version-control.html +6 -6
  307. mage_ai/server/frontend_dist_base_path_template/404.html +5 -5
  308. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
  309. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{1557-b3502f3f1aa92ac7.js → 1557-25a7d985d5564fd3.js} +1 -1
  310. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
  311. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
  312. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
  313. mage_ai/server/{frontend_dist/_next/static/chunks/3763-61b542dafdbf5754.js → frontend_dist_base_path_template/_next/static/chunks/3763-40780c6d1e4b261d.js} +1 -1
  314. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
  315. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
  316. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
  317. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
  318. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
  319. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
  320. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
  321. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
  322. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
  323. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
  324. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
  325. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
  326. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
  327. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
  328. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
  329. mage_ai/server/{frontend_dist/_next/static/chunks/pages/overview-597b74828bf105db.js → frontend_dist_base_path_template/_next/static/chunks/pages/overview-9f1ac4ec003884f3.js} +1 -1
  330. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-7181b086c93784d2.js → [...slug]-7e737f6fc7e83e9b.js} +1 -1
  331. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
  332. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
  333. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
  334. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
  335. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
  336. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
  337. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
  338. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
  339. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js} +1 -1
  340. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
  341. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
  342. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
  343. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
  344. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
  345. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
  346. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
  347. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
  348. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
  349. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
  350. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
  351. mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
  352. mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
  353. mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
  354. mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
  355. mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
  356. mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
  357. mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
  358. mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
  359. mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
  360. mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
  361. mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
  362. mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
  363. mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
  364. mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
  365. mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
  366. mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
  367. mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
  368. mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
  369. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  370. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
  371. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
  372. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
  373. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
  374. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  375. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  376. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
  377. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
  378. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
  379. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
  380. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
  381. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  382. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
  383. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
  384. mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
  385. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
  386. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
  387. mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
  388. mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
  389. mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
  390. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
  391. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
  392. mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
  393. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
  394. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
  395. mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
  396. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
  397. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
  398. mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
  399. mage_ai/server/frontend_dist_base_path_template/sign-in.html +12 -12
  400. mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
  401. mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
  402. mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
  403. mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
  404. mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
  405. mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
  406. mage_ai/server/kernel_output_parser.py +4 -1
  407. mage_ai/server/scheduler_manager.py +9 -0
  408. mage_ai/server/server.py +35 -31
  409. mage_ai/server/utils/custom_output.py +284 -0
  410. mage_ai/server/utils/execute_custom_code.py +245 -0
  411. mage_ai/server/utils/output_display.py +123 -289
  412. mage_ai/server/websocket_server.py +116 -69
  413. mage_ai/services/k8s/config.py +23 -0
  414. mage_ai/services/k8s/job_manager.py +6 -1
  415. mage_ai/services/ssh/aws/emr/utils.py +8 -8
  416. mage_ai/settings/keys/auth.py +1 -0
  417. mage_ai/settings/platform/__init__.py +159 -38
  418. mage_ai/settings/platform/constants.py +5 -0
  419. mage_ai/settings/platform/utils.py +53 -10
  420. mage_ai/settings/repo.py +26 -12
  421. mage_ai/settings/server.py +128 -37
  422. mage_ai/shared/array.py +24 -1
  423. mage_ai/shared/complex.py +45 -0
  424. mage_ai/shared/config.py +2 -1
  425. mage_ai/shared/custom_logger.py +11 -0
  426. mage_ai/shared/dates.py +10 -6
  427. mage_ai/shared/files.py +63 -8
  428. mage_ai/shared/hash.py +33 -9
  429. mage_ai/shared/io.py +9 -5
  430. mage_ai/shared/models.py +82 -24
  431. mage_ai/shared/outputs.py +87 -0
  432. mage_ai/shared/parsers.py +141 -15
  433. mage_ai/shared/path_fixer.py +11 -7
  434. mage_ai/shared/singletons/__init__.py +0 -0
  435. mage_ai/shared/singletons/base.py +47 -0
  436. mage_ai/shared/singletons/memory.py +38 -0
  437. mage_ai/shared/strings.py +34 -1
  438. mage_ai/shared/yaml.py +24 -0
  439. mage_ai/streaming/sinks/oracledb.py +57 -0
  440. mage_ai/streaming/sinks/sink_factory.py +4 -0
  441. mage_ai/system/__init__.py +0 -0
  442. mage_ai/system/constants.py +14 -0
  443. mage_ai/system/memory/__init__.py +0 -0
  444. mage_ai/system/memory/constants.py +1 -0
  445. mage_ai/system/memory/manager.py +174 -0
  446. mage_ai/system/memory/presenters.py +158 -0
  447. mage_ai/system/memory/process.py +216 -0
  448. mage_ai/system/memory/samples.py +13 -0
  449. mage_ai/system/memory/utils.py +656 -0
  450. mage_ai/system/memory/wrappers.py +177 -0
  451. mage_ai/system/models.py +58 -0
  452. mage_ai/system/storage/__init__.py +0 -0
  453. mage_ai/system/storage/utils.py +29 -0
  454. mage_ai/tests/api/endpoints/mixins.py +2 -2
  455. mage_ai/tests/api/endpoints/test_blocks.py +2 -1
  456. mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
  457. mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
  458. mage_ai/tests/api/endpoints/test_projects.py +2 -1
  459. mage_ai/tests/api/operations/base/test_base.py +27 -27
  460. mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
  461. mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
  462. mage_ai/tests/api/operations/test_syncs.py +6 -4
  463. mage_ai/tests/api/resources/test_pipeline_resource.py +9 -2
  464. mage_ai/tests/authentication/providers/test_oidc.py +59 -0
  465. mage_ai/tests/base_test.py +2 -2
  466. mage_ai/tests/data/__init__.py +0 -0
  467. mage_ai/tests/data/models/__init__.py +0 -0
  468. mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
  469. mage_ai/tests/data_preparation/git/test_git.py +4 -1
  470. mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
  471. mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
  472. mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
  473. mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
  474. mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
  475. mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
  476. mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
  477. mage_ai/tests/data_preparation/models/block/test_global_data_product.py +3 -2
  478. mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
  479. mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
  480. mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
  481. mage_ai/tests/data_preparation/models/test_block.py +115 -120
  482. mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
  483. mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
  484. mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
  485. mage_ai/tests/data_preparation/models/test_project.py +4 -1
  486. mage_ai/tests/data_preparation/models/test_utils.py +80 -0
  487. mage_ai/tests/data_preparation/models/test_variable.py +242 -69
  488. mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
  489. mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
  490. mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
  491. mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
  492. mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
  493. mage_ai/tests/factory.py +64 -43
  494. mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
  495. mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
  496. mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
  497. mage_ai/tests/orchestration/triggers/test_global_data_product.py +3 -2
  498. mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
  499. mage_ai/tests/services/k8s/test_job_manager.py +18 -0
  500. mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
  501. mage_ai/tests/test_shared.py +61 -0
  502. mage_ai/usage_statistics/logger.py +7 -2
  503. mage_ai/utils/code.py +33 -19
  504. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/METADATA +5 -2
  505. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/RECORD +513 -417
  506. mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
  507. mage_ai/server/frontend_dist/_next/static/RhDiJSkcjCsh4xxX4BFBk/_buildManifest.js +0 -1
  508. mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
  509. mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
  510. mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
  511. mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
  512. mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
  513. mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
  514. mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
  515. mage_ai/server/frontend_dist/_next/static/chunks/7966-b9b85ba10667e654.js +0 -1
  516. mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
  517. mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
  518. mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
  519. mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
  520. mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
  521. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +0 -1
  522. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
  523. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
  524. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
  525. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +0 -1
  526. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
  527. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
  528. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
  529. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +0 -1
  530. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
  531. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +0 -1
  532. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +0 -1
  533. mage_ai/server/frontend_dist_base_path_template/_next/static/TdpLLFome13qvM0gXvpHs/_buildManifest.js +0 -1
  534. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
  535. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
  536. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
  537. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
  538. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
  539. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
  540. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
  541. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-b9b85ba10667e654.js +0 -1
  542. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
  543. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
  544. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
  545. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
  546. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
  547. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +0 -1
  548. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
  549. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
  550. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
  551. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +0 -1
  552. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
  553. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
  554. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
  555. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +0 -1
  556. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
  557. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +0 -1
  558. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +0 -1
  559. mage_ai/shared/memory.py +0 -90
  560. mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
  561. /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
  562. /mage_ai/server/frontend_dist/_next/static/{RhDiJSkcjCsh4xxX4BFBk → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
  563. /mage_ai/server/frontend_dist_base_path_template/_next/static/{TdpLLFome13qvM0gXvpHs → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
  564. /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
  565. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
  566. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
  567. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
  568. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
@@ -39,13 +39,22 @@ class PipelineScheduleProjectPlatformMixin:
39
39
  def repo_query_project_platform(cls):
40
40
  repo_paths = []
41
41
 
42
- queries = Project().repo_path_for_database_query('pipeline_schedules')
42
+ context_data = dict()
43
+ queries = Project(
44
+ context_data=context_data,
45
+ ).repo_path_for_database_query('pipeline_schedules')
43
46
  if queries:
44
47
  repo_paths.extend(queries)
45
48
 
46
- repo_paths.extend([d.get(
47
- 'full_path',
48
- ) for d in build_repo_path_for_all_projects(mage_projects_only=True).values()])
49
+ repo_paths.extend([
50
+ d.get(
51
+ 'full_path',
52
+ )
53
+ for d in build_repo_path_for_all_projects(
54
+ context_data=context_data,
55
+ mage_projects_only=True,
56
+ ).values()
57
+ ])
49
58
 
50
59
  return cls.query.filter(
51
60
  or_(
@@ -149,14 +158,11 @@ class PipelineScheduleProjectPlatformMixin:
149
158
 
150
159
  pipeline_use = pipeline or self.pipeline
151
160
  if not pipeline_use:
152
- try:
153
- Pipeline.get(self.pipeline_uuid)
154
- except Exception:
155
- print(
156
- f'[WARNING] Pipeline {self.pipeline_uuid} cannot be found '
157
- + f'for pipeline schedule ID {self.id}.',
158
- )
159
- return False
161
+ print(
162
+ f'[WARNING] Pipeline {self.pipeline_uuid} cannot be found '
163
+ + f'for pipeline schedule ID {self.id}.',
164
+ )
165
+ return False
160
166
 
161
167
  if self.schedule_interval == ScheduleInterval.ONCE:
162
168
  pipeline_run_count = self.pipeline_runs_count
@@ -1,13 +1,14 @@
1
1
  from mage_ai.orchestration.db import db_connection, safe_db_query
2
- from mage_ai.orchestration.db.models.schedules import (
3
- Backfill,
4
- PipelineRun,
5
- PipelineSchedule,
6
- )
7
2
 
8
3
 
9
4
  @safe_db_query
10
5
  def transfer_related_models_for_pipeline(old_uuid: str, new_uuid: str):
6
+ from mage_ai.orchestration.db.models.schedules import (
7
+ Backfill,
8
+ PipelineRun,
9
+ PipelineSchedule,
10
+ )
11
+
11
12
  # Migrate pipeline schedules
12
13
  PipelineSchedule.query.filter(PipelineSchedule.pipeline_uuid == old_uuid).update({
13
14
  PipelineSchedule.pipeline_uuid: new_uuid
@@ -21,3 +22,43 @@ def transfer_related_models_for_pipeline(old_uuid: str, new_uuid: str):
21
22
  Backfill.pipeline_uuid: new_uuid
22
23
  }, synchronize_session=False)
23
24
  db_connection.session.commit()
25
+
26
+
27
+ @safe_db_query
28
+ def get_active_project_for_user(user, root_project_uuid: str) -> str:
29
+ from mage_ai.orchestration.db.models.projects import UserProject
30
+ return UserProject.query.filter(
31
+ UserProject.user_id == user.id,
32
+ UserProject.root_project_uuid == root_project_uuid,
33
+ UserProject.active.is_(True),
34
+ ).one_or_none()
35
+
36
+
37
+ @safe_db_query
38
+ def activate_project_for_user(user, root_project_uuid: str, project_name: str):
39
+ from mage_ai.orchestration.db.models.projects import UserProject
40
+ projects = UserProject.query.filter(
41
+ UserProject.user_id == user.id,
42
+ UserProject.root_project_uuid == root_project_uuid,
43
+ )
44
+
45
+ projects.update({
46
+ UserProject.active: False
47
+ }, synchronize_session=False)
48
+
49
+ project_updated = False
50
+ for project in projects:
51
+ if project.project_name == project_name:
52
+ project.update(active=True, commit=False)
53
+ project_updated = True
54
+
55
+ if not project_updated:
56
+ UserProject.create(
57
+ user_id=user.id,
58
+ root_project_uuid=root_project_uuid,
59
+ project_name=project_name,
60
+ active=True,
61
+ commit=False,
62
+ )
63
+
64
+ db_connection.session.commit()
@@ -4,9 +4,6 @@ from copy import deepcopy
4
4
  from typing import Dict, List, Tuple
5
5
 
6
6
  from mage_ai.data_preparation.models.constants import PipelineType
7
- from mage_ai.data_preparation.models.pipelines.integration_pipeline import (
8
- IntegrationPipeline,
9
- )
10
7
  from mage_ai.orchestration.db.models.schedules import BlockRun, PipelineRun
11
8
  from mage_ai.shared.hash import merge_dict
12
9
 
@@ -40,6 +37,7 @@ KEY_TO_METRICS = {
40
37
 
41
38
  def calculate_pipeline_run_metrics(
42
39
  pipeline_run: PipelineRun,
40
+ pipeline,
43
41
  logger=None,
44
42
  logging_tags: Dict = None,
45
43
  ) -> Dict:
@@ -54,7 +52,7 @@ def calculate_pipeline_run_metrics(
54
52
  **logging_tags,
55
53
  )
56
54
  try:
57
- __calculate_metrics(pipeline_run)
55
+ __calculate_metrics(pipeline_run, pipeline)
58
56
  if logger:
59
57
  logger.info(
60
58
  f'Calculate metrics for pipeline run {pipeline_run.id} completed.',
@@ -72,6 +70,7 @@ def calculate_pipeline_run_metrics(
72
70
  def calculate_source_metrics(
73
71
  pipeline_run: PipelineRun,
74
72
  block_run: BlockRun,
73
+ pipeline,
75
74
  stream: str,
76
75
  logger=None,
77
76
  logging_tags: Dict = None,
@@ -79,6 +78,7 @@ def calculate_source_metrics(
79
78
  return __calculate_block_metrics(
80
79
  pipeline_run,
81
80
  block_run,
81
+ pipeline,
82
82
  stream,
83
83
  KEY_SOURCE,
84
84
  logger=logger,
@@ -89,6 +89,7 @@ def calculate_source_metrics(
89
89
  def calculate_destination_metrics(
90
90
  pipeline_run: PipelineRun,
91
91
  block_run: BlockRun,
92
+ pipeline,
92
93
  stream: str,
93
94
  logger=None,
94
95
  logging_tags: Dict = None,
@@ -96,6 +97,7 @@ def calculate_destination_metrics(
96
97
  return __calculate_block_metrics(
97
98
  pipeline_run,
98
99
  block_run,
100
+ pipeline,
99
101
  stream,
100
102
  KEY_DESTINATION,
101
103
  logger=logger,
@@ -106,6 +108,7 @@ def calculate_destination_metrics(
106
108
  def __calculate_block_metrics(
107
109
  pipeline_run: PipelineRun,
108
110
  block_run: BlockRun,
111
+ pipeline,
109
112
  stream: str,
110
113
  key: str,
111
114
  logger=None,
@@ -135,8 +138,6 @@ def __calculate_block_metrics(
135
138
  **logging_tags,
136
139
  )
137
140
  try:
138
- pipeline = IntegrationPipeline.get(pipeline_run.pipeline_uuid)
139
-
140
141
  logs_arr = block_run.logs['content'].split('\n')
141
142
  logs_by_uuid = {key: [logs_arr]}
142
143
  metrics = get_metrics(
@@ -192,7 +193,7 @@ def __calculate_block_metrics(
192
193
  )
193
194
 
194
195
 
195
- def __calculate_metrics(pipeline_run: PipelineRun) -> Dict:
196
+ def __calculate_metrics(pipeline_run: PipelineRun, pipeline) -> Dict:
196
197
  """
197
198
  Calculate metrics for an integration pipeline run. Only the "pipeline" field
198
199
  in the metrics will be updated by calling this method.
@@ -205,8 +206,6 @@ def __calculate_metrics(pipeline_run: PipelineRun) -> Dict:
205
206
  Returns:
206
207
  Dict: The calculated metrics.
207
208
  """
208
- pipeline = IntegrationPipeline.get(pipeline_run.pipeline_uuid)
209
-
210
209
  if PipelineType.INTEGRATION != pipeline.type:
211
210
  return
212
211
 
@@ -179,6 +179,7 @@ class NotificationSender:
179
179
  pipeline_run_url=self.__pipeline_run_url(pipeline, pipeline_run),
180
180
  pipeline_schedule_id=pipeline_run.pipeline_schedule.id,
181
181
  pipeline_schedule_name=pipeline_run.pipeline_schedule.name,
182
+ pipeline_schedule_description=pipeline_run.pipeline_schedule.description,
182
183
  pipeline_uuid=pipeline.uuid,
183
184
  stacktrace=stacktrace,
184
185
  )
@@ -23,7 +23,7 @@ from mage_ai.data_preparation.models.triggers import (
23
23
  ScheduleInterval,
24
24
  ScheduleStatus,
25
25
  ScheduleType,
26
- get_triggers_by_pipeline,
26
+ get_triggers_by_pipeline_with_cache,
27
27
  )
28
28
  from mage_ai.data_preparation.repo_manager import get_repo_config
29
29
  from mage_ai.data_preparation.sync.git_sync import get_sync_config
@@ -155,10 +155,12 @@ class PipelineScheduler:
155
155
  if is_integration:
156
156
  clear_source_output_files(
157
157
  self.pipeline_run,
158
+ self.pipeline,
158
159
  self.logger,
159
160
  )
160
161
  initialize_state_and_runs(
161
162
  self.pipeline_run,
163
+ self.pipeline,
162
164
  self.logger,
163
165
  self.pipeline_run.get_variables(),
164
166
  )
@@ -228,6 +230,7 @@ class PipelineScheduler:
228
230
  tags = self.build_tags()
229
231
  calculate_pipeline_run_metrics(
230
232
  self.pipeline_run,
233
+ self.pipeline,
231
234
  logger=self.logger,
232
235
  logging_tags=tags,
233
236
  )
@@ -463,6 +466,7 @@ class PipelineScheduler:
463
466
 
464
467
  calculate_pipeline_run_metrics(
465
468
  self.pipeline_run,
469
+ self.pipeline,
466
470
  logger=self.logger,
467
471
  logging_tags=tags,
468
472
  )
@@ -485,6 +489,7 @@ class PipelineScheduler:
485
489
  if PipelineType.INTEGRATION == self.pipeline.type:
486
490
  calculate_pipeline_run_metrics(
487
491
  self.pipeline_run,
492
+ self.pipeline,
488
493
  logger=self.logger,
489
494
  logging_tags=tags,
490
495
  )
@@ -1069,6 +1074,7 @@ def run_integration_stream(
1069
1074
  calculate_source_metrics(
1070
1075
  pipeline_run,
1071
1076
  block_run,
1077
+ pipeline,
1072
1078
  stream=tap_stream_id,
1073
1079
  logger=pipeline_scheduler.logger,
1074
1080
  logging_tags=merge_dict(tags_updated, dict(tags=tags2)),
@@ -1077,6 +1083,7 @@ def run_integration_stream(
1077
1083
  calculate_destination_metrics(
1078
1084
  pipeline_run,
1079
1085
  block_run,
1086
+ pipeline,
1080
1087
  stream=tap_stream_id,
1081
1088
  logger=pipeline_scheduler.logger,
1082
1089
  logging_tags=merge_dict(tags_updated, dict(tags=tags2)),
@@ -1163,7 +1170,7 @@ def run_block(
1163
1170
  else:
1164
1171
  repo_path = get_repo_path()
1165
1172
  retry_config = merge_dict(
1166
- get_repo_config(repo_path).retry_config or dict(),
1173
+ get_repo_config(repo_path=repo_path).retry_config or dict(),
1167
1174
  block.retry_config or dict(),
1168
1175
  )
1169
1176
 
@@ -1253,9 +1260,10 @@ def configure_pipeline_run_payload(
1253
1260
  @safe_db_query
1254
1261
  def retry_pipeline_run(
1255
1262
  pipeline_run: Dict,
1263
+ repo_path: str,
1256
1264
  ) -> 'PipelineRun':
1257
1265
  pipeline_uuid = pipeline_run['pipeline_uuid']
1258
- pipeline = Pipeline.get(pipeline_uuid, check_if_exists=True)
1266
+ pipeline = Pipeline.get(pipeline_uuid, check_if_exists=True, repo_path=repo_path)
1259
1267
  if pipeline is None or not pipeline.is_valid_pipeline(pipeline.dir_path):
1260
1268
  raise Exception(f'Pipeline {pipeline_uuid} is not a valid pipeline.')
1261
1269
 
@@ -1399,7 +1407,7 @@ def check_sla():
1399
1407
  else pipeline_run.created_at
1400
1408
  if compare(start_date + timedelta(seconds=sla), current_time) == -1:
1401
1409
  # passed SLA for pipeline_run
1402
- pipeline = Pipeline.get(pipeline_schedule.pipeline_uuid)
1410
+ pipeline = pipeline_schedule.pipeline
1403
1411
  notification_sender = NotificationSender(
1404
1412
  NotificationConfig.load(
1405
1413
  config=merge_dict(
@@ -1436,7 +1444,7 @@ def schedule_all():
1436
1444
  schedules, so that needs to be addressed at some point.
1437
1445
  """
1438
1446
  db_connection.session.expire_all()
1439
-
1447
+ repo_config = get_repo_config()
1440
1448
  repo_pipelines = set(Pipeline.get_all_pipelines_all_projects(
1441
1449
  get_repo_path(),
1442
1450
  disable_pipelines_folder_creation=True,
@@ -1501,6 +1509,7 @@ def schedule_all():
1501
1509
 
1502
1510
  for pipeline_uuid, pipeline, active_schedules in gen_pipeline_with_schedules(
1503
1511
  active_pipeline_schedules,
1512
+ repo_config=repo_config,
1504
1513
  ):
1505
1514
  concurrency_config = ConcurrencyConfig.load(config=pipeline.concurrency_config)
1506
1515
  pipeline_runs_to_start = []
@@ -1531,6 +1540,7 @@ def schedule_all():
1531
1540
  should_schedule = pipeline_schedule.should_schedule(
1532
1541
  previous_runtimes=previous_runtimes,
1533
1542
  pipeline=pipeline,
1543
+ repo_config=repo_config,
1534
1544
  )
1535
1545
  initial_pipeline_runs = [
1536
1546
  r for r in pipeline_schedule.pipeline_runs
@@ -1668,6 +1678,7 @@ def schedule_all():
1668
1678
 
1669
1679
  def gen_pipeline_with_schedules_single_project(
1670
1680
  active_pipeline_schedules: List[PipelineSchedule],
1681
+ repo_config=None,
1671
1682
  ) -> Generator[Tuple[str, Pipeline, List[PipelineSchedule]], None, None]:
1672
1683
  """
1673
1684
  Generate pipelines with associated schedules for a single project.
@@ -1681,6 +1692,8 @@ def gen_pipeline_with_schedules_single_project(
1681
1692
  - The pipeline object.
1682
1693
  - A list of active schedules associated with the pipeline.
1683
1694
  """
1695
+ # Avoid calling get_repo_config repeatedly in Pipeline init method.
1696
+ repo_config = repo_config or get_repo_config()
1684
1697
  pipeline_schedules_by_pipeline = collections.defaultdict(list)
1685
1698
  for schedule in active_pipeline_schedules:
1686
1699
  pipeline_schedules_by_pipeline[schedule.pipeline_uuid].append(schedule)
@@ -1689,15 +1702,21 @@ def gen_pipeline_with_schedules_single_project(
1689
1702
  # each pipeline.
1690
1703
  for pipeline_uuid, active_schedules in pipeline_schedules_by_pipeline.items():
1691
1704
  try:
1692
- pipeline = Pipeline.get(pipeline_uuid)
1705
+ pipeline = Pipeline.get(
1706
+ pipeline_uuid,
1707
+ repo_path=get_repo_path(),
1708
+ repo_config=repo_config,
1709
+ )
1693
1710
  except Exception as e:
1694
1711
  print(f'Error fetching pipeline {pipeline_uuid}: {e}')
1712
+ traceback.print_exc()
1695
1713
  continue
1696
1714
  yield pipeline_uuid, pipeline, active_schedules
1697
1715
 
1698
1716
 
1699
1717
  def gen_pipeline_with_schedules_project_platform(
1700
1718
  active_pipeline_schedules: List[PipelineSchedule],
1719
+ repo_config=None,
1701
1720
  ) -> Generator[Tuple[str, Pipeline, List[PipelineSchedule]], None, None]:
1702
1721
  """
1703
1722
  Generate pipelines with associated schedules for project platform.
@@ -1761,6 +1780,7 @@ def gen_pipeline_with_schedules_project_platform(
1761
1780
  )
1762
1781
  except Exception as e:
1763
1782
  print(f'Error fetching pipeline {pipeline_uuid}: {e}')
1783
+ traceback.print_exc()
1764
1784
  continue
1765
1785
  yield pipeline_uuid, pipeline, active_schedules
1766
1786
 
@@ -1812,8 +1832,12 @@ def sync_schedules(pipeline_uuids: List[str]):
1812
1832
 
1813
1833
  # Sync schedule configs from triggers.yaml to DB
1814
1834
  for pipeline_uuid in pipeline_uuids:
1815
- pipeline_triggers = get_triggers_by_pipeline(pipeline_uuid)
1816
- logger.debug(f'Sync pipeline trigger configs for {pipeline_uuid}: {pipeline_triggers}.')
1835
+ pipeline_triggers, from_cache = get_triggers_by_pipeline_with_cache(pipeline_uuid)
1836
+ logger.debug(f'Sync pipeline trigger configs for {pipeline_uuid}: {pipeline_triggers}.'
1837
+ f'From cache: {from_cache}')
1838
+ if from_cache:
1839
+ # No updates need to be made to database
1840
+ continue
1817
1841
  for pipeline_trigger in pipeline_triggers:
1818
1842
  if pipeline_trigger.envs and get_env() not in pipeline_trigger.envs:
1819
1843
  continue
@@ -1120,7 +1120,7 @@ def run_block(
1120
1120
  repo_path = get_repo_path()
1121
1121
 
1122
1122
  retry_config = merge_dict(
1123
- get_repo_config(repo_path).retry_config or dict(),
1123
+ get_repo_config(repo_path=repo_path).retry_config or dict(),
1124
1124
  block.retry_config or dict(),
1125
1125
  )
1126
1126
 
@@ -1,6 +1,7 @@
1
1
  from datetime import datetime, timedelta, timezone
2
2
 
3
3
  from mage_ai.data_preparation.models.pipeline import Pipeline
4
+ from mage_ai.data_preparation.models.utils import warn_for_repo_path
4
5
  from mage_ai.orchestration.db.models.schedules import (
5
6
  BlockRun,
6
7
  PipelineRun,
@@ -23,10 +24,11 @@ BLOCK_FAILURE_STATUSES = [
23
24
  def check_status(
24
25
  pipeline_uuid: str,
25
26
  execution_date: datetime,
27
+ repo_path: str = None,
26
28
  block_uuid: str = None,
27
29
  hours: int = 24,
28
30
  ) -> bool:
29
- __validate_pipeline_and_block(pipeline_uuid, block_uuid)
31
+ __validate_pipeline_and_block(pipeline_uuid, block_uuid, repo_path=repo_path)
30
32
 
31
33
  execution_date = execution_date.replace(tzinfo=timezone.utc)
32
34
  pipeline_run = (
@@ -62,11 +64,16 @@ def check_status(
62
64
  return pipeline_run.status == PipelineRun.PipelineRunStatus.COMPLETED
63
65
 
64
66
 
65
- def __validate_pipeline_and_block(pipeline_uuid, block_uuid):
66
- if pipeline_uuid not in Pipeline.get_all_pipelines(get_repo_path()):
67
+ def __validate_pipeline_and_block(pipeline_uuid: str, block_uuid: str, repo_path: str = None):
68
+ warn_for_repo_path(repo_path=repo_path)
69
+
70
+ if repo_path is None:
71
+ repo_path = get_repo_path()
72
+
73
+ if pipeline_uuid not in Pipeline.get_all_pipelines(repo_path=repo_path):
67
74
  raise Exception('Pipeline not found, stopping sensor...')
68
75
 
69
- pipeline = Pipeline(pipeline_uuid, get_repo_path())
76
+ pipeline = Pipeline.get(pipeline_uuid, repo_path=repo_path)
70
77
 
71
78
  if block_uuid is not None:
72
79
  block = pipeline.get_block(block_uuid)
@@ -5,6 +5,7 @@ from mage_ai.api.resources.PipelineScheduleResource import PipelineScheduleResou
5
5
  from mage_ai.data_preparation.models.block.remote.models import RemoteBlock
6
6
  from mage_ai.data_preparation.models.pipeline import Pipeline
7
7
  from mage_ai.data_preparation.models.triggers import ScheduleStatus, ScheduleType
8
+ from mage_ai.data_preparation.models.utils import warn_for_repo_path
8
9
  from mage_ai.orchestration.db.models.schedules import PipelineRun, PipelineSchedule
9
10
  from mage_ai.orchestration.triggers.constants import (
10
11
  DEFAULT_POLL_INTERVAL,
@@ -15,10 +16,12 @@ from mage_ai.orchestration.triggers.utils import (
15
16
  create_and_start_pipeline_run,
16
17
  )
17
18
  from mage_ai.settings.platform import project_platform_activated
19
+ from mage_ai.settings.repo import get_repo_path
18
20
 
19
21
 
20
22
  def trigger_pipeline(
21
23
  pipeline_uuid: str,
24
+ repo_path: str = None,
22
25
  variables: Dict = None,
23
26
  check_status: bool = False,
24
27
  error_on_failure: bool = False,
@@ -33,6 +36,8 @@ def trigger_pipeline(
33
36
  if variables is None:
34
37
  variables = {}
35
38
 
39
+ warn_for_repo_path(repo_path)
40
+
36
41
  if remote_blocks:
37
42
  arr = []
38
43
  for remote_block in remote_blocks:
@@ -41,7 +46,13 @@ def trigger_pipeline(
41
46
  arr.append(remote_block.to_dict())
42
47
  variables['remote_blocks'] = arr
43
48
 
44
- pipeline = Pipeline.get(pipeline_uuid, all_projects=project_platform_activated())
49
+ repo_path_use = repo_path or get_repo_path()
50
+ pipeline = Pipeline.get(
51
+ pipeline_uuid,
52
+ all_projects=project_platform_activated(),
53
+ repo_path=repo_path_use,
54
+ use_repo_path=repo_path is not None,
55
+ )
45
56
 
46
57
  pipeline_schedule = __fetch_or_create_pipeline_schedule(pipeline, schedule_name=schedule_name)
47
58
 
@@ -1,6 +1,8 @@
1
1
  from abc import ABC, abstractmethod
2
+ from typing import Any
2
3
 
3
4
  from mage_ai.data_preparation.models.pipeline import Pipeline
5
+ from mage_ai.settings.repo import get_repo_path
4
6
 
5
7
 
6
8
  class ChartDataSourceBase(ABC):
@@ -18,10 +20,10 @@ class ChartDataSourceBase(ABC):
18
20
  @property
19
21
  def pipeline(self):
20
22
  if self.pipeline_uuid and not self._pipeline:
21
- self._pipeline = Pipeline.get(self.pipeline_uuid)
23
+ self._pipeline = Pipeline.get(self.pipeline_uuid, repo_path=get_repo_path())
22
24
 
23
25
  return self._pipeline
24
26
 
25
27
  @abstractmethod
26
- def load_data(self, **kwargs):
28
+ def load_data(self, **kwargs) -> Any:
27
29
  pass
@@ -39,9 +39,13 @@ class ChartDataSourceBlock(ChartDataSourceBase):
39
39
  )
40
40
 
41
41
  if partitions >= 0:
42
- pipeline_runs = pipeline_runs.order_by(PipelineRun.execution_date.desc())
42
+ pipeline_runs = pipeline_runs.order_by(
43
+ PipelineRun.execution_date.desc()
44
+ )
43
45
  else:
44
- pipeline_runs = pipeline_runs.order_by(PipelineRun.execution_date.asc())
46
+ pipeline_runs = pipeline_runs.order_by(
47
+ PipelineRun.execution_date.asc()
48
+ )
45
49
 
46
50
  pipeline_runs = pipeline_runs.limit(abs(partitions))
47
51
 
@@ -54,18 +58,20 @@ class ChartDataSourceBlock(ChartDataSourceBase):
54
58
  )
55
59
 
56
60
  for v in output_variable_objects:
57
- arr.append(self.pipeline.variable_manager.get_variable(
58
- self.pipeline.uuid,
59
- block.uuid,
60
- v.uuid,
61
- ))
62
- else:
61
+ arr.append(
62
+ self.pipeline.variable_manager.get_variable(
63
+ self.pipeline.uuid,
64
+ block.uuid,
65
+ v.uuid,
66
+ )
67
+ )
68
+ elif block:
63
69
  block_output = block.execute_with_callback(
64
70
  global_vars=merge_dict(
65
71
  get_global_variables(self.pipeline_uuid),
66
72
  variables or {},
67
73
  ),
68
74
  )
69
- arr += block_output['output'] or []
75
+ arr += block_output["output"] or []
70
76
 
71
77
  return arr
@@ -1,4 +1,4 @@
1
- from typing import Dict
1
+ from typing import Dict, Optional
2
2
 
3
3
  from mage_ai.data_preparation.models.constants import BlockLanguage, BlockType
4
4
  from mage_ai.data_preparation.models.widget import Widget
@@ -10,9 +10,10 @@ from mage_ai.shared.hash import merge_dict
10
10
  class ChartDataSourceChartCode(ChartDataSourceBase):
11
11
  def load_data(
12
12
  self,
13
- block: Widget = None,
14
- configuration: Dict = None,
15
- variables: Dict = None,
13
+ block: Optional[Widget] = None,
14
+ configuration: Optional[Dict] = None,
15
+ custom_code: Optional[str] = None,
16
+ variables: Optional[Dict] = None,
16
17
  **kwargs,
17
18
  ):
18
19
  block_use = block or Widget.get_block(
@@ -22,7 +23,9 @@ class ChartDataSourceChartCode(ChartDataSourceBase):
22
23
  configuration=configuration,
23
24
  language=BlockLanguage.PYTHON,
24
25
  )
26
+
25
27
  block_output = block_use.execute_with_callback(
28
+ custom_code=custom_code,
26
29
  disable_json_serialization=True,
27
30
  global_vars=merge_dict(
28
31
  get_global_variables(self.pipeline_uuid) if self.pipeline_uuid else {},
@@ -30,4 +33,4 @@ class ChartDataSourceChartCode(ChartDataSourceBase):
30
33
  ),
31
34
  )
32
35
 
33
- return block_output['output'] or []
36
+ return block_output["output"] or []
@@ -10,3 +10,4 @@ class ChartDataSourceType(str, Enum):
10
10
  PIPELINES = 'pipelines'
11
11
  PIPELINE_RUNS = 'pipeline_runs'
12
12
  PIPELINE_SCHEDULES = 'pipeline_schedules'
13
+ SYSTEM_METRICS = 'system_metrics'
@@ -0,0 +1,22 @@
1
+ import os
2
+
3
+ import pandas as pd
4
+
5
+ from mage_ai.data_preparation.models.constants import PIPELINES_FOLDER
6
+ from mage_ai.presenters.charts.data_sources.base import ChartDataSourceBase
7
+ from mage_ai.system.memory.presenters import to_dataframe
8
+
9
+
10
+ class ChartDataSourceSystemMetrics(ChartDataSourceBase):
11
+ def load_data(
12
+ self,
13
+ **kwargs,
14
+ ) -> pd.DataFrame:
15
+ df = to_dataframe(
16
+ os.path.join(
17
+ PIPELINES_FOLDER,
18
+ self.pipeline_uuid or '',
19
+ self.block_uuid or '',
20
+ ),
21
+ ).to_pandas()
22
+ return df
@@ -144,14 +144,14 @@ class Interaction:
144
144
  return self._content
145
145
 
146
146
  async def content_parsed(self, interpolate_variables: bool = False) -> Dict:
147
- if self._content_parsed is not None or \
148
- BlockLanguage.YAML != self.language:
149
-
147
+ if self._content_parsed is not None or BlockLanguage.YAML != self.language:
150
148
  return self._content_parsed
151
149
 
152
150
  text = await self.content_async()
153
151
  self._content_parsed = yaml.safe_load(
154
- interpolate_content(text, self.__pipeline_variables) if interpolate_variables else text,
152
+ interpolate_content(text, self.__pipeline_variables)
153
+ if interpolate_variables
154
+ else text,
155
155
  )
156
156
 
157
157
  return self._content_parsed
@@ -173,7 +173,7 @@ class Interaction:
173
173
  async def layout(self) -> Dict:
174
174
  settings = await self.content_parsed() or {}
175
175
  rows = []
176
- for row in (settings.get('layout') or []):
176
+ for row in settings.get('layout') or []:
177
177
  items = []
178
178
  for item in row:
179
179
  items.append(InteractionLayoutItem(**item))
@@ -199,7 +199,7 @@ class Interaction:
199
199
  for _uuid, item in (settings.get('inputs') or {}).items():
200
200
  InteractionInput(**item)
201
201
 
202
- for row in (settings.get('layout') or []):
202
+ for row in settings.get('layout') or []:
203
203
  for item in row:
204
204
  InteractionLayoutItem(**item)
205
205
 
@@ -287,4 +287,8 @@ class Interaction:
287
287
 
288
288
  @property
289
289
  def __repo_path(self) -> str:
290
- return self.pipeline.repo_path if self.pipeline is not None else get_repo_path()
290
+ return (
291
+ self.pipeline.repo_path
292
+ if self.pipeline is not None
293
+ else get_repo_path(root_project=True)
294
+ )
@@ -3,11 +3,12 @@ from typing import Any, Dict, List, Union
3
3
 
4
4
  from mage_ai.data_preparation.models.pipeline import Pipeline
5
5
  from mage_ai.presenters.pages.loaders.base import BaseLoader
6
+ from mage_ai.settings.repo import get_repo_path
6
7
 
7
8
 
8
- async def get_pipeline(uuid: str) -> Pipeline:
9
+ async def get_pipeline(uuid: str, repo_path: str) -> Pipeline:
9
10
  try:
10
- return await Pipeline.get_async(uuid)
11
+ return await Pipeline.get_async(uuid, repo_path=repo_path)
11
12
  except Exception as err:
12
13
  err_message = f'Error loading pipeline {uuid}: {err}.'
13
14
  if err.__class__.__name__ == 'OSError' and 'Too many open files' in err.strerror:
@@ -20,4 +21,5 @@ async def get_pipeline(uuid: str) -> Pipeline:
20
21
  class Loader(BaseLoader):
21
22
  @classmethod
22
23
  async def load(self, ids: List[Union[int, str]], query: Dict = None, **kwargs) -> List[Any]:
23
- return await asyncio.gather(*[get_pipeline(uuid) for uuid in ids])
24
+ repo_path = get_repo_path()
25
+ return await asyncio.gather(*[get_pipeline(uuid, repo_path) for uuid in ids])