mage-ai 0.9.70__py3-none-any.whl → 0.9.71__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mage-ai might be problematic. Click here for more details.

Files changed (568) hide show
  1. mage_ai/ai/utils/xgboost.py +222 -0
  2. mage_ai/api/errors.py +37 -25
  3. mage_ai/api/operations/base.py +13 -1
  4. mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
  5. mage_ai/api/policies/BlockOutputPolicy.py +40 -17
  6. mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
  7. mage_ai/api/policies/KernelPolicy.py +55 -32
  8. mage_ai/api/policies/KernelProcessPolicy.py +56 -0
  9. mage_ai/api/policies/OutputPolicy.py +73 -41
  10. mage_ai/api/policies/PipelinePolicy.py +206 -138
  11. mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
  12. mage_ai/api/presenters/BlockPresenter.py +1 -1
  13. mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
  14. mage_ai/api/presenters/KernelPresenter.py +5 -26
  15. mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
  16. mage_ai/api/presenters/PipelinePresenter.py +18 -5
  17. mage_ai/api/presenters/StatusPresenter.py +2 -0
  18. mage_ai/api/presenters/SyncPresenter.py +25 -0
  19. mage_ai/api/resources/AutocompleteItemResource.py +1 -1
  20. mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
  21. mage_ai/api/resources/BlockOutputResource.py +42 -9
  22. mage_ai/api/resources/BlockResource.py +4 -3
  23. mage_ai/api/resources/BlockRunResource.py +27 -22
  24. mage_ai/api/resources/ClusterResource.py +4 -1
  25. mage_ai/api/resources/CustomTemplateResource.py +34 -14
  26. mage_ai/api/resources/DataProviderResource.py +1 -1
  27. mage_ai/api/resources/ExecutionStateResource.py +3 -1
  28. mage_ai/api/resources/FileContentResource.py +8 -2
  29. mage_ai/api/resources/FileResource.py +10 -4
  30. mage_ai/api/resources/FileVersionResource.py +3 -1
  31. mage_ai/api/resources/GitBranchResource.py +46 -9
  32. mage_ai/api/resources/GlobalDataProductResource.py +44 -7
  33. mage_ai/api/resources/GlobalHookResource.py +4 -1
  34. mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
  35. mage_ai/api/resources/IntegrationSourceResource.py +8 -4
  36. mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
  37. mage_ai/api/resources/KernelProcessResource.py +44 -0
  38. mage_ai/api/resources/KernelResource.py +25 -3
  39. mage_ai/api/resources/OutputResource.py +33 -11
  40. mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
  41. mage_ai/api/resources/PipelineInteractionResource.py +31 -15
  42. mage_ai/api/resources/PipelineResource.py +250 -123
  43. mage_ai/api/resources/PipelineRunResource.py +11 -3
  44. mage_ai/api/resources/PipelineScheduleResource.py +7 -2
  45. mage_ai/api/resources/PipelineTriggerResource.py +6 -1
  46. mage_ai/api/resources/ProjectResource.py +18 -7
  47. mage_ai/api/resources/SecretResource.py +1 -1
  48. mage_ai/api/resources/SeedResource.py +8 -1
  49. mage_ai/api/resources/StatusResource.py +21 -6
  50. mage_ai/api/resources/SyncResource.py +6 -8
  51. mage_ai/api/resources/VariableResource.py +46 -26
  52. mage_ai/api/resources/VersionControlProjectResource.py +9 -2
  53. mage_ai/api/resources/WidgetResource.py +1 -1
  54. mage_ai/api/resources/WorkspaceResource.py +1 -1
  55. mage_ai/api/views.py +47 -40
  56. mage_ai/authentication/permissions/seed.py +16 -2
  57. mage_ai/authentication/providers/oidc.py +21 -1
  58. mage_ai/autocomplete/utils.py +13 -9
  59. mage_ai/cache/base.py +1 -1
  60. mage_ai/cache/block.py +18 -12
  61. mage_ai/cache/block_action_object/__init__.py +32 -4
  62. mage_ai/cache/file.py +22 -19
  63. mage_ai/cache/pipeline.py +18 -12
  64. mage_ai/cli/main.py +1 -0
  65. mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
  66. mage_ai/cluster_manager/config.py +2 -2
  67. mage_ai/cluster_manager/manage.py +1 -1
  68. mage_ai/cluster_manager/workspace/base.py +1 -1
  69. mage_ai/command_center/applications/factory.py +10 -7
  70. mage_ai/command_center/files/factory.py +17 -15
  71. mage_ai/command_center/utils.py +25 -13
  72. mage_ai/data/__init__.py +0 -0
  73. mage_ai/data/constants.py +45 -0
  74. mage_ai/data/models/__init__.py +0 -0
  75. mage_ai/data/models/base.py +119 -0
  76. mage_ai/data/models/constants.py +1 -0
  77. mage_ai/data/models/generator.py +115 -0
  78. mage_ai/data/models/manager.py +168 -0
  79. mage_ai/data/models/pyarrow/__init__.py +0 -0
  80. mage_ai/data/models/pyarrow/record_batch.py +55 -0
  81. mage_ai/data/models/pyarrow/shared.py +21 -0
  82. mage_ai/data/models/pyarrow/table.py +8 -0
  83. mage_ai/data/models/reader.py +103 -0
  84. mage_ai/data/models/utils.py +59 -0
  85. mage_ai/data/models/writer.py +91 -0
  86. mage_ai/data/tabular/__init__.py +0 -0
  87. mage_ai/data/tabular/constants.py +23 -0
  88. mage_ai/data/tabular/mocks.py +19 -0
  89. mage_ai/data/tabular/models.py +126 -0
  90. mage_ai/data/tabular/reader.py +602 -0
  91. mage_ai/data/tabular/utils.py +102 -0
  92. mage_ai/data/tabular/writer.py +266 -0
  93. mage_ai/data/variables/__init__.py +0 -0
  94. mage_ai/data/variables/wrapper.py +54 -0
  95. mage_ai/data_cleaner/analysis/charts.py +61 -39
  96. mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
  97. mage_ai/data_cleaner/estimators/encoders.py +5 -2
  98. mage_ai/data_integrations/utils/scheduler.py +16 -11
  99. mage_ai/data_preparation/decorators.py +1 -0
  100. mage_ai/data_preparation/executors/block_executor.py +237 -155
  101. mage_ai/data_preparation/executors/streaming_pipeline_executor.py +1 -1
  102. mage_ai/data_preparation/git/__init__.py +27 -7
  103. mage_ai/data_preparation/git/api.py +7 -1
  104. mage_ai/data_preparation/git/utils.py +22 -16
  105. mage_ai/data_preparation/logging/logger_manager.py +4 -3
  106. mage_ai/data_preparation/models/block/__init__.py +1542 -878
  107. mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
  108. mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
  109. mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
  110. mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
  111. mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
  112. mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
  113. mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
  114. mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
  115. mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
  116. mage_ai/data_preparation/models/block/dynamic/utils.py +291 -168
  117. mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
  118. mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
  119. mage_ai/data_preparation/models/block/extension/utils.py +10 -1
  120. mage_ai/data_preparation/models/block/global_data_product/__init__.py +10 -1
  121. mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
  122. mage_ai/data_preparation/models/block/outputs.py +722 -0
  123. mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
  124. mage_ai/data_preparation/models/block/r/__init__.py +56 -38
  125. mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
  126. mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
  127. mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
  128. mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
  129. mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
  130. mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
  131. mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
  132. mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
  133. mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
  134. mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
  135. mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
  136. mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
  137. mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
  138. mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
  139. mage_ai/data_preparation/models/block/utils.py +127 -70
  140. mage_ai/data_preparation/models/constants.py +19 -14
  141. mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
  142. mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
  143. mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
  144. mage_ai/data_preparation/models/file.py +41 -28
  145. mage_ai/data_preparation/models/global_data_product/__init__.py +88 -58
  146. mage_ai/data_preparation/models/global_hooks/models.py +1 -0
  147. mage_ai/data_preparation/models/interfaces.py +29 -0
  148. mage_ai/data_preparation/models/pipeline.py +365 -180
  149. mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
  150. mage_ai/data_preparation/models/pipelines/seed.py +1 -1
  151. mage_ai/data_preparation/models/project/__init__.py +66 -18
  152. mage_ai/data_preparation/models/project/constants.py +2 -0
  153. mage_ai/data_preparation/models/triggers/__init__.py +120 -24
  154. mage_ai/data_preparation/models/utils.py +467 -17
  155. mage_ai/data_preparation/models/variable.py +1028 -137
  156. mage_ai/data_preparation/models/variables/__init__.py +0 -0
  157. mage_ai/data_preparation/models/variables/cache.py +149 -0
  158. mage_ai/data_preparation/models/variables/constants.py +72 -0
  159. mage_ai/data_preparation/models/variables/summarizer.py +336 -0
  160. mage_ai/data_preparation/models/variables/utils.py +77 -0
  161. mage_ai/data_preparation/models/widget/__init__.py +63 -41
  162. mage_ai/data_preparation/models/widget/charts.py +40 -27
  163. mage_ai/data_preparation/models/widget/constants.py +2 -0
  164. mage_ai/data_preparation/models/widget/utils.py +3 -3
  165. mage_ai/data_preparation/preferences.py +3 -3
  166. mage_ai/data_preparation/repo_manager.py +55 -21
  167. mage_ai/data_preparation/storage/base_storage.py +2 -2
  168. mage_ai/data_preparation/storage/gcs_storage.py +7 -4
  169. mage_ai/data_preparation/storage/local_storage.py +6 -3
  170. mage_ai/data_preparation/storage/s3_storage.py +5 -2
  171. mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
  172. mage_ai/data_preparation/variable_manager.py +281 -76
  173. mage_ai/io/base.py +3 -2
  174. mage_ai/io/bigquery.py +1 -0
  175. mage_ai/io/redshift.py +7 -5
  176. mage_ai/kernels/__init__.py +0 -0
  177. mage_ai/kernels/models.py +188 -0
  178. mage_ai/kernels/utils.py +169 -0
  179. mage_ai/orchestration/concurrency.py +6 -2
  180. mage_ai/orchestration/db/__init__.py +1 -0
  181. mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
  182. mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
  183. mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
  184. mage_ai/orchestration/db/models/oauth.py +2 -9
  185. mage_ai/orchestration/db/models/projects.py +10 -0
  186. mage_ai/orchestration/db/models/schedules.py +204 -187
  187. mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
  188. mage_ai/orchestration/db/models/utils.py +46 -5
  189. mage_ai/orchestration/metrics/pipeline_run.py +8 -9
  190. mage_ai/orchestration/notification/sender.py +1 -0
  191. mage_ai/orchestration/pipeline_scheduler_original.py +32 -8
  192. mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
  193. mage_ai/orchestration/run_status_checker.py +11 -4
  194. mage_ai/orchestration/triggers/api.py +12 -1
  195. mage_ai/presenters/charts/data_sources/base.py +4 -2
  196. mage_ai/presenters/charts/data_sources/block.py +15 -9
  197. mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
  198. mage_ai/presenters/charts/data_sources/constants.py +1 -0
  199. mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
  200. mage_ai/presenters/interactions/models.py +11 -7
  201. mage_ai/presenters/pages/loaders/pipelines.py +5 -3
  202. mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
  203. mage_ai/presenters/utils.py +2 -0
  204. mage_ai/server/api/blocks.py +2 -1
  205. mage_ai/server/api/downloads.py +5 -1
  206. mage_ai/server/api/triggers.py +3 -1
  207. mage_ai/server/constants.py +1 -1
  208. mage_ai/server/frontend_dist/404.html +5 -5
  209. mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
  210. mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
  211. mage_ai/server/frontend_dist/_next/static/chunks/{1557-b3502f3f1aa92ac7.js → 1557-25a7d985d5564fd3.js} +1 -1
  212. mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
  213. mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
  214. mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
  215. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/3763-61b542dafdbf5754.js → frontend_dist/_next/static/chunks/3763-40780c6d1e4b261d.js} +1 -1
  216. mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
  217. mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
  218. mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
  219. mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
  220. mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
  221. mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
  222. mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
  223. mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
  224. mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
  225. mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
  226. mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
  227. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
  228. mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
  229. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
  230. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
  231. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/overview-597b74828bf105db.js → frontend_dist/_next/static/chunks/pages/overview-9f1ac4ec003884f3.js} +1 -1
  232. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-7181b086c93784d2.js → [...slug]-7e737f6fc7e83e9b.js} +1 -1
  233. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
  234. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
  235. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
  236. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
  237. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
  238. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
  239. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
  240. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
  241. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js → frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js} +1 -1
  242. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
  243. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
  244. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
  245. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
  246. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
  247. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
  248. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
  249. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
  250. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
  251. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
  252. mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
  253. mage_ai/server/frontend_dist/block-layout.html +3 -3
  254. mage_ai/server/frontend_dist/compute.html +6 -6
  255. mage_ai/server/frontend_dist/files.html +6 -6
  256. mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
  257. mage_ai/server/frontend_dist/global-data-products.html +6 -6
  258. mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
  259. mage_ai/server/frontend_dist/global-hooks.html +6 -6
  260. mage_ai/server/frontend_dist/index.html +3 -3
  261. mage_ai/server/frontend_dist/manage/files.html +6 -6
  262. mage_ai/server/frontend_dist/manage/settings.html +6 -6
  263. mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
  264. mage_ai/server/frontend_dist/manage/users/new.html +6 -6
  265. mage_ai/server/frontend_dist/manage/users.html +6 -6
  266. mage_ai/server/frontend_dist/manage.html +6 -6
  267. mage_ai/server/frontend_dist/oauth.html +5 -5
  268. mage_ai/server/frontend_dist/overview.html +6 -6
  269. mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
  270. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  271. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
  272. mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
  273. mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
  274. mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
  275. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  276. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  277. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
  278. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
  279. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
  280. mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
  281. mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
  282. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  283. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
  284. mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
  285. mage_ai/server/frontend_dist/pipelines.html +6 -6
  286. mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
  287. mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
  288. mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
  289. mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
  290. mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
  291. mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
  292. mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
  293. mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
  294. mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
  295. mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
  296. mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
  297. mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
  298. mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
  299. mage_ai/server/frontend_dist/settings.html +3 -3
  300. mage_ai/server/frontend_dist/sign-in.html +12 -12
  301. mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
  302. mage_ai/server/frontend_dist/templates.html +6 -6
  303. mage_ai/server/frontend_dist/terminal.html +6 -6
  304. mage_ai/server/frontend_dist/test.html +3 -3
  305. mage_ai/server/frontend_dist/triggers.html +6 -6
  306. mage_ai/server/frontend_dist/version-control.html +6 -6
  307. mage_ai/server/frontend_dist_base_path_template/404.html +5 -5
  308. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
  309. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{1557-b3502f3f1aa92ac7.js → 1557-25a7d985d5564fd3.js} +1 -1
  310. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
  311. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
  312. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
  313. mage_ai/server/{frontend_dist/_next/static/chunks/3763-61b542dafdbf5754.js → frontend_dist_base_path_template/_next/static/chunks/3763-40780c6d1e4b261d.js} +1 -1
  314. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
  315. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
  316. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
  317. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
  318. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
  319. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
  320. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
  321. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
  322. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
  323. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
  324. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
  325. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
  326. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
  327. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
  328. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
  329. mage_ai/server/{frontend_dist/_next/static/chunks/pages/overview-597b74828bf105db.js → frontend_dist_base_path_template/_next/static/chunks/pages/overview-9f1ac4ec003884f3.js} +1 -1
  330. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-7181b086c93784d2.js → [...slug]-7e737f6fc7e83e9b.js} +1 -1
  331. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
  332. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
  333. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
  334. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
  335. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
  336. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
  337. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
  338. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
  339. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js} +1 -1
  340. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
  341. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
  342. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
  343. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
  344. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
  345. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
  346. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
  347. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
  348. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
  349. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
  350. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
  351. mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
  352. mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
  353. mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
  354. mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
  355. mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
  356. mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
  357. mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
  358. mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
  359. mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
  360. mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
  361. mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
  362. mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
  363. mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
  364. mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
  365. mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
  366. mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
  367. mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
  368. mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
  369. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  370. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
  371. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
  372. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
  373. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
  374. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  375. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  376. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
  377. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
  378. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
  379. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
  380. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
  381. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  382. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
  383. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
  384. mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
  385. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
  386. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
  387. mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
  388. mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
  389. mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
  390. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
  391. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
  392. mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
  393. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
  394. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
  395. mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
  396. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
  397. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
  398. mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
  399. mage_ai/server/frontend_dist_base_path_template/sign-in.html +12 -12
  400. mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
  401. mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
  402. mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
  403. mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
  404. mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
  405. mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
  406. mage_ai/server/kernel_output_parser.py +4 -1
  407. mage_ai/server/scheduler_manager.py +9 -0
  408. mage_ai/server/server.py +35 -31
  409. mage_ai/server/utils/custom_output.py +284 -0
  410. mage_ai/server/utils/execute_custom_code.py +245 -0
  411. mage_ai/server/utils/output_display.py +123 -289
  412. mage_ai/server/websocket_server.py +116 -69
  413. mage_ai/services/k8s/config.py +23 -0
  414. mage_ai/services/k8s/job_manager.py +6 -1
  415. mage_ai/services/ssh/aws/emr/utils.py +8 -8
  416. mage_ai/settings/keys/auth.py +1 -0
  417. mage_ai/settings/platform/__init__.py +159 -38
  418. mage_ai/settings/platform/constants.py +5 -0
  419. mage_ai/settings/platform/utils.py +53 -10
  420. mage_ai/settings/repo.py +26 -12
  421. mage_ai/settings/server.py +128 -37
  422. mage_ai/shared/array.py +24 -1
  423. mage_ai/shared/complex.py +45 -0
  424. mage_ai/shared/config.py +2 -1
  425. mage_ai/shared/custom_logger.py +11 -0
  426. mage_ai/shared/dates.py +10 -6
  427. mage_ai/shared/files.py +63 -8
  428. mage_ai/shared/hash.py +33 -9
  429. mage_ai/shared/io.py +9 -5
  430. mage_ai/shared/models.py +82 -24
  431. mage_ai/shared/outputs.py +87 -0
  432. mage_ai/shared/parsers.py +141 -15
  433. mage_ai/shared/path_fixer.py +11 -7
  434. mage_ai/shared/singletons/__init__.py +0 -0
  435. mage_ai/shared/singletons/base.py +47 -0
  436. mage_ai/shared/singletons/memory.py +38 -0
  437. mage_ai/shared/strings.py +34 -1
  438. mage_ai/shared/yaml.py +24 -0
  439. mage_ai/streaming/sinks/oracledb.py +57 -0
  440. mage_ai/streaming/sinks/sink_factory.py +4 -0
  441. mage_ai/system/__init__.py +0 -0
  442. mage_ai/system/constants.py +14 -0
  443. mage_ai/system/memory/__init__.py +0 -0
  444. mage_ai/system/memory/constants.py +1 -0
  445. mage_ai/system/memory/manager.py +174 -0
  446. mage_ai/system/memory/presenters.py +158 -0
  447. mage_ai/system/memory/process.py +216 -0
  448. mage_ai/system/memory/samples.py +13 -0
  449. mage_ai/system/memory/utils.py +656 -0
  450. mage_ai/system/memory/wrappers.py +177 -0
  451. mage_ai/system/models.py +58 -0
  452. mage_ai/system/storage/__init__.py +0 -0
  453. mage_ai/system/storage/utils.py +29 -0
  454. mage_ai/tests/api/endpoints/mixins.py +2 -2
  455. mage_ai/tests/api/endpoints/test_blocks.py +2 -1
  456. mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
  457. mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
  458. mage_ai/tests/api/endpoints/test_projects.py +2 -1
  459. mage_ai/tests/api/operations/base/test_base.py +27 -27
  460. mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
  461. mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
  462. mage_ai/tests/api/operations/test_syncs.py +6 -4
  463. mage_ai/tests/api/resources/test_pipeline_resource.py +9 -2
  464. mage_ai/tests/authentication/providers/test_oidc.py +59 -0
  465. mage_ai/tests/base_test.py +2 -2
  466. mage_ai/tests/data/__init__.py +0 -0
  467. mage_ai/tests/data/models/__init__.py +0 -0
  468. mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
  469. mage_ai/tests/data_preparation/git/test_git.py +4 -1
  470. mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
  471. mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
  472. mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
  473. mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
  474. mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
  475. mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
  476. mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
  477. mage_ai/tests/data_preparation/models/block/test_global_data_product.py +3 -2
  478. mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
  479. mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
  480. mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
  481. mage_ai/tests/data_preparation/models/test_block.py +115 -120
  482. mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
  483. mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
  484. mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
  485. mage_ai/tests/data_preparation/models/test_project.py +4 -1
  486. mage_ai/tests/data_preparation/models/test_utils.py +80 -0
  487. mage_ai/tests/data_preparation/models/test_variable.py +242 -69
  488. mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
  489. mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
  490. mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
  491. mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
  492. mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
  493. mage_ai/tests/factory.py +64 -43
  494. mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
  495. mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
  496. mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
  497. mage_ai/tests/orchestration/triggers/test_global_data_product.py +3 -2
  498. mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
  499. mage_ai/tests/services/k8s/test_job_manager.py +18 -0
  500. mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
  501. mage_ai/tests/test_shared.py +61 -0
  502. mage_ai/usage_statistics/logger.py +7 -2
  503. mage_ai/utils/code.py +33 -19
  504. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/METADATA +5 -2
  505. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/RECORD +513 -417
  506. mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
  507. mage_ai/server/frontend_dist/_next/static/RhDiJSkcjCsh4xxX4BFBk/_buildManifest.js +0 -1
  508. mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
  509. mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
  510. mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
  511. mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
  512. mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
  513. mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
  514. mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
  515. mage_ai/server/frontend_dist/_next/static/chunks/7966-b9b85ba10667e654.js +0 -1
  516. mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
  517. mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
  518. mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
  519. mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
  520. mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
  521. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +0 -1
  522. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
  523. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
  524. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
  525. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +0 -1
  526. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
  527. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
  528. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
  529. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +0 -1
  530. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
  531. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +0 -1
  532. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +0 -1
  533. mage_ai/server/frontend_dist_base_path_template/_next/static/TdpLLFome13qvM0gXvpHs/_buildManifest.js +0 -1
  534. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
  535. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
  536. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
  537. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
  538. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
  539. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
  540. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
  541. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-b9b85ba10667e654.js +0 -1
  542. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
  543. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
  544. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
  545. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
  546. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
  547. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +0 -1
  548. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
  549. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
  550. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
  551. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +0 -1
  552. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
  553. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
  554. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
  555. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +0 -1
  556. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
  557. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +0 -1
  558. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +0 -1
  559. mage_ai/shared/memory.py +0 -90
  560. mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
  561. /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
  562. /mage_ai/server/frontend_dist/_next/static/{RhDiJSkcjCsh4xxX4BFBk → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
  563. /mage_ai/server/frontend_dist_base_path_template/_next/static/{TdpLLFome13qvM0gXvpHs → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
  564. /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
  565. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
  566. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
  567. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
  568. {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
@@ -2,6 +2,7 @@ from mage_ai.api.operations import constants
2
2
  from mage_ai.api.presenters.BasePresenter import BasePresenter
3
3
  from mage_ai.data_preparation.models.constants import (
4
4
  DATAFRAME_SAMPLE_COUNT_PREVIEW,
5
+ MAX_RESULTS_FOR_BLOCK_OUTPUTS_PREVIEW,
5
6
  PipelineType,
6
7
  )
7
8
  from mage_ai.data_preparation.models.project import Project
@@ -68,10 +69,18 @@ class PipelinePresenter(BasePresenter):
68
69
  if include_block_metadata:
69
70
  include_block_metadata = include_block_metadata[0]
70
71
 
71
- include_block_catalog = PipelineType.PYTHON == self.model.type and \
72
- Project(self.model.repo_config).is_feature_enabled(
73
- FeatureUUID.DATA_INTEGRATION_IN_BATCH_PIPELINE,
74
- )
72
+ max_results_for_block_outputs = query.get(
73
+ 'max_results_for_block_outputs', [MAX_RESULTS_FOR_BLOCK_OUTPUTS_PREVIEW]
74
+ )
75
+ if max_results_for_block_outputs:
76
+ max_results_for_block_outputs = max_results_for_block_outputs[0]
77
+
78
+ include_block_catalog = PipelineType.PYTHON == self.model.type and Project(
79
+ context_data=kwargs.get('context_data'),
80
+ repo_config=self.model.repo_config,
81
+ ).is_feature_enabled(
82
+ FeatureUUID.DATA_INTEGRATION_IN_BATCH_PIPELINE,
83
+ )
75
84
 
76
85
  data = await self.model.to_dict_async(
77
86
  include_block_catalog=include_block_catalog,
@@ -85,6 +94,9 @@ class PipelinePresenter(BasePresenter):
85
94
  include_outputs=include_outputs,
86
95
  include_outputs_spark=include_outputs_spark,
87
96
  sample_count=DATAFRAME_SAMPLE_COUNT_PREVIEW,
97
+ max_results=max_results_for_block_outputs,
98
+ disable_block_output_previews=True,
99
+ exclude_blank_variable_uuids=True,
88
100
  )
89
101
  elif constants.UPDATE == display_format:
90
102
  data = self.model.to_dict(include_extensions=include_extensions)
@@ -103,7 +115,8 @@ class PipelinePresenter(BasePresenter):
103
115
 
104
116
  PipelinePresenter.register_format(
105
117
  constants.LIST,
106
- PipelinePresenter.default_attributes + [
118
+ PipelinePresenter.default_attributes
119
+ + [
107
120
  'schedules',
108
121
  ],
109
122
  )
@@ -9,6 +9,8 @@ class StatusPresenter(BasePresenter):
9
9
  'repo_path_relative',
10
10
  'repo_path_relative_root',
11
11
  'repo_path_root',
12
+ 'server_process_id',
13
+ 'scheduler_process_id',
12
14
  'scheduler_status',
13
15
  'instance_type',
14
16
  'disable_pipeline_edit_access',
@@ -1,16 +1,21 @@
1
1
  from mage_ai.api.presenters.BasePresenter import BasePresenter
2
+ from mage_ai.data_preparation.shared.secrets import get_secret_value
3
+ from mage_ai.settings.repo import get_repo_path
2
4
 
3
5
 
4
6
  class SyncPresenter(BasePresenter):
5
7
  default_attributes = [
6
8
  'access_token',
9
+ 'access_token_secret_name',
7
10
  'auth_type',
8
11
  'branch',
9
12
  'email',
10
13
  'remote_repo_link',
11
14
  'repo_path',
12
15
  'ssh_private_key',
16
+ 'ssh_private_key_secret_name',
13
17
  'ssh_public_key',
18
+ 'ssh_public_key_secret_name',
14
19
  'sync_on_pipeline_run',
15
20
  'sync_on_start',
16
21
  'sync_submodules',
@@ -18,3 +23,23 @@ class SyncPresenter(BasePresenter):
18
23
  'user_git_settings',
19
24
  'username',
20
25
  ]
26
+
27
+ def present(self, **kwargs):
28
+ data = self.model
29
+
30
+ def filter_invalid_secret_values(data):
31
+ for attribute in self.default_attributes:
32
+ if attribute.endswith('_secret_name'):
33
+ secret_name = data.get(attribute)
34
+ if secret_name and not get_secret_value(
35
+ secret_name,
36
+ repo_name=get_repo_path(user=self.current_user),
37
+ suppress_warning=False,
38
+ ):
39
+ data[attribute] = None
40
+
41
+ filter_invalid_secret_values(self.model)
42
+ if 'user_git_settings' in data:
43
+ filter_invalid_secret_values(data['user_git_settings'])
44
+
45
+ return self.model
@@ -13,7 +13,7 @@ class AutocompleteItemResource(GenericResource):
13
13
  @classmethod
14
14
  @safe_db_query
15
15
  async def collection(self, query, meta, user, **kwargs):
16
- repo_path = get_repo_path()
16
+ repo_path = get_repo_path(user=user)
17
17
 
18
18
  collection = []
19
19
  for file_group, mapping in [
@@ -21,6 +21,9 @@ from mage_ai.presenters.charts.data_sources.pipeline_schedules import (
21
21
  ChartDataSourcePipelineSchedules,
22
22
  )
23
23
  from mage_ai.presenters.charts.data_sources.pipelines import ChartDataSourcePipelines
24
+ from mage_ai.presenters.charts.data_sources.system_metrics import (
25
+ ChartDataSourceSystemMetrics,
26
+ )
24
27
  from mage_ai.shared.hash import extract, merge_dict
25
28
  from mage_ai.usage_statistics.logger import UsageStatisticLogger
26
29
 
@@ -38,6 +41,13 @@ class BlockLayoutItemResource(GenericResource):
38
41
  else:
39
42
  variables[k] = v
40
43
 
44
+ content_override = query.get('content_override', [None])
45
+ if content_override:
46
+ content_override = content_override[0]
47
+ skip_render = query.get('skip_render', [False])
48
+ if skip_render:
49
+ skip_render = skip_render[0]
50
+
41
51
  configuration_override = variables.pop('configuration_override', None)
42
52
  if configuration_override:
43
53
  configuration_override = json.loads(urllib.parse.unquote(configuration_override))
@@ -57,6 +67,7 @@ class BlockLayoutItemResource(GenericResource):
57
67
 
58
68
  content = None
59
69
  data = None
70
+ error = None
60
71
  block_type = block_config.get('type')
61
72
 
62
73
  if BlockType.CHART == block_type:
@@ -67,39 +78,58 @@ class BlockLayoutItemResource(GenericResource):
67
78
  else:
68
79
  data_source_config = data_source_override
69
80
 
70
- configuration_to_use = configuration_override or block_config.get('configuration')
71
-
72
- if data_source_config or ChartType.CUSTOM == configuration_to_use.get('chart_type'):
73
- data_source_type = data_source_config.get('type')
74
- pipeline_uuid = data_source_config.get('pipeline_uuid')
75
-
76
- block = BlockFactory.get_block(
77
- block_config.get('name') or file_path or uuid,
78
- block_uuid,
79
- block_type,
80
- configuration=configuration_to_use,
81
- **extract(block_config, [
81
+ configuration_to_use = (
82
+ configuration_override or block_config.get('configuration') or {}
83
+ )
84
+
85
+ block = BlockFactory.get_block(
86
+ block_config.get('name') or file_path or uuid,
87
+ block_uuid,
88
+ block_type,
89
+ configuration=configuration_to_use,
90
+ **extract(
91
+ block_config,
92
+ [
82
93
  'language',
83
- ]),
84
- )
94
+ ],
95
+ ),
96
+ )
85
97
 
98
+ content = content_override
99
+ if not content and block.file and block.file.exists():
86
100
  content = block.content
87
101
 
102
+ if data_source_config or (
103
+ configuration_to_use and ChartType.CUSTOM == configuration_to_use.get('chart_type')
104
+ ):
105
+ data_source_type = data_source_config.get('type')
106
+ pipeline_uuid = data_source_config.get('pipeline_uuid')
107
+
88
108
  if ChartDataSourceType.CHART_CODE == data_source_type:
89
109
  data_source = ChartDataSourceChartCode(
90
110
  block_uuid=block.uuid,
91
111
  pipeline_uuid=pipeline_uuid,
92
112
  )
93
- data = data_source.load_data(block=block)
113
+ data = data_source.load_data(
114
+ block=block,
115
+ configuration=configuration_to_use,
116
+ custom_code=content_override,
117
+ )
94
118
  else:
95
119
  data_source_block_uuid = data_source_config.get('block_uuid')
96
120
 
97
- data_source_class_options = merge_dict(extract(data_source_config, [
98
- 'pipeline_schedule_id',
99
- ]), dict(
100
- block_uuid=data_source_block_uuid,
101
- pipeline_uuid=pipeline_uuid,
102
- ))
121
+ data_source_class_options = merge_dict(
122
+ extract(
123
+ data_source_config,
124
+ [
125
+ 'pipeline_schedule_id',
126
+ ],
127
+ ),
128
+ dict(
129
+ block_uuid=data_source_block_uuid,
130
+ pipeline_uuid=pipeline_uuid,
131
+ ),
132
+ )
103
133
  data_source_output = None
104
134
 
105
135
  if ChartDataSourceType.BLOCK == data_source_type:
@@ -132,25 +162,32 @@ class BlockLayoutItemResource(GenericResource):
132
162
  user=user,
133
163
  **kwargs,
134
164
  )
165
+ elif ChartDataSourceType.SYSTEM_METRICS == data_source_type:
166
+ data_source = ChartDataSourceSystemMetrics(**data_source_class_options)
167
+ data_source_output = data_source.load_data(
168
+ user=user,
169
+ **kwargs,
170
+ )
135
171
 
136
- try:
137
- input_args = None
138
- if data_source_output is not None:
139
- input_args = [data_source_output]
140
-
141
- data = block.execute_with_callback(
142
- disable_json_serialization=True,
143
- input_args=input_args,
144
- global_vars=merge_dict(
145
- get_global_variables(pipeline_uuid) if pipeline_uuid else {},
146
- variables or {},
147
- ),
148
- ).get('output', None)
149
- except Exception as err:
150
- error = ApiError(ApiError.RESOURCE_NOT_FOUND.copy())
151
- error.message = str(err)
152
- error.errors = traceback.format_exc()
153
- raise error
172
+ if not skip_render:
173
+ try:
174
+ input_args = None
175
+ if data_source_output is not None:
176
+ input_args = [data_source_output]
177
+
178
+ data = block.execute_with_callback(
179
+ custom_code=content_override,
180
+ disable_json_serialization=True,
181
+ input_args=input_args,
182
+ global_vars=merge_dict(
183
+ get_global_variables(pipeline_uuid) if pipeline_uuid else {},
184
+ variables or {},
185
+ ),
186
+ ).get('output', None)
187
+ except Exception as err:
188
+ error = ApiError(ApiError.RESOURCE_NOT_FOUND.copy())
189
+ error.message = str(err)
190
+ error.errors = traceback.format_exc()
154
191
 
155
192
  block_config_to_show = {}
156
193
  block_config_to_show.update(block_config)
@@ -162,8 +199,17 @@ class BlockLayoutItemResource(GenericResource):
162
199
 
163
200
  await UsageStatisticLogger().chart_impression(block_config_to_show)
164
201
 
165
- return self(merge_dict(block_config_to_show, dict(
166
- content=content,
167
- data=data,
168
- uuid=uuid,
169
- )), user, **kwargs)
202
+ return self(
203
+ merge_dict(
204
+ block_config_to_show,
205
+ dict(
206
+ content=content,
207
+ data=data,
208
+ error=error,
209
+ skip_render=skip_render,
210
+ uuid=uuid,
211
+ ),
212
+ ),
213
+ user,
214
+ **kwargs,
215
+ )
@@ -1,8 +1,10 @@
1
1
  from mage_ai.api.errors import ApiError
2
2
  from mage_ai.api.resources.GenericResource import GenericResource
3
+ from mage_ai.data_preparation.models.constants import DATAFRAME_SAMPLE_COUNT
3
4
  from mage_ai.data_preparation.models.pipeline import Pipeline
4
- from mage_ai.data_preparation.models.variable import VariableType
5
+ from mage_ai.data_preparation.models.variables.constants import VariableType
5
6
  from mage_ai.orchestration.db import safe_db_query
7
+ from mage_ai.settings.repo import get_repo_path
6
8
 
7
9
 
8
10
  class BlockOutputResource(GenericResource):
@@ -10,29 +12,60 @@ class BlockOutputResource(GenericResource):
10
12
  Resource to fetch block output for the notebook. Created to support legacy
11
13
  endpoint /api/pipelines/<pipeline_uuid>/blocks/<block_uuid>/outputs
12
14
  """
15
+
13
16
  @classmethod
14
17
  @safe_db_query
15
- def member(self, pk, user, **kwargs):
18
+ def member(cls, pk, user, **kwargs):
19
+ query = kwargs.get('query', {})
16
20
  block_uuid = pk
17
21
 
18
- query = kwargs.get('query', {})
19
22
  pipeline_uuid = query.get('pipeline_uuid', [None])
20
23
  if pipeline_uuid:
21
24
  pipeline_uuid = pipeline_uuid[0]
25
+
26
+ outputs_query = {}
27
+ for key in [
28
+ 'block_uuid',
29
+ 'csv_lines_only',
30
+ 'dynamic_block_index',
31
+ 'exclude_blank_variable_uuids',
32
+ 'execution_partition',
33
+ 'include_print_outputs',
34
+ 'sample',
35
+ 'sample_count',
36
+ 'variable_type',
37
+ ]:
38
+ value = query.get(key, [None])
39
+ if value is not None:
40
+ value = value[0]
41
+ if value is not None:
42
+ outputs_query[key] = value
43
+
44
+ for key, value in [
45
+ ('exclude_blank_variable_uuids', True),
46
+ ('include_print_outputs', False),
47
+ ('sample_count', DATAFRAME_SAMPLE_COUNT),
48
+ ('variable_type', VariableType.DATAFRAME),
49
+ ]:
50
+ if key not in outputs_query:
51
+ outputs_query[key] = value
52
+
22
53
  outputs = []
23
54
  if pipeline_uuid is not None:
24
- pipeline = Pipeline.get(pipeline_uuid)
55
+ repo_path = get_repo_path(user=user)
56
+ pipeline = Pipeline.get(pipeline_uuid, repo_path=repo_path)
25
57
  block = pipeline.get_block(block_uuid)
58
+
26
59
  error = ApiError.RESOURCE_ERROR.copy()
27
60
  if block is None:
28
61
  error.update(
29
- message=f'Block {block_uuid} does not exist in pipeline {pipeline_uuid}')
62
+ message=f'Block {block_uuid} does not exist in pipeline {pipeline_uuid}'
63
+ )
30
64
  raise ApiError(error)
65
+
31
66
  # Only fetch dataframe variables by default
32
67
  outputs = block.get_outputs(
33
- include_print_outputs=False,
34
- sample_count=None,
35
- variable_type=VariableType.DATAFRAME,
68
+ **outputs_query,
36
69
  )
37
70
 
38
- return self(dict(outputs=outputs), user, **kwargs)
71
+ return cls(dict(outputs=outputs), user, **kwargs)
@@ -110,6 +110,7 @@ class BlockResource(GenericResource):
110
110
  name = payload.get('name')
111
111
  block_name = name or payload.get('uuid')
112
112
  require_unique_name = payload.get('require_unique_name')
113
+ repo_path = get_repo_path(user=user)
113
114
 
114
115
  payload_config = payload.get('config') or {}
115
116
  replicated_block = None
@@ -208,7 +209,7 @@ class BlockResource(GenericResource):
208
209
 
209
210
  if payload_config and payload_config.get('custom_template_uuid'):
210
211
  template_uuid = payload_config.get('custom_template_uuid')
211
- custom_template = CustomBlockTemplate.load(template_uuid=template_uuid)
212
+ custom_template = CustomBlockTemplate.load(repo_path, template_uuid=template_uuid)
212
213
  block = custom_template.create_block(
213
214
  block_name,
214
215
  pipeline,
@@ -221,7 +222,7 @@ class BlockResource(GenericResource):
221
222
  block = Block.create(
222
223
  block_name,
223
224
  block_type,
224
- get_repo_path(),
225
+ repo_path,
225
226
  require_unique_name=require_unique_name,
226
227
  **block_attributes,
227
228
  )
@@ -234,7 +235,7 @@ class BlockResource(GenericResource):
234
235
 
235
236
  if pipeline:
236
237
  cache = await BlockCache.initialize_cache()
237
- cache.add_pipeline(block.to_dict(), pipeline)
238
+ cache.add_pipeline(block.to_dict(), pipeline, repo_path)
238
239
 
239
240
  cache_block_action_object = await BlockActionObjectCache.initialize_cache()
240
241
  cache_block_action_object.update_block(block)
@@ -27,12 +27,22 @@ class BlockRunResource(DatabaseResource):
27
27
  started_at = tup.started_at if hasattr(tup, 'started_at') else None
28
28
  status = tup.status if hasattr(tup, 'status') else None
29
29
  updated_at = tup.updated_at if hasattr(tup, 'updated_at') else None
30
- pipeline_schedule_id = tup.pipeline_schedule_id if hasattr(
31
- tup, 'pipeline_schedule_id',
32
- ) else None
33
- pipeline_schedule_name = tup.pipeline_schedule_name if hasattr(
34
- tup, 'pipeline_schedule_name',
35
- ) else None
30
+ pipeline_schedule_id = (
31
+ tup.pipeline_schedule_id
32
+ if hasattr(
33
+ tup,
34
+ 'pipeline_schedule_id',
35
+ )
36
+ else None
37
+ )
38
+ pipeline_schedule_name = (
39
+ tup.pipeline_schedule_name
40
+ if hasattr(
41
+ tup,
42
+ 'pipeline_schedule_name',
43
+ )
44
+ else None
45
+ )
36
46
  else:
37
47
  (
38
48
  block_uuid,
@@ -98,29 +108,22 @@ class BlockRunResource(DatabaseResource):
98
108
  ]
99
109
 
100
110
  query = (
101
- BlockRun.
102
- select(*columns).
103
- join(b, a.pipeline_run_id == b.id).
104
- join(c, b.pipeline_schedule_id == c.id)
111
+ BlockRun.select(*columns)
112
+ .join(b, a.pipeline_run_id == b.id)
113
+ .join(c, b.pipeline_schedule_id == c.id)
105
114
  )
106
115
 
107
116
  pipeline_run_id = query_arg.get('pipeline_run_id', [None])
108
117
  if pipeline_run_id:
109
118
  pipeline_run_id = pipeline_run_id[0]
110
119
  if pipeline_run_id and is_number(pipeline_run_id):
111
- query = (
112
- query.
113
- filter(a.pipeline_run_id == int(pipeline_run_id))
114
- )
120
+ query = query.filter(a.pipeline_run_id == int(pipeline_run_id))
115
121
 
116
122
  pipeline_uuid = query_arg.get('pipeline_uuid', [None])
117
123
  if pipeline_uuid:
118
124
  pipeline_uuid = pipeline_uuid[0]
119
125
  if pipeline_uuid:
120
- query = (
121
- query.
122
- filter(c.pipeline_uuid == pipeline_uuid)
123
- )
126
+ query = query.filter(c.pipeline_uuid == pipeline_uuid)
124
127
 
125
128
  # The order_by value should be an attribute on the BlockRun model.
126
129
  order_by_arg = query_arg.get('order_by', [None])
@@ -138,10 +141,12 @@ class BlockRunResource(DatabaseResource):
138
141
  try:
139
142
  br_col = getattr(a, col)
140
143
  initial_results = query.order_by(getattr(br_col, asc_desc)())
141
- except (AttributeError):
142
- raise Exception('Block run sort column/query is invalid. The sort column ' +
143
- 'must be an attribute of the BlockRun model. The sort direction ' +
144
- 'is either "asc" (ascending order) or "desc" (descending order).')
144
+ except AttributeError:
145
+ raise Exception(
146
+ 'Block run sort column/query is invalid. The sort column '
147
+ + 'must be an attribute of the BlockRun model. The sort direction '
148
+ + 'is either "asc" (ascending order) or "desc" (descending order).'
149
+ )
145
150
  else:
146
151
  initial_results = query.order_by(
147
152
  a.started_at.desc(),
@@ -4,6 +4,7 @@ from mage_ai.cluster_manager.constants import ClusterType
4
4
  from mage_ai.orchestration.db import safe_db_query
5
5
  from mage_ai.server.active_kernel import get_active_kernel_name
6
6
  from mage_ai.server.kernels import KernelName
7
+ from mage_ai.settings.repo import get_repo_path
7
8
 
8
9
 
9
10
  class ClusterResource(GenericResource):
@@ -41,7 +42,9 @@ class ClusterResource(GenericResource):
41
42
  error['message'] = 'Please include cluster_id in the request payload.'
42
43
  raise ApiError(error)
43
44
 
44
- emr_cluster_manager.set_active_cluster(cluster_id=cluster_id)
45
+ repo_path = get_repo_path(user=self.current_user)
46
+
47
+ emr_cluster_manager.set_active_cluster(repo_path=repo_path, cluster_id=cluster_id)
45
48
  success = True
46
49
 
47
50
  self.model.update(
@@ -20,6 +20,7 @@ from mage_ai.data_preparation.models.custom_templates.utils import (
20
20
  )
21
21
  from mage_ai.data_preparation.models.pipeline import Pipeline
22
22
  from mage_ai.data_preparation.templates.template import fetch_template_source
23
+ from mage_ai.settings.repo import get_repo_path
23
24
  from mage_ai.shared.hash import ignore_keys
24
25
  from mage_ai.shared.utils import clean_name
25
26
  from mage_ai.usage_statistics.logger import UsageStatisticLogger
@@ -64,14 +65,20 @@ class CustomTemplateResource(GenericResource):
64
65
  template_uuid = clean_name(template_uuid)
65
66
  payload['template_uuid'] = template_uuid
66
67
 
68
+ repo_path = get_repo_path(user=user)
67
69
  if DIRECTORY_FOR_BLOCK_TEMPLATES == object_type:
68
- custom_template = CustomBlockTemplate.load(template_uuid=template_uuid)
69
-
70
+ custom_template = CustomBlockTemplate.load(repo_path, template_uuid=template_uuid)
70
71
  if not custom_template:
71
- custom_template = CustomBlockTemplate(**ignore_keys(payload, [
72
- 'uuid',
73
- OBJECT_TYPE_KEY,
74
- ]))
72
+ custom_template = CustomBlockTemplate(
73
+ repo_path=repo_path,
74
+ **ignore_keys(
75
+ payload,
76
+ [
77
+ 'uuid',
78
+ OBJECT_TYPE_KEY,
79
+ ],
80
+ )
81
+ )
75
82
  if user:
76
83
  custom_template.user = dict(
77
84
  username=user.username,
@@ -88,10 +95,12 @@ class CustomTemplateResource(GenericResource):
88
95
  cache = await BlockActionObjectCache.initialize_cache()
89
96
  cache.update_custom_block_template(custom_template)
90
97
  elif DIRECTORY_FOR_PIPELINE_TEMPLATES == object_type:
91
- custom_template = CustomPipelineTemplate.load(template_uuid=template_uuid)
98
+ custom_template = CustomPipelineTemplate.load(
99
+ template_uuid=template_uuid, repo_path=repo_path
100
+ )
92
101
 
93
102
  if not custom_template:
94
- pipeline = Pipeline.get(payload.get('pipeline_uuid'))
103
+ pipeline = Pipeline.get(payload.get('pipeline_uuid'), repo_path=repo_path)
95
104
  custom_template = CustomPipelineTemplate.create_from_pipeline(
96
105
  pipeline,
97
106
  template_uuid,
@@ -120,12 +129,20 @@ class CustomTemplateResource(GenericResource):
120
129
 
121
130
  template_uuid = urllib.parse.unquote(pk)
122
131
 
132
+ repo_path = get_repo_path(user=user)
123
133
  try:
124
134
  if DIRECTORY_FOR_BLOCK_TEMPLATES == object_type:
125
- return self(CustomBlockTemplate.load(template_uuid=template_uuid), user, **kwargs)
135
+ return self(
136
+ CustomBlockTemplate.load(repo_path, template_uuid=template_uuid),
137
+ user,
138
+ **kwargs,
139
+ )
126
140
  elif DIRECTORY_FOR_PIPELINE_TEMPLATES == object_type:
127
141
  return self(
128
- CustomPipelineTemplate.load(template_uuid=template_uuid),
142
+ CustomPipelineTemplate.load(
143
+ repo_path,
144
+ template_uuid=template_uuid,
145
+ ),
129
146
  user,
130
147
  **kwargs,
131
148
  )
@@ -152,10 +169,13 @@ class CustomTemplateResource(GenericResource):
152
169
  cache = await BlockActionObjectCache.initialize_cache()
153
170
  cache.update_custom_block_template(self.model, remove=True)
154
171
 
155
- for key, value in ignore_keys(payload, [
156
- 'uuid',
157
- OBJECT_TYPE_KEY,
158
- ]).items():
172
+ for key, value in ignore_keys(
173
+ payload,
174
+ [
175
+ 'uuid',
176
+ OBJECT_TYPE_KEY,
177
+ ],
178
+ ).items():
159
179
  setattr(self.model, key, value)
160
180
  self.model.save()
161
181
 
@@ -43,7 +43,7 @@ class DataProviderResource(GenericResource):
43
43
  @classmethod
44
44
  @safe_db_query
45
45
  async def collection(self, query, meta, user, **kwargs):
46
- file_path = f'{get_repo_path()}/io_config.yaml'
46
+ file_path = f'{get_repo_path(user=user)}/io_config.yaml'
47
47
  profiles = []
48
48
  if os.path.exists(file_path):
49
49
  async with aiofiles.open(file_path, mode='r') as file:
@@ -1,6 +1,7 @@
1
1
  from mage_ai.api.resources.GenericResource import GenericResource
2
2
  from mage_ai.data_preparation.models.constants import PipelineType
3
3
  from mage_ai.data_preparation.models.pipeline import Pipeline
4
+ from mage_ai.settings.repo import get_repo_path
4
5
 
5
6
 
6
7
  class ExecutionStateResource(GenericResource):
@@ -17,7 +18,8 @@ class ExecutionStateResource(GenericResource):
17
18
  block_uuid = block_uuid[0]
18
19
 
19
20
  if pipeline_uuid and block_uuid:
20
- pipeline = await Pipeline.get_async(pipeline_uuid)
21
+ repo_path = get_repo_path(user=user)
22
+ pipeline = await Pipeline.get_async(pipeline_uuid, repo_path=repo_path)
21
23
  if pipeline and pipeline.type in [PipelineType.PYTHON, PipelineType.PYSPARK]:
22
24
  block = pipeline.get_block(block_uuid)
23
25
  if block and block.compute_management_enabled() and block.is_using_spark():
@@ -22,7 +22,10 @@ class FileContentResource(GenericResource):
22
22
  if 'payload' in kwargs and 'file_content' in kwargs['payload']:
23
23
  payload = kwargs['payload']['file_content']
24
24
  if 'block_uuid' in payload and 'pipeline_uuid' in payload:
25
- pipeline = await Pipeline.get_async(payload.get('pipeline_uuid'))
25
+ repo_path = get_repo_path(user=user)
26
+ pipeline = await Pipeline.get_async(
27
+ payload.get('pipeline_uuid'), repo_path=repo_path
28
+ )
26
29
  if pipeline:
27
30
  block = pipeline.get_block(payload.get('block_uuid'))
28
31
  if block:
@@ -57,7 +60,10 @@ class FileContentResource(GenericResource):
57
60
 
58
61
  await self.model.update_content_async(content)
59
62
 
60
- block_type = Block.block_type_from_path(self.model.dir_path)
63
+ block_type = Block.block_type_from_path(
64
+ self.model.dir_path,
65
+ repo_path=get_repo_path(user=self.current_user),
66
+ )
61
67
  if block_type:
62
68
  cache_block_action_object = await BlockActionObjectCache.initialize_cache()
63
69
  cache_block_action_object.update_block(block_file_absolute_path=self.model.file_path)