mage-ai 0.9.69__py3-none-any.whl → 0.9.71__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mage-ai might be problematic. Click here for more details.

Files changed (624) hide show
  1. mage_ai/ai/utils/xgboost.py +222 -0
  2. mage_ai/api/errors.py +37 -25
  3. mage_ai/api/operations/base.py +13 -1
  4. mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
  5. mage_ai/api/policies/BackfillPolicy.py +1 -0
  6. mage_ai/api/policies/BlockOutputPolicy.py +40 -17
  7. mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
  8. mage_ai/api/policies/KernelPolicy.py +55 -32
  9. mage_ai/api/policies/KernelProcessPolicy.py +56 -0
  10. mage_ai/api/policies/OutputPolicy.py +73 -41
  11. mage_ai/api/policies/PipelinePolicy.py +206 -137
  12. mage_ai/api/policies/WorkspacePolicy.py +1 -0
  13. mage_ai/api/presenters/BackfillPresenter.py +1 -0
  14. mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
  15. mage_ai/api/presenters/BlockPresenter.py +1 -1
  16. mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
  17. mage_ai/api/presenters/KernelPresenter.py +5 -26
  18. mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
  19. mage_ai/api/presenters/PipelinePresenter.py +18 -5
  20. mage_ai/api/presenters/StatusPresenter.py +2 -0
  21. mage_ai/api/presenters/SyncPresenter.py +25 -0
  22. mage_ai/api/resources/AutocompleteItemResource.py +1 -1
  23. mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
  24. mage_ai/api/resources/BlockOutputResource.py +42 -9
  25. mage_ai/api/resources/BlockResource.py +4 -3
  26. mage_ai/api/resources/BlockRunResource.py +27 -22
  27. mage_ai/api/resources/ClusterResource.py +4 -1
  28. mage_ai/api/resources/CustomTemplateResource.py +34 -14
  29. mage_ai/api/resources/DataProviderResource.py +1 -1
  30. mage_ai/api/resources/ExecutionStateResource.py +3 -1
  31. mage_ai/api/resources/FileContentResource.py +8 -2
  32. mage_ai/api/resources/FileResource.py +10 -4
  33. mage_ai/api/resources/FileVersionResource.py +3 -1
  34. mage_ai/api/resources/GitBranchResource.py +101 -31
  35. mage_ai/api/resources/GitCustomBranchResource.py +29 -1
  36. mage_ai/api/resources/GlobalDataProductResource.py +44 -7
  37. mage_ai/api/resources/GlobalHookResource.py +4 -1
  38. mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
  39. mage_ai/api/resources/IntegrationSourceResource.py +8 -4
  40. mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
  41. mage_ai/api/resources/KernelProcessResource.py +44 -0
  42. mage_ai/api/resources/KernelResource.py +25 -3
  43. mage_ai/api/resources/OauthResource.py +1 -1
  44. mage_ai/api/resources/OutputResource.py +33 -11
  45. mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
  46. mage_ai/api/resources/PipelineInteractionResource.py +31 -15
  47. mage_ai/api/resources/PipelineResource.py +258 -125
  48. mage_ai/api/resources/PipelineRunResource.py +52 -7
  49. mage_ai/api/resources/PipelineScheduleResource.py +11 -2
  50. mage_ai/api/resources/PipelineTriggerResource.py +6 -1
  51. mage_ai/api/resources/ProjectResource.py +18 -7
  52. mage_ai/api/resources/PullRequestResource.py +6 -4
  53. mage_ai/api/resources/SecretResource.py +1 -1
  54. mage_ai/api/resources/SeedResource.py +8 -1
  55. mage_ai/api/resources/StatusResource.py +21 -6
  56. mage_ai/api/resources/SyncResource.py +6 -8
  57. mage_ai/api/resources/VariableResource.py +46 -26
  58. mage_ai/api/resources/VersionControlProjectResource.py +9 -2
  59. mage_ai/api/resources/WidgetResource.py +1 -1
  60. mage_ai/api/resources/WorkspaceResource.py +6 -5
  61. mage_ai/api/views.py +47 -40
  62. mage_ai/authentication/permissions/seed.py +16 -2
  63. mage_ai/authentication/providers/oidc.py +21 -1
  64. mage_ai/autocomplete/utils.py +13 -9
  65. mage_ai/cache/base.py +1 -1
  66. mage_ai/cache/block.py +18 -12
  67. mage_ai/cache/block_action_object/__init__.py +33 -5
  68. mage_ai/cache/file.py +22 -19
  69. mage_ai/cache/pipeline.py +18 -12
  70. mage_ai/cli/main.py +1 -0
  71. mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
  72. mage_ai/cluster_manager/config.py +2 -2
  73. mage_ai/cluster_manager/kubernetes/workload_manager.py +52 -1
  74. mage_ai/cluster_manager/manage.py +1 -1
  75. mage_ai/cluster_manager/workspace/base.py +7 -1
  76. mage_ai/cluster_manager/workspace/kubernetes.py +22 -1
  77. mage_ai/command_center/applications/factory.py +10 -7
  78. mage_ai/command_center/applications/utils.py +2 -2
  79. mage_ai/command_center/files/factory.py +17 -15
  80. mage_ai/command_center/presenters/text.py +1 -1
  81. mage_ai/command_center/utils.py +25 -13
  82. mage_ai/data/__init__.py +0 -0
  83. mage_ai/data/constants.py +45 -0
  84. mage_ai/data/models/__init__.py +0 -0
  85. mage_ai/data/models/base.py +119 -0
  86. mage_ai/data/models/constants.py +1 -0
  87. mage_ai/data/models/generator.py +115 -0
  88. mage_ai/data/models/manager.py +168 -0
  89. mage_ai/data/models/pyarrow/__init__.py +0 -0
  90. mage_ai/data/models/pyarrow/record_batch.py +55 -0
  91. mage_ai/data/models/pyarrow/shared.py +21 -0
  92. mage_ai/data/models/pyarrow/table.py +8 -0
  93. mage_ai/data/models/reader.py +103 -0
  94. mage_ai/data/models/utils.py +59 -0
  95. mage_ai/data/models/writer.py +91 -0
  96. mage_ai/data/tabular/__init__.py +0 -0
  97. mage_ai/data/tabular/constants.py +23 -0
  98. mage_ai/data/tabular/mocks.py +19 -0
  99. mage_ai/data/tabular/models.py +126 -0
  100. mage_ai/data/tabular/reader.py +602 -0
  101. mage_ai/data/tabular/utils.py +102 -0
  102. mage_ai/data/tabular/writer.py +266 -0
  103. mage_ai/data/variables/__init__.py +0 -0
  104. mage_ai/data/variables/wrapper.py +54 -0
  105. mage_ai/data_cleaner/analysis/charts.py +61 -39
  106. mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
  107. mage_ai/data_cleaner/estimators/encoders.py +5 -2
  108. mage_ai/data_integrations/utils/scheduler.py +16 -11
  109. mage_ai/data_preparation/decorators.py +1 -0
  110. mage_ai/data_preparation/executors/block_executor.py +237 -155
  111. mage_ai/data_preparation/executors/k8s_block_executor.py +30 -7
  112. mage_ai/data_preparation/executors/k8s_pipeline_executor.py +30 -7
  113. mage_ai/data_preparation/executors/streaming_pipeline_executor.py +2 -2
  114. mage_ai/data_preparation/git/__init__.py +77 -29
  115. mage_ai/data_preparation/git/api.py +69 -8
  116. mage_ai/data_preparation/git/utils.py +64 -34
  117. mage_ai/data_preparation/logging/logger_manager.py +4 -3
  118. mage_ai/data_preparation/models/block/__init__.py +1562 -879
  119. mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
  120. mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
  121. mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
  122. mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
  123. mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
  124. mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
  125. mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
  126. mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
  127. mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
  128. mage_ai/data_preparation/models/block/dynamic/utils.py +295 -167
  129. mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
  130. mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
  131. mage_ai/data_preparation/models/block/extension/utils.py +10 -1
  132. mage_ai/data_preparation/models/block/global_data_product/__init__.py +35 -3
  133. mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
  134. mage_ai/data_preparation/models/block/outputs.py +722 -0
  135. mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
  136. mage_ai/data_preparation/models/block/r/__init__.py +56 -38
  137. mage_ai/data_preparation/models/block/remote/__init__.py +0 -0
  138. mage_ai/data_preparation/models/block/remote/models.py +58 -0
  139. mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
  140. mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
  141. mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
  142. mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
  143. mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
  144. mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
  145. mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
  146. mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
  147. mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
  148. mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
  149. mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
  150. mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
  151. mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
  152. mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
  153. mage_ai/data_preparation/models/block/utils.py +164 -69
  154. mage_ai/data_preparation/models/constants.py +21 -14
  155. mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
  156. mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
  157. mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
  158. mage_ai/data_preparation/models/file.py +41 -28
  159. mage_ai/data_preparation/models/global_data_product/__init__.py +100 -58
  160. mage_ai/data_preparation/models/global_hooks/models.py +1 -0
  161. mage_ai/data_preparation/models/interfaces.py +29 -0
  162. mage_ai/data_preparation/models/pipeline.py +374 -185
  163. mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
  164. mage_ai/data_preparation/models/pipelines/seed.py +1 -1
  165. mage_ai/data_preparation/models/project/__init__.py +66 -18
  166. mage_ai/data_preparation/models/project/constants.py +2 -0
  167. mage_ai/data_preparation/models/triggers/__init__.py +124 -26
  168. mage_ai/data_preparation/models/utils.py +467 -17
  169. mage_ai/data_preparation/models/variable.py +1028 -137
  170. mage_ai/data_preparation/models/variables/__init__.py +0 -0
  171. mage_ai/data_preparation/models/variables/cache.py +149 -0
  172. mage_ai/data_preparation/models/variables/constants.py +72 -0
  173. mage_ai/data_preparation/models/variables/summarizer.py +336 -0
  174. mage_ai/data_preparation/models/variables/utils.py +77 -0
  175. mage_ai/data_preparation/models/widget/__init__.py +63 -41
  176. mage_ai/data_preparation/models/widget/charts.py +40 -27
  177. mage_ai/data_preparation/models/widget/constants.py +2 -0
  178. mage_ai/data_preparation/models/widget/utils.py +3 -3
  179. mage_ai/data_preparation/preferences.py +3 -3
  180. mage_ai/data_preparation/repo_manager.py +55 -21
  181. mage_ai/data_preparation/storage/base_storage.py +2 -2
  182. mage_ai/data_preparation/storage/gcs_storage.py +7 -4
  183. mage_ai/data_preparation/storage/local_storage.py +18 -9
  184. mage_ai/data_preparation/storage/s3_storage.py +5 -2
  185. mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
  186. mage_ai/data_preparation/variable_manager.py +281 -76
  187. mage_ai/io/base.py +3 -2
  188. mage_ai/io/bigquery.py +1 -0
  189. mage_ai/io/redshift.py +7 -5
  190. mage_ai/kernels/__init__.py +0 -0
  191. mage_ai/kernels/models.py +188 -0
  192. mage_ai/kernels/utils.py +169 -0
  193. mage_ai/orchestration/concurrency.py +6 -2
  194. mage_ai/orchestration/db/__init__.py +1 -0
  195. mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
  196. mage_ai/orchestration/db/migrations/versions/42a14d6143f1_update_token_column_type.py +54 -0
  197. mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
  198. mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
  199. mage_ai/orchestration/db/models/oauth.py +12 -18
  200. mage_ai/orchestration/db/models/projects.py +10 -0
  201. mage_ai/orchestration/db/models/schedules.py +225 -187
  202. mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
  203. mage_ai/orchestration/db/models/utils.py +46 -5
  204. mage_ai/orchestration/metrics/pipeline_run.py +8 -9
  205. mage_ai/orchestration/notification/sender.py +38 -15
  206. mage_ai/orchestration/pipeline_scheduler_original.py +64 -33
  207. mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
  208. mage_ai/orchestration/run_status_checker.py +11 -4
  209. mage_ai/orchestration/triggers/api.py +41 -2
  210. mage_ai/orchestration/triggers/global_data_product.py +9 -4
  211. mage_ai/orchestration/triggers/utils.py +10 -1
  212. mage_ai/orchestration/utils/resources.py +3 -0
  213. mage_ai/presenters/charts/data_sources/base.py +4 -2
  214. mage_ai/presenters/charts/data_sources/block.py +15 -9
  215. mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
  216. mage_ai/presenters/charts/data_sources/constants.py +1 -0
  217. mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
  218. mage_ai/presenters/interactions/models.py +11 -7
  219. mage_ai/presenters/pages/loaders/pipelines.py +5 -3
  220. mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
  221. mage_ai/presenters/utils.py +2 -0
  222. mage_ai/server/api/blocks.py +2 -1
  223. mage_ai/server/api/downloads.py +9 -2
  224. mage_ai/server/api/runs.py +151 -0
  225. mage_ai/server/api/triggers.py +3 -1
  226. mage_ai/server/constants.py +1 -1
  227. mage_ai/server/frontend_dist/404.html +8 -8
  228. mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
  229. mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
  230. mage_ai/server/frontend_dist/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
  231. mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
  232. mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
  233. mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
  234. mage_ai/server/frontend_dist/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
  235. mage_ai/server/frontend_dist/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
  236. mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
  237. mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
  238. mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
  239. mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
  240. mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
  241. mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
  242. mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
  243. mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
  244. mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
  245. mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
  246. mage_ai/server/frontend_dist/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
  247. mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
  248. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
  249. mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
  250. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
  251. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
  252. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
  253. mage_ai/server/frontend_dist/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
  254. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
  255. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
  256. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
  257. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
  258. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
  259. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
  260. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
  261. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
  262. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
  263. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
  264. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
  265. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
  266. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
  267. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
  268. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
  269. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
  270. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
  271. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
  272. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
  273. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
  274. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
  275. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
  276. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
  277. mage_ai/server/frontend_dist/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
  278. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
  279. mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
  280. mage_ai/server/frontend_dist/block-layout.html +3 -3
  281. mage_ai/server/frontend_dist/compute.html +6 -6
  282. mage_ai/server/frontend_dist/files.html +6 -6
  283. mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
  284. mage_ai/server/frontend_dist/global-data-products.html +6 -6
  285. mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
  286. mage_ai/server/frontend_dist/global-hooks.html +6 -6
  287. mage_ai/server/frontend_dist/index.html +3 -3
  288. mage_ai/server/frontend_dist/manage/files.html +6 -6
  289. mage_ai/server/frontend_dist/manage/settings.html +6 -6
  290. mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
  291. mage_ai/server/frontend_dist/manage/users/new.html +6 -6
  292. mage_ai/server/frontend_dist/manage/users.html +6 -6
  293. mage_ai/server/frontend_dist/manage.html +6 -6
  294. mage_ai/server/frontend_dist/oauth.html +5 -5
  295. mage_ai/server/frontend_dist/overview.html +6 -6
  296. mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
  297. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  298. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
  299. mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
  300. mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
  301. mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
  302. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  303. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  304. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
  305. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
  306. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
  307. mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
  308. mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
  309. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  310. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
  311. mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
  312. mage_ai/server/frontend_dist/pipelines.html +6 -6
  313. mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
  314. mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
  315. mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
  316. mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
  317. mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
  318. mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
  319. mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
  320. mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
  321. mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
  322. mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
  323. mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
  324. mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
  325. mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
  326. mage_ai/server/frontend_dist/settings.html +3 -3
  327. mage_ai/server/frontend_dist/sign-in.html +15 -15
  328. mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
  329. mage_ai/server/frontend_dist/templates.html +6 -6
  330. mage_ai/server/frontend_dist/terminal.html +6 -6
  331. mage_ai/server/frontend_dist/test.html +3 -3
  332. mage_ai/server/frontend_dist/triggers.html +6 -6
  333. mage_ai/server/frontend_dist/version-control.html +6 -6
  334. mage_ai/server/frontend_dist_base_path_template/404.html +8 -8
  335. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
  336. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
  337. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
  338. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
  339. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
  340. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
  341. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
  342. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
  343. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
  344. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
  345. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
  346. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
  347. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
  348. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
  349. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
  350. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
  351. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
  352. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
  353. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
  354. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
  355. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
  356. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
  357. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
  358. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
  359. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
  360. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
  361. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
  362. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
  363. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
  364. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
  365. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
  366. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
  367. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
  368. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
  369. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
  370. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
  371. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
  372. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
  373. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
  374. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
  375. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
  376. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
  377. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
  378. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
  379. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
  380. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
  381. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
  382. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
  383. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
  384. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
  385. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
  386. mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
  387. mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
  388. mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
  389. mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
  390. mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
  391. mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
  392. mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
  393. mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
  394. mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
  395. mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
  396. mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
  397. mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
  398. mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
  399. mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
  400. mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
  401. mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
  402. mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
  403. mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
  404. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  405. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
  406. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
  407. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
  408. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
  409. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  410. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  411. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
  412. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
  413. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
  414. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
  415. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
  416. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  417. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
  418. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
  419. mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
  420. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
  421. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
  422. mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
  423. mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
  424. mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
  425. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
  426. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
  427. mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
  428. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
  429. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
  430. mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
  431. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
  432. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
  433. mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
  434. mage_ai/server/frontend_dist_base_path_template/sign-in.html +15 -15
  435. mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
  436. mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
  437. mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
  438. mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
  439. mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
  440. mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
  441. mage_ai/server/kernel_output_parser.py +4 -1
  442. mage_ai/server/scheduler_manager.py +12 -1
  443. mage_ai/server/server.py +69 -42
  444. mage_ai/server/utils/custom_output.py +284 -0
  445. mage_ai/server/utils/execute_custom_code.py +245 -0
  446. mage_ai/server/utils/output_display.py +123 -289
  447. mage_ai/server/websocket_server.py +116 -69
  448. mage_ai/services/aws/ecs/ecs.py +1 -0
  449. mage_ai/services/k8s/config.py +27 -4
  450. mage_ai/services/k8s/job_manager.py +6 -1
  451. mage_ai/services/k8s/utils.py +97 -0
  452. mage_ai/services/ssh/aws/emr/utils.py +8 -8
  453. mage_ai/settings/keys/auth.py +1 -0
  454. mage_ai/settings/platform/__init__.py +159 -38
  455. mage_ai/settings/platform/constants.py +5 -0
  456. mage_ai/settings/platform/utils.py +53 -10
  457. mage_ai/settings/repo.py +26 -12
  458. mage_ai/settings/server.py +128 -37
  459. mage_ai/shared/array.py +24 -1
  460. mage_ai/shared/complex.py +45 -0
  461. mage_ai/shared/config.py +2 -1
  462. mage_ai/shared/custom_logger.py +11 -0
  463. mage_ai/shared/dates.py +10 -6
  464. mage_ai/shared/files.py +63 -8
  465. mage_ai/shared/hash.py +33 -9
  466. mage_ai/shared/io.py +9 -5
  467. mage_ai/shared/models.py +82 -24
  468. mage_ai/shared/outputs.py +87 -0
  469. mage_ai/shared/parsers.py +144 -13
  470. mage_ai/shared/path_fixer.py +11 -7
  471. mage_ai/shared/singletons/__init__.py +0 -0
  472. mage_ai/shared/singletons/base.py +47 -0
  473. mage_ai/shared/singletons/memory.py +38 -0
  474. mage_ai/shared/strings.py +34 -1
  475. mage_ai/shared/yaml.py +24 -0
  476. mage_ai/streaming/sinks/oracledb.py +57 -0
  477. mage_ai/streaming/sinks/sink_factory.py +4 -0
  478. mage_ai/system/__init__.py +0 -0
  479. mage_ai/system/constants.py +14 -0
  480. mage_ai/system/memory/__init__.py +0 -0
  481. mage_ai/system/memory/constants.py +1 -0
  482. mage_ai/system/memory/manager.py +174 -0
  483. mage_ai/system/memory/presenters.py +158 -0
  484. mage_ai/system/memory/process.py +216 -0
  485. mage_ai/system/memory/samples.py +13 -0
  486. mage_ai/system/memory/utils.py +656 -0
  487. mage_ai/system/memory/wrappers.py +177 -0
  488. mage_ai/system/models.py +58 -0
  489. mage_ai/system/storage/__init__.py +0 -0
  490. mage_ai/system/storage/utils.py +29 -0
  491. mage_ai/tests/api/endpoints/mixins.py +2 -2
  492. mage_ai/tests/api/endpoints/test_blocks.py +2 -1
  493. mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
  494. mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
  495. mage_ai/tests/api/endpoints/test_projects.py +2 -1
  496. mage_ai/tests/api/operations/base/mixins.py +1 -1
  497. mage_ai/tests/api/operations/base/test_base.py +27 -27
  498. mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
  499. mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
  500. mage_ai/tests/api/operations/test_syncs.py +6 -4
  501. mage_ai/tests/api/resources/test_pipeline_resource.py +11 -4
  502. mage_ai/tests/authentication/oauth/test_utils.py +1 -1
  503. mage_ai/tests/authentication/providers/test_oidc.py +59 -0
  504. mage_ai/tests/base_test.py +2 -2
  505. mage_ai/tests/data/__init__.py +0 -0
  506. mage_ai/tests/data/models/__init__.py +0 -0
  507. mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
  508. mage_ai/tests/data_preparation/git/test_git.py +4 -1
  509. mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
  510. mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
  511. mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
  512. mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
  513. mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
  514. mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
  515. mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
  516. mage_ai/tests/data_preparation/models/block/test_global_data_product.py +5 -2
  517. mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
  518. mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
  519. mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
  520. mage_ai/tests/data_preparation/models/test_block.py +115 -120
  521. mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
  522. mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
  523. mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
  524. mage_ai/tests/data_preparation/models/test_project.py +4 -1
  525. mage_ai/tests/data_preparation/models/test_utils.py +80 -0
  526. mage_ai/tests/data_preparation/models/test_variable.py +242 -69
  527. mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
  528. mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
  529. mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
  530. mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
  531. mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
  532. mage_ai/tests/factory.py +64 -43
  533. mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
  534. mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
  535. mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
  536. mage_ai/tests/orchestration/triggers/test_global_data_product.py +141 -138
  537. mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
  538. mage_ai/tests/server/test_server.py +19 -0
  539. mage_ai/tests/services/k8s/test_job_manager.py +27 -6
  540. mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
  541. mage_ai/tests/test_shared.py +61 -0
  542. mage_ai/usage_statistics/logger.py +7 -2
  543. mage_ai/utils/code.py +33 -19
  544. mage_ai/version_control/branch/utils.py +2 -1
  545. mage_ai/version_control/models.py +3 -2
  546. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/METADATA +6 -3
  547. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/RECORD +555 -454
  548. mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
  549. mage_ai/server/frontend_dist/_next/static/_krrrgup_C-dPOpX36S8I/_buildManifest.js +0 -1
  550. mage_ai/server/frontend_dist/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
  551. mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
  552. mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
  553. mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
  554. mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
  555. mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
  556. mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
  557. mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
  558. mage_ai/server/frontend_dist/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
  559. mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
  560. mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
  561. mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
  562. mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
  563. mage_ai/server/frontend_dist/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
  564. mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
  565. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
  566. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
  567. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
  568. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
  569. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
  570. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
  571. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
  572. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
  573. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
  574. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
  575. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
  576. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
  577. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
  578. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
  579. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
  580. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
  581. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
  582. mage_ai/server/frontend_dist_base_path_template/_next/static/KLL5mirre9d7_ZeEpaw3s/_buildManifest.js +0 -1
  583. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
  584. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
  585. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
  586. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
  587. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
  588. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
  589. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
  590. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
  591. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
  592. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
  593. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
  594. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
  595. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
  596. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
  597. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
  598. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
  599. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
  600. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
  601. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
  602. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
  603. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
  604. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
  605. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
  606. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
  607. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
  608. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
  609. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
  610. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
  611. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
  612. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
  613. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
  614. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
  615. mage_ai/shared/memory.py +0 -90
  616. mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
  617. /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
  618. /mage_ai/server/frontend_dist/_next/static/{_krrrgup_C-dPOpX36S8I → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
  619. /mage_ai/server/frontend_dist_base_path_template/_next/static/{KLL5mirre9d7_ZeEpaw3s → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
  620. /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
  621. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
  622. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
  623. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
  624. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
@@ -1,3 +1,4 @@
1
+ from sqlalchemy import and_, desc, func
1
2
  from sqlalchemy.orm import selectinload
2
3
 
3
4
  from mage_ai.api.operations.constants import META_KEY_LIMIT, META_KEY_OFFSET
@@ -5,7 +6,10 @@ from mage_ai.api.resources.DatabaseResource import DatabaseResource
5
6
  from mage_ai.api.utils import get_query_timestamps
6
7
  from mage_ai.cache.tag import TagCache
7
8
  from mage_ai.data_preparation.models.block.utils import get_all_descendants
8
- from mage_ai.data_preparation.models.constants import PipelineType
9
+ from mage_ai.data_preparation.models.constants import (
10
+ PIPELINE_RUN_STATUS_LAST_RUN_FAILED,
11
+ PipelineType,
12
+ )
9
13
  from mage_ai.data_preparation.models.pipeline import Pipeline
10
14
  from mage_ai.data_preparation.models.triggers import (
11
15
  ScheduleInterval,
@@ -94,9 +98,38 @@ class PipelineRunResource(DatabaseResource):
94
98
 
95
99
  repo_pipeline_schedule_ids = [s.id for s in PipelineSchedule.repo_query]
96
100
 
101
+ if status == PIPELINE_RUN_STATUS_LAST_RUN_FAILED:
102
+ """
103
+ The initial query below is used to get the last pipeline run retry
104
+ (or individual pipeline run if there are no retries) for each
105
+ grouping of pipeline runs with the same execution_date,
106
+ pipeline_uuid, and pipeline_schedule_id.
107
+ """
108
+ latest_pipeline_runs = PipelineRun.select(
109
+ PipelineRun.id,
110
+ func.row_number()
111
+ .over(
112
+ partition_by=(
113
+ PipelineRun.execution_date,
114
+ PipelineRun.pipeline_schedule_id,
115
+ PipelineRun.pipeline_uuid,
116
+ ),
117
+ order_by=desc(PipelineRun.id))
118
+ .label('row_number')
119
+ ).cte(name='latest_pipeline_runs')
120
+ query = (PipelineRun.select(
121
+ PipelineRun,
122
+ )
123
+ .join(latest_pipeline_runs, and_(
124
+ PipelineRun.id == latest_pipeline_runs.c.id,
125
+ latest_pipeline_runs.c.row_number == 1,
126
+ ))
127
+ )
128
+ else:
129
+ query = PipelineRun.query
130
+
97
131
  results = (
98
- PipelineRun
99
- .query
132
+ query
100
133
  .filter(PipelineRun.pipeline_schedule_id.in_(repo_pipeline_schedule_ids))
101
134
  .options(selectinload(PipelineRun.block_runs))
102
135
  .options(selectinload(PipelineRun.pipeline_schedule))
@@ -120,8 +153,12 @@ class PipelineRunResource(DatabaseResource):
120
153
  results = results.filter(PipelineRun.pipeline_uuid.in_(pipeline_uuids))
121
154
  if pipeline_uuids_with_tags:
122
155
  results = results.filter(PipelineRun.pipeline_uuid.in_(pipeline_uuids_with_tags))
123
- if status is not None:
156
+
157
+ if status == PIPELINE_RUN_STATUS_LAST_RUN_FAILED:
158
+ results = results.filter(PipelineRun.status == PipelineRun.PipelineRunStatus.FAILED)
159
+ elif status is not None:
124
160
  results = results.filter(PipelineRun.status == status)
161
+
125
162
  if statuses:
126
163
  results = results.filter(PipelineRun.status.in_(statuses))
127
164
  if start_timestamp is not None:
@@ -152,6 +189,7 @@ class PipelineRunResource(DatabaseResource):
152
189
  @classmethod
153
190
  @safe_db_query
154
191
  async def process_collection(self, query_arg, meta, user, **kwargs):
192
+ context_data = kwargs.get('context_data')
155
193
  total_results = await self.collection(query_arg, meta, user, **kwargs)
156
194
  total_count = total_results.count()
157
195
 
@@ -262,7 +300,11 @@ class PipelineRunResource(DatabaseResource):
262
300
  }
263
301
 
264
302
  if include_pipeline_uuids:
265
- pipeline_uuids = Pipeline.get_all_pipelines_all_projects(get_repo_path())
303
+ repo_path = get_repo_path(context_data=context_data, user=user)
304
+ pipeline_uuids = Pipeline.get_all_pipelines_all_projects(
305
+ context_data=context_data,
306
+ repo_path=repo_path,
307
+ )
266
308
  result_set.metadata['pipeline_uuids'] = pipeline_uuids
267
309
 
268
310
  return result_set
@@ -272,7 +314,8 @@ class PipelineRunResource(DatabaseResource):
272
314
  def create(self, payload, user, **kwargs):
273
315
  pipeline_schedule = kwargs.get('parent_model')
274
316
 
275
- pipeline = Pipeline.get(pipeline_schedule.pipeline_uuid)
317
+ repo_path = get_repo_path(user=user)
318
+ pipeline = Pipeline.get(pipeline_schedule.pipeline_uuid, repo_path=repo_path)
276
319
  configured_payload, _ = configure_pipeline_run_payload(
277
320
  pipeline_schedule,
278
321
  pipeline.type,
@@ -295,9 +338,10 @@ class PipelineRunResource(DatabaseResource):
295
338
 
296
339
  @safe_db_query
297
340
  def update(self, payload, **kwargs):
341
+ repo_path = get_repo_path(user=self.current_user)
298
342
  if 'retry_blocks' == payload.get('pipeline_run_action'):
299
343
  self.model.refresh()
300
- pipeline = Pipeline.get(self.model.pipeline_uuid)
344
+ pipeline = Pipeline.get(self.model.pipeline_uuid, repo_path=repo_path)
301
345
  block_runs_to_retry = []
302
346
  from_block_uuid = payload.get('from_block_uuid')
303
347
  if from_block_uuid is not None:
@@ -348,6 +392,7 @@ class PipelineRunResource(DatabaseResource):
348
392
  pipeline = Pipeline.get(
349
393
  self.model.pipeline_uuid,
350
394
  check_if_exists=True,
395
+ repo_path=repo_path,
351
396
  )
352
397
 
353
398
  stop_pipeline_run(self.model, pipeline)
@@ -253,7 +253,10 @@ class PipelineScheduleResource(DatabaseResource):
253
253
  payload['pipeline_uuid'] = pipeline.uuid
254
254
 
255
255
  if 'repo_path' not in payload:
256
- payload['repo_path'] = (pipeline.repo_path if pipeline else None) or get_repo_path()
256
+ payload['repo_path'] = (
257
+ (pipeline.repo_path if pipeline else None)
258
+ or get_repo_path(user=user)
259
+ )
257
260
  if 'token' not in payload:
258
261
  payload['token'] = uuid.uuid4().hex
259
262
  if payload.get('status') == ScheduleStatus.ACTIVE and \
@@ -403,10 +406,13 @@ class PipelineScheduleResource(DatabaseResource):
403
406
  if updated_status == ScheduleStatus.ACTIVE and self.model.status == ScheduleStatus.INACTIVE:
404
407
  payload['last_enabled_at'] = datetime.now(tz=pytz.UTC)
405
408
 
409
+ old_name = self.model.name
410
+
406
411
  resource = super().update(payload)
407
412
  updated_model = resource.model
408
413
 
409
- pipeline = Pipeline.get(updated_model.pipeline_uuid)
414
+ repo_path = get_repo_path(user=self.current_user)
415
+ pipeline = Pipeline.get(updated_model.pipeline_uuid, repo_path=repo_path)
410
416
  if pipeline:
411
417
  trigger = Trigger(
412
418
  last_enabled_at=updated_model.last_enabled_at,
@@ -418,17 +424,20 @@ class PipelineScheduleResource(DatabaseResource):
418
424
  sla=updated_model.sla,
419
425
  start_time=updated_model.start_time,
420
426
  status=updated_model.status,
427
+ token=updated_model.token,
421
428
  variables=updated_model.variables,
422
429
  )
423
430
 
424
431
  update_only_if_exists = (
425
432
  not pipeline.should_save_trigger_in_code_automatically()
426
433
  )
434
+ old_trigger_name = old_name if old_name != updated_model.name else None
427
435
 
428
436
  add_or_update_trigger_for_pipeline_and_persist(
429
437
  trigger,
430
438
  pipeline.uuid,
431
439
  update_only_if_exists=update_only_if_exists,
440
+ old_trigger_name=old_trigger_name,
432
441
  )
433
442
 
434
443
  return self
@@ -7,6 +7,7 @@ from mage_ai.data_preparation.models.triggers import (
7
7
  )
8
8
  from mage_ai.orchestration.db import safe_db_query
9
9
  from mage_ai.orchestration.db.models.schedules import PipelineSchedule
10
+ from mage_ai.settings.repo import get_repo_path
10
11
 
11
12
 
12
13
  class PipelineTriggerResource(GenericResource):
@@ -16,7 +17,10 @@ class PipelineTriggerResource(GenericResource):
16
17
  parent_model = kwargs['parent_model']
17
18
 
18
19
  return self.build_result_set(
19
- get_triggers_by_pipeline(parent_model.uuid),
20
+ get_triggers_by_pipeline(
21
+ parent_model.uuid,
22
+ repo_path=get_repo_path(context_data=kwargs.get('context_data'), user=user),
23
+ ),
20
24
  user,
21
25
  **kwargs,
22
26
  )
@@ -46,6 +50,7 @@ class PipelineTriggerResource(GenericResource):
46
50
  sla=pipeline_schedule.sla,
47
51
  start_time=pipeline_schedule.start_time,
48
52
  status=pipeline_schedule.status,
53
+ token=pipeline_schedule.token,
49
54
  variables=pipeline_schedule.variables,
50
55
  )
51
56
  else:
@@ -26,8 +26,19 @@ from mage_ai.shared.hash import combine_into, merge_dict
26
26
  from mage_ai.usage_statistics.logger import UsageStatisticLogger
27
27
 
28
28
 
29
- async def build_project(repo_config=None, root_project: bool = False, **kwargs):
30
- project = Project(repo_config=repo_config, root_project=root_project)
29
+ async def build_project(
30
+ repo_config=None,
31
+ repo_path: str = None,
32
+ root_project: bool = False,
33
+ user=None,
34
+ **kwargs,
35
+ ):
36
+ project = Project(
37
+ repo_config=repo_config,
38
+ context_data=kwargs.get('context_data'),
39
+ root_project=root_project,
40
+ user=user,
41
+ )
31
42
 
32
43
  model = merge_dict(project.repo_config.to_dict(), dict(
33
44
  emr_config=project.emr_config,
@@ -36,7 +47,7 @@ async def build_project(repo_config=None, root_project: bool = False, **kwargs):
36
47
  features_override=project.features_override,
37
48
  latest_version=await project.latest_version(),
38
49
  name=project.name,
39
- platform_settings=project.platform_settings,
50
+ platform_settings=project.platform_settings(),
40
51
  project_uuid=project.project_uuid,
41
52
  projects=project.projects(),
42
53
  remote_variables_dir=project.remote_variables_dir,
@@ -137,13 +148,13 @@ class ProjectResource(GenericResource):
137
148
  @classmethod
138
149
  @safe_db_query
139
150
  async def member(self, _, user, **kwargs):
140
- model = await build_project(**kwargs)
151
+ model = await build_project(user=user, **kwargs)
141
152
  return self(model, user, **kwargs)
142
153
 
143
154
  @safe_db_query
144
155
  async def update(self, payload, **kwargs):
145
156
  if payload.get('activate_project'):
146
- activate_project(payload.get('activate_project'))
157
+ activate_project(payload.get('activate_project'), user=self.current_user)
147
158
 
148
159
  platform_settings = payload.get('platform_settings')
149
160
  root_project = payload.get('root_project')
@@ -151,7 +162,7 @@ class ProjectResource(GenericResource):
151
162
 
152
163
  if root_project and platform_settings:
153
164
  update_settings(settings=platform_settings)
154
- self.model = await build_project(repo_config)
165
+ self.model = await build_project(repo_config, user=self.current_user)
155
166
  return self
156
167
 
157
168
  data = {}
@@ -206,7 +217,7 @@ class ProjectResource(GenericResource):
206
217
  if len(data.keys()) >= 1:
207
218
  repo_config.save(**data)
208
219
 
209
- self.model = await build_project(repo_config)
220
+ self.model = await build_project(repo_config, user=self.current_user)
210
221
 
211
222
  if should_log_project:
212
223
  await UsageStatisticLogger().project_impression()
@@ -3,9 +3,11 @@ from typing import Dict
3
3
  from mage_ai.api.errors import ApiError
4
4
  from mage_ai.api.resources.GenericResource import GenericResource
5
5
  from mage_ai.authentication.oauth.constants import ProviderName
6
- from mage_ai.data_preparation.git import api
7
6
  from mage_ai.data_preparation.git.clients.base import Client as GitClient
8
- from mage_ai.data_preparation.git.utils import get_provider_from_remote_url
7
+ from mage_ai.data_preparation.git.utils import (
8
+ get_oauth_access_token_for_user,
9
+ get_provider_from_remote_url,
10
+ )
9
11
 
10
12
 
11
13
  def pull_request_to_dict(pr) -> Dict:
@@ -40,7 +42,7 @@ class PullRequestResource(GenericResource):
40
42
  if remote_url:
41
43
  provider = get_provider_from_remote_url(remote_url)
42
44
 
43
- access_token = api.get_access_token_for_user(user, provider=provider)
45
+ access_token = get_oauth_access_token_for_user(user, provider=provider)
44
46
 
45
47
  if access_token:
46
48
  arr = GitClient.get_client_for_provider(provider)(
@@ -84,7 +86,7 @@ class PullRequestResource(GenericResource):
84
86
  remote_url = payload.get('remote_url')
85
87
  provider = get_provider_from_remote_url(remote_url)
86
88
 
87
- access_token = api.get_access_token_for_user(user, provider=provider)
89
+ access_token = get_oauth_access_token_for_user(user, provider=provider)
88
90
  if not access_token:
89
91
  error.update(
90
92
  dict(
@@ -56,7 +56,7 @@ class SecretResource(DatabaseResource):
56
56
  @classmethod
57
57
  @safe_db_query
58
58
  def member(self, pk, user, **kwargs):
59
- repo_path = get_repo_path()
59
+ repo_path = get_repo_path(user=user)
60
60
  model = Secret.query.filter(Secret.repo_name == repo_path, Secret.name == pk).first()
61
61
 
62
62
  if not model:
@@ -1,3 +1,6 @@
1
+ import asyncio
2
+ import threading
3
+
1
4
  from mage_ai.api.resources.GenericResource import GenericResource
2
5
  from mage_ai.authentication.permissions.seed import bootstrap_permissions
3
6
 
@@ -7,6 +10,10 @@ class SeedResource(GenericResource):
7
10
  async def create(self, payload, user, **kwargs):
8
11
  if payload.get('roles') and payload.get('permissions'):
9
12
  policy_names = payload.get('policy_names')
10
- await bootstrap_permissions(policy_names=policy_names)
13
+ t = threading.Thread(
14
+ target=asyncio.run,
15
+ args=(bootstrap_permissions(policy_names=policy_names),),
16
+ )
17
+ t.start()
11
18
 
12
19
  return self({}, user, **kwargs)
@@ -31,9 +31,10 @@ class StatusResource(GenericResource):
31
31
  KUBE_NAMESPACE,
32
32
  )
33
33
 
34
+ context_data = kwargs.get('context_data')
34
35
  instance_type = None
35
- project_type = get_project_type()
36
- repo_config = get_repo_config()
36
+ repo_config = get_repo_config(context_data=context_data, user=user)
37
+ project_type = get_project_type(repo_config=repo_config)
37
38
  if repo_config.cluster_type:
38
39
  instance_type = repo_config.cluster_type
39
40
  elif os.getenv(ECS_CLUSTER_NAME):
@@ -53,10 +54,24 @@ class StatusResource(GenericResource):
53
54
 
54
55
  status = {
55
56
  'is_instance_manager': os.getenv(MANAGE_ENV_VAR) == '1',
56
- 'repo_path': get_repo_path(root_project=False),
57
- 'repo_path_relative': get_repo_path(root_project=False, absolute_path=False),
58
- 'repo_path_relative_root': get_repo_path(root_project=True, absolute_path=False),
59
- 'repo_path_root': get_repo_path(root_project=True),
57
+ 'repo_path': get_repo_path(
58
+ context_data=context_data,
59
+ root_project=False,
60
+ user=user),
61
+ 'repo_path_relative': get_repo_path(
62
+ context_data=context_data,
63
+ root_project=False,
64
+ absolute_path=False,
65
+ user=user),
66
+ 'repo_path_relative_root': get_repo_path(
67
+ context_data=context_data,
68
+ root_project=True,
69
+ absolute_path=False),
70
+ 'repo_path_root': get_repo_path(
71
+ context_data=context_data,
72
+ root_project=True),
73
+ 'server_process_id': os.getpid(),
74
+ 'scheduler_process_id': scheduler_manager.get_scheduler_pid(),
60
75
  'scheduler_status': scheduler_manager.get_status(),
61
76
  'instance_type': instance_type,
62
77
  'disable_pipeline_edit_access': is_disable_pipeline_edit_access(),
@@ -40,6 +40,7 @@ class SyncResource(GenericResource):
40
40
  @classmethod
41
41
  def collection(self, query, meta, user, **kwargs):
42
42
  sync_config = self.get_project_sync_config(user)
43
+
43
44
  return self.build_result_set(
44
45
  [sync_config],
45
46
  user,
@@ -66,9 +67,9 @@ class SyncResource(GenericResource):
66
67
  user_payload = self.update_user_settings(user_settings, user=user, repo_name=repo_name)
67
68
  UserGitConfig.load(config=user_payload)
68
69
 
69
- repo_path = kwargs.get('repo_path') or get_repo_path()
70
+ repo_path = kwargs.get('repo_path') or get_repo_path(user=user)
70
71
  user_preferences = user.preferences or {}
71
- user_git_settings = user.git_settings or {}
72
+ user_git_settings = user.get_git_settings(repo_path) or {}
72
73
  user_preferences[repo_path] = {
73
74
  **user_preferences.get(repo_path, {}),
74
75
  'git_settings': {
@@ -146,16 +147,14 @@ class SyncResource(GenericResource):
146
147
 
147
148
  if ssh_public_key:
148
149
  secret_name = get_ssh_public_key_secret_name(user=user)
149
- secret = Secret.query.filter(
150
- Secret.name == secret_name).one_or_none()
150
+ secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
151
151
  if secret:
152
152
  secret.delete()
153
153
  create_secret(secret_name, ssh_public_key, repo_name=repo_name)
154
154
  user_payload['ssh_public_key_secret_name'] = secret_name
155
155
  if ssh_private_key:
156
156
  secret_name = get_ssh_private_key_secret_name(user=user)
157
- secret = Secret.query.filter(
158
- Secret.name == secret_name).one_or_none()
157
+ secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
159
158
  if secret:
160
159
  secret.delete()
161
160
  create_secret(secret_name, ssh_private_key, repo_name=repo_name)
@@ -164,8 +163,7 @@ class SyncResource(GenericResource):
164
163
  access_token = user_payload.pop('access_token', None)
165
164
  if access_token:
166
165
  secret_name = get_access_token_secret_name(user=user)
167
- secret = Secret.query.filter(
168
- Secret.name == secret_name).one_or_none()
166
+ secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
169
167
  if secret:
170
168
  secret.delete()
171
169
  create_secret(secret_name, access_token, repo_name=repo_name)
@@ -2,7 +2,7 @@ from typing import Dict
2
2
 
3
3
  from mage_ai.api.errors import ApiError
4
4
  from mage_ai.api.resources.GenericResource import GenericResource
5
- from mage_ai.data_preparation.models.variable import VariableType
5
+ from mage_ai.data_preparation.models.variables.constants import VariableType
6
6
  from mage_ai.data_preparation.variable_manager import (
7
7
  VariableManager,
8
8
  delete_global_variable,
@@ -10,7 +10,7 @@ from mage_ai.data_preparation.variable_manager import (
10
10
  set_global_variable,
11
11
  )
12
12
  from mage_ai.orchestration.db import safe_db_query
13
- from mage_ai.settings.repo import get_variables_dir
13
+ from mage_ai.settings.repo import get_repo_path, get_variables_dir
14
14
 
15
15
 
16
16
  def get_variable_value(
@@ -41,6 +41,9 @@ class VariableResource(GenericResource):
41
41
  @classmethod
42
42
  @safe_db_query
43
43
  def collection(self, query, meta, user, **kwargs):
44
+ context_data = kwargs.get('context_data')
45
+ repo_path = get_repo_path(context_data=context_data, user=user)
46
+ variables_dir = get_variables_dir(repo_path=repo_path)
44
47
  pipeline_uuid = kwargs['parent_model'].uuid
45
48
 
46
49
  global_only = query.get('global_only', [False])
@@ -49,12 +52,13 @@ class VariableResource(GenericResource):
49
52
 
50
53
  # Get global variables from project's path
51
54
  global_variables = [
52
- dict(
53
- uuid=uuid,
54
- type=str(type(value)),
55
- value=value
56
- )
57
- for uuid, value in get_global_variables(pipeline_uuid).items()
55
+ dict(uuid=uuid, type=str(type(value)), value=value)
56
+ for uuid, value in get_global_variables(
57
+ pipeline_uuid,
58
+ context_data=context_data,
59
+ repo_path=repo_path,
60
+ variables_dir=variables_dir,
61
+ ).items()
58
62
  ]
59
63
  global_variables_arr = [
60
64
  dict(
@@ -65,7 +69,10 @@ class VariableResource(GenericResource):
65
69
  ]
66
70
  variables = global_variables_arr
67
71
  if not global_only:
68
- variable_manager = VariableManager(variables_dir=get_variables_dir())
72
+ variable_manager = VariableManager(
73
+ repo_path=repo_path,
74
+ variables_dir=variables_dir,
75
+ )
69
76
  variables_dict = variable_manager.get_variables_by_pipeline(pipeline_uuid)
70
77
  variables = [
71
78
  dict(
@@ -77,12 +84,14 @@ class VariableResource(GenericResource):
77
84
  pipeline_uuid,
78
85
  uuid,
79
86
  var,
80
- ) for var in arr
87
+ )
88
+ for var in arr
81
89
  # Not return printed outputs
82
90
  if var == 'df' or var.startswith('output')
83
91
  ],
84
92
  )
85
- for uuid, arr in variables_dict.items() if uuid != 'global'
93
+ for uuid, arr in variables_dict.items()
94
+ if uuid != 'global'
86
95
  ] + global_variables_arr
87
96
 
88
97
  return self.build_result_set(
@@ -100,7 +109,9 @@ class VariableResource(GenericResource):
100
109
 
101
110
  variable_uuid = payload.get('name')
102
111
  if not variable_uuid.isidentifier():
103
- error.update(message=f'Invalid variable name syntax for variable name {variable_uuid}.')
112
+ error.update(
113
+ message=f'Invalid variable name syntax for variable name {variable_uuid}.'
114
+ )
104
115
  raise ApiError(error)
105
116
 
106
117
  variable_value = payload.get('value')
@@ -116,13 +127,17 @@ class VariableResource(GenericResource):
116
127
 
117
128
  global_variables = get_global_variables(pipeline_uuid)
118
129
 
119
- return self(dict(
120
- block=dict(uuid='global'),
121
- name=variable_uuid,
122
- pipeline=dict(uuid=pipeline_uuid),
123
- value=variable_value,
124
- variables=list(global_variables.keys()),
125
- ), user, **kwargs)
130
+ return self(
131
+ dict(
132
+ block=dict(uuid='global'),
133
+ name=variable_uuid,
134
+ pipeline=dict(uuid=pipeline_uuid),
135
+ value=variable_value,
136
+ variables=list(global_variables.keys()),
137
+ ),
138
+ user,
139
+ **kwargs,
140
+ )
126
141
 
127
142
  @classmethod
128
143
  @safe_db_query
@@ -138,6 +153,9 @@ class VariableResource(GenericResource):
138
153
  variable_uuid = payload.get('name')
139
154
  if not variable_uuid.isidentifier():
140
155
  error.update(message=f'Invalid variable name syntax for variable name {self.name}.')
156
+ error.update(
157
+ message=f'Invalid variable name syntax for variable name {self.name}.'
158
+ )
141
159
  raise ApiError(error)
142
160
 
143
161
  variable_value = payload.get('value')
@@ -153,13 +171,15 @@ class VariableResource(GenericResource):
153
171
 
154
172
  global_variables = get_global_variables(pipeline_uuid)
155
173
 
156
- self.model.update(dict(
157
- block=dict(uuid='global'),
158
- name=variable_uuid,
159
- pipeline=dict(uuid=pipeline_uuid),
160
- value=variable_value,
161
- variables=list(global_variables.keys()),
162
- ))
174
+ self.model.update(
175
+ dict(
176
+ block=dict(uuid='global'),
177
+ name=variable_uuid,
178
+ pipeline=dict(uuid=pipeline_uuid),
179
+ value=variable_value,
180
+ variables=list(global_variables.keys()),
181
+ )
182
+ )
163
183
 
164
184
  return self
165
185
 
@@ -1,3 +1,4 @@
1
+ import inspect
1
2
  import urllib.parse
2
3
  from typing import Dict
3
4
 
@@ -41,11 +42,14 @@ class VersionControlProjectResource(VersionControlErrors, AsyncBaseResource):
41
42
  model = self.get_model(pk)
42
43
 
43
44
  sync_resource = SyncResource.member(None, user, repo_path=model.repo_path)
44
- model.sync_config = await SyncResource.presenter_class()(
45
+ result = SyncResource.presenter_class()(
45
46
  sync_resource,
46
47
  user,
47
48
  **kwargs,
48
49
  ).present()
50
+ if result and inspect.isawaitable(result):
51
+ result = await result
52
+ model.sync_config = result
49
53
 
50
54
  res = self(model, user, **kwargs)
51
55
  res.validate_output()
@@ -71,11 +75,14 @@ class VersionControlProjectResource(VersionControlErrors, AsyncBaseResource):
71
75
  self.current_user,
72
76
  repo_path=self.model.repo_path,
73
77
  )
74
- self.model.sync_config = await SyncResource.presenter_class()(
78
+ result = SyncResource.presenter_class()(
75
79
  sync_resource,
76
80
  self.current_user,
77
81
  **kwargs,
78
82
  ).present()
83
+ if result and inspect.isawaitable(result):
84
+ result = await result
85
+ self.model.sync_config = result
79
86
 
80
87
  self.model.update_attributes()
81
88
  self.validate_output()
@@ -39,7 +39,7 @@ class WidgetResource(GenericResource):
39
39
  resource = Widget.create(
40
40
  payload.get('name') or payload.get('uuid'),
41
41
  payload.get('type'),
42
- get_repo_path(),
42
+ get_repo_path(user=user),
43
43
  config=payload.get('config'),
44
44
  language=payload.get('language'),
45
45
  pipeline=pipeline,
@@ -123,14 +123,15 @@ class WorkspaceResource(GenericResource):
123
123
  error = ApiError.RESOURCE_ERROR.copy()
124
124
  try:
125
125
  action = payload.pop('action')
126
- args = ignore_keys(payload, ['name', 'cluster_type'])
127
126
  if action == 'stop':
128
- workspace.stop(**args)
127
+ workspace.stop(**payload)
129
128
  elif action == 'resume':
130
- workspace.resume(**args)
129
+ workspace.resume(**payload)
130
+ elif action == 'patch':
131
+ workspace.update(payload)
131
132
  elif action == 'add_to_ingress':
132
133
  if isinstance(workspace, KubernetesWorkspace):
133
- workspace.add_to_ingress(**args)
134
+ workspace.add_to_ingress(**payload)
134
135
  else:
135
136
  raise Exception('This workspace does not support ingress.')
136
137
  except Exception as ex:
@@ -162,7 +163,7 @@ class WorkspaceResource(GenericResource):
162
163
  raise ApiError(error)
163
164
 
164
165
  if project_type == ProjectType.MAIN and subproject:
165
- repo_path = get_repo_path()
166
+ repo_path = get_repo_path(user=self.current_user)
166
167
  projects_folder = os.path.join(repo_path, 'projects')
167
168
  projects = [
168
169
  f.name.split('.')[0]