mage-ai 0.9.69__py3-none-any.whl → 0.9.71__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mage-ai might be problematic. Click here for more details.

Files changed (624) hide show
  1. mage_ai/ai/utils/xgboost.py +222 -0
  2. mage_ai/api/errors.py +37 -25
  3. mage_ai/api/operations/base.py +13 -1
  4. mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
  5. mage_ai/api/policies/BackfillPolicy.py +1 -0
  6. mage_ai/api/policies/BlockOutputPolicy.py +40 -17
  7. mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
  8. mage_ai/api/policies/KernelPolicy.py +55 -32
  9. mage_ai/api/policies/KernelProcessPolicy.py +56 -0
  10. mage_ai/api/policies/OutputPolicy.py +73 -41
  11. mage_ai/api/policies/PipelinePolicy.py +206 -137
  12. mage_ai/api/policies/WorkspacePolicy.py +1 -0
  13. mage_ai/api/presenters/BackfillPresenter.py +1 -0
  14. mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
  15. mage_ai/api/presenters/BlockPresenter.py +1 -1
  16. mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
  17. mage_ai/api/presenters/KernelPresenter.py +5 -26
  18. mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
  19. mage_ai/api/presenters/PipelinePresenter.py +18 -5
  20. mage_ai/api/presenters/StatusPresenter.py +2 -0
  21. mage_ai/api/presenters/SyncPresenter.py +25 -0
  22. mage_ai/api/resources/AutocompleteItemResource.py +1 -1
  23. mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
  24. mage_ai/api/resources/BlockOutputResource.py +42 -9
  25. mage_ai/api/resources/BlockResource.py +4 -3
  26. mage_ai/api/resources/BlockRunResource.py +27 -22
  27. mage_ai/api/resources/ClusterResource.py +4 -1
  28. mage_ai/api/resources/CustomTemplateResource.py +34 -14
  29. mage_ai/api/resources/DataProviderResource.py +1 -1
  30. mage_ai/api/resources/ExecutionStateResource.py +3 -1
  31. mage_ai/api/resources/FileContentResource.py +8 -2
  32. mage_ai/api/resources/FileResource.py +10 -4
  33. mage_ai/api/resources/FileVersionResource.py +3 -1
  34. mage_ai/api/resources/GitBranchResource.py +101 -31
  35. mage_ai/api/resources/GitCustomBranchResource.py +29 -1
  36. mage_ai/api/resources/GlobalDataProductResource.py +44 -7
  37. mage_ai/api/resources/GlobalHookResource.py +4 -1
  38. mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
  39. mage_ai/api/resources/IntegrationSourceResource.py +8 -4
  40. mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
  41. mage_ai/api/resources/KernelProcessResource.py +44 -0
  42. mage_ai/api/resources/KernelResource.py +25 -3
  43. mage_ai/api/resources/OauthResource.py +1 -1
  44. mage_ai/api/resources/OutputResource.py +33 -11
  45. mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
  46. mage_ai/api/resources/PipelineInteractionResource.py +31 -15
  47. mage_ai/api/resources/PipelineResource.py +258 -125
  48. mage_ai/api/resources/PipelineRunResource.py +52 -7
  49. mage_ai/api/resources/PipelineScheduleResource.py +11 -2
  50. mage_ai/api/resources/PipelineTriggerResource.py +6 -1
  51. mage_ai/api/resources/ProjectResource.py +18 -7
  52. mage_ai/api/resources/PullRequestResource.py +6 -4
  53. mage_ai/api/resources/SecretResource.py +1 -1
  54. mage_ai/api/resources/SeedResource.py +8 -1
  55. mage_ai/api/resources/StatusResource.py +21 -6
  56. mage_ai/api/resources/SyncResource.py +6 -8
  57. mage_ai/api/resources/VariableResource.py +46 -26
  58. mage_ai/api/resources/VersionControlProjectResource.py +9 -2
  59. mage_ai/api/resources/WidgetResource.py +1 -1
  60. mage_ai/api/resources/WorkspaceResource.py +6 -5
  61. mage_ai/api/views.py +47 -40
  62. mage_ai/authentication/permissions/seed.py +16 -2
  63. mage_ai/authentication/providers/oidc.py +21 -1
  64. mage_ai/autocomplete/utils.py +13 -9
  65. mage_ai/cache/base.py +1 -1
  66. mage_ai/cache/block.py +18 -12
  67. mage_ai/cache/block_action_object/__init__.py +33 -5
  68. mage_ai/cache/file.py +22 -19
  69. mage_ai/cache/pipeline.py +18 -12
  70. mage_ai/cli/main.py +1 -0
  71. mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
  72. mage_ai/cluster_manager/config.py +2 -2
  73. mage_ai/cluster_manager/kubernetes/workload_manager.py +52 -1
  74. mage_ai/cluster_manager/manage.py +1 -1
  75. mage_ai/cluster_manager/workspace/base.py +7 -1
  76. mage_ai/cluster_manager/workspace/kubernetes.py +22 -1
  77. mage_ai/command_center/applications/factory.py +10 -7
  78. mage_ai/command_center/applications/utils.py +2 -2
  79. mage_ai/command_center/files/factory.py +17 -15
  80. mage_ai/command_center/presenters/text.py +1 -1
  81. mage_ai/command_center/utils.py +25 -13
  82. mage_ai/data/__init__.py +0 -0
  83. mage_ai/data/constants.py +45 -0
  84. mage_ai/data/models/__init__.py +0 -0
  85. mage_ai/data/models/base.py +119 -0
  86. mage_ai/data/models/constants.py +1 -0
  87. mage_ai/data/models/generator.py +115 -0
  88. mage_ai/data/models/manager.py +168 -0
  89. mage_ai/data/models/pyarrow/__init__.py +0 -0
  90. mage_ai/data/models/pyarrow/record_batch.py +55 -0
  91. mage_ai/data/models/pyarrow/shared.py +21 -0
  92. mage_ai/data/models/pyarrow/table.py +8 -0
  93. mage_ai/data/models/reader.py +103 -0
  94. mage_ai/data/models/utils.py +59 -0
  95. mage_ai/data/models/writer.py +91 -0
  96. mage_ai/data/tabular/__init__.py +0 -0
  97. mage_ai/data/tabular/constants.py +23 -0
  98. mage_ai/data/tabular/mocks.py +19 -0
  99. mage_ai/data/tabular/models.py +126 -0
  100. mage_ai/data/tabular/reader.py +602 -0
  101. mage_ai/data/tabular/utils.py +102 -0
  102. mage_ai/data/tabular/writer.py +266 -0
  103. mage_ai/data/variables/__init__.py +0 -0
  104. mage_ai/data/variables/wrapper.py +54 -0
  105. mage_ai/data_cleaner/analysis/charts.py +61 -39
  106. mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
  107. mage_ai/data_cleaner/estimators/encoders.py +5 -2
  108. mage_ai/data_integrations/utils/scheduler.py +16 -11
  109. mage_ai/data_preparation/decorators.py +1 -0
  110. mage_ai/data_preparation/executors/block_executor.py +237 -155
  111. mage_ai/data_preparation/executors/k8s_block_executor.py +30 -7
  112. mage_ai/data_preparation/executors/k8s_pipeline_executor.py +30 -7
  113. mage_ai/data_preparation/executors/streaming_pipeline_executor.py +2 -2
  114. mage_ai/data_preparation/git/__init__.py +77 -29
  115. mage_ai/data_preparation/git/api.py +69 -8
  116. mage_ai/data_preparation/git/utils.py +64 -34
  117. mage_ai/data_preparation/logging/logger_manager.py +4 -3
  118. mage_ai/data_preparation/models/block/__init__.py +1562 -879
  119. mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
  120. mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
  121. mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
  122. mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
  123. mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
  124. mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
  125. mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
  126. mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
  127. mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
  128. mage_ai/data_preparation/models/block/dynamic/utils.py +295 -167
  129. mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
  130. mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
  131. mage_ai/data_preparation/models/block/extension/utils.py +10 -1
  132. mage_ai/data_preparation/models/block/global_data_product/__init__.py +35 -3
  133. mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
  134. mage_ai/data_preparation/models/block/outputs.py +722 -0
  135. mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
  136. mage_ai/data_preparation/models/block/r/__init__.py +56 -38
  137. mage_ai/data_preparation/models/block/remote/__init__.py +0 -0
  138. mage_ai/data_preparation/models/block/remote/models.py +58 -0
  139. mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
  140. mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
  141. mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
  142. mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
  143. mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
  144. mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
  145. mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
  146. mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
  147. mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
  148. mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
  149. mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
  150. mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
  151. mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
  152. mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
  153. mage_ai/data_preparation/models/block/utils.py +164 -69
  154. mage_ai/data_preparation/models/constants.py +21 -14
  155. mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
  156. mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
  157. mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
  158. mage_ai/data_preparation/models/file.py +41 -28
  159. mage_ai/data_preparation/models/global_data_product/__init__.py +100 -58
  160. mage_ai/data_preparation/models/global_hooks/models.py +1 -0
  161. mage_ai/data_preparation/models/interfaces.py +29 -0
  162. mage_ai/data_preparation/models/pipeline.py +374 -185
  163. mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
  164. mage_ai/data_preparation/models/pipelines/seed.py +1 -1
  165. mage_ai/data_preparation/models/project/__init__.py +66 -18
  166. mage_ai/data_preparation/models/project/constants.py +2 -0
  167. mage_ai/data_preparation/models/triggers/__init__.py +124 -26
  168. mage_ai/data_preparation/models/utils.py +467 -17
  169. mage_ai/data_preparation/models/variable.py +1028 -137
  170. mage_ai/data_preparation/models/variables/__init__.py +0 -0
  171. mage_ai/data_preparation/models/variables/cache.py +149 -0
  172. mage_ai/data_preparation/models/variables/constants.py +72 -0
  173. mage_ai/data_preparation/models/variables/summarizer.py +336 -0
  174. mage_ai/data_preparation/models/variables/utils.py +77 -0
  175. mage_ai/data_preparation/models/widget/__init__.py +63 -41
  176. mage_ai/data_preparation/models/widget/charts.py +40 -27
  177. mage_ai/data_preparation/models/widget/constants.py +2 -0
  178. mage_ai/data_preparation/models/widget/utils.py +3 -3
  179. mage_ai/data_preparation/preferences.py +3 -3
  180. mage_ai/data_preparation/repo_manager.py +55 -21
  181. mage_ai/data_preparation/storage/base_storage.py +2 -2
  182. mage_ai/data_preparation/storage/gcs_storage.py +7 -4
  183. mage_ai/data_preparation/storage/local_storage.py +18 -9
  184. mage_ai/data_preparation/storage/s3_storage.py +5 -2
  185. mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
  186. mage_ai/data_preparation/variable_manager.py +281 -76
  187. mage_ai/io/base.py +3 -2
  188. mage_ai/io/bigquery.py +1 -0
  189. mage_ai/io/redshift.py +7 -5
  190. mage_ai/kernels/__init__.py +0 -0
  191. mage_ai/kernels/models.py +188 -0
  192. mage_ai/kernels/utils.py +169 -0
  193. mage_ai/orchestration/concurrency.py +6 -2
  194. mage_ai/orchestration/db/__init__.py +1 -0
  195. mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
  196. mage_ai/orchestration/db/migrations/versions/42a14d6143f1_update_token_column_type.py +54 -0
  197. mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
  198. mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
  199. mage_ai/orchestration/db/models/oauth.py +12 -18
  200. mage_ai/orchestration/db/models/projects.py +10 -0
  201. mage_ai/orchestration/db/models/schedules.py +225 -187
  202. mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
  203. mage_ai/orchestration/db/models/utils.py +46 -5
  204. mage_ai/orchestration/metrics/pipeline_run.py +8 -9
  205. mage_ai/orchestration/notification/sender.py +38 -15
  206. mage_ai/orchestration/pipeline_scheduler_original.py +64 -33
  207. mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
  208. mage_ai/orchestration/run_status_checker.py +11 -4
  209. mage_ai/orchestration/triggers/api.py +41 -2
  210. mage_ai/orchestration/triggers/global_data_product.py +9 -4
  211. mage_ai/orchestration/triggers/utils.py +10 -1
  212. mage_ai/orchestration/utils/resources.py +3 -0
  213. mage_ai/presenters/charts/data_sources/base.py +4 -2
  214. mage_ai/presenters/charts/data_sources/block.py +15 -9
  215. mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
  216. mage_ai/presenters/charts/data_sources/constants.py +1 -0
  217. mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
  218. mage_ai/presenters/interactions/models.py +11 -7
  219. mage_ai/presenters/pages/loaders/pipelines.py +5 -3
  220. mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
  221. mage_ai/presenters/utils.py +2 -0
  222. mage_ai/server/api/blocks.py +2 -1
  223. mage_ai/server/api/downloads.py +9 -2
  224. mage_ai/server/api/runs.py +151 -0
  225. mage_ai/server/api/triggers.py +3 -1
  226. mage_ai/server/constants.py +1 -1
  227. mage_ai/server/frontend_dist/404.html +8 -8
  228. mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
  229. mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
  230. mage_ai/server/frontend_dist/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
  231. mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
  232. mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
  233. mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
  234. mage_ai/server/frontend_dist/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
  235. mage_ai/server/frontend_dist/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
  236. mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
  237. mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
  238. mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
  239. mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
  240. mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
  241. mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
  242. mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
  243. mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
  244. mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
  245. mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
  246. mage_ai/server/frontend_dist/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
  247. mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
  248. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
  249. mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
  250. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
  251. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
  252. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
  253. mage_ai/server/frontend_dist/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
  254. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
  255. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
  256. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
  257. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
  258. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
  259. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
  260. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
  261. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
  262. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
  263. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
  264. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
  265. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
  266. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
  267. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
  268. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
  269. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
  270. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
  271. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
  272. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
  273. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
  274. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
  275. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
  276. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
  277. mage_ai/server/frontend_dist/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
  278. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
  279. mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
  280. mage_ai/server/frontend_dist/block-layout.html +3 -3
  281. mage_ai/server/frontend_dist/compute.html +6 -6
  282. mage_ai/server/frontend_dist/files.html +6 -6
  283. mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
  284. mage_ai/server/frontend_dist/global-data-products.html +6 -6
  285. mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
  286. mage_ai/server/frontend_dist/global-hooks.html +6 -6
  287. mage_ai/server/frontend_dist/index.html +3 -3
  288. mage_ai/server/frontend_dist/manage/files.html +6 -6
  289. mage_ai/server/frontend_dist/manage/settings.html +6 -6
  290. mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
  291. mage_ai/server/frontend_dist/manage/users/new.html +6 -6
  292. mage_ai/server/frontend_dist/manage/users.html +6 -6
  293. mage_ai/server/frontend_dist/manage.html +6 -6
  294. mage_ai/server/frontend_dist/oauth.html +5 -5
  295. mage_ai/server/frontend_dist/overview.html +6 -6
  296. mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
  297. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  298. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
  299. mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
  300. mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
  301. mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
  302. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  303. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  304. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
  305. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
  306. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
  307. mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
  308. mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
  309. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  310. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
  311. mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
  312. mage_ai/server/frontend_dist/pipelines.html +6 -6
  313. mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
  314. mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
  315. mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
  316. mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
  317. mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
  318. mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
  319. mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
  320. mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
  321. mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
  322. mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
  323. mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
  324. mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
  325. mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
  326. mage_ai/server/frontend_dist/settings.html +3 -3
  327. mage_ai/server/frontend_dist/sign-in.html +15 -15
  328. mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
  329. mage_ai/server/frontend_dist/templates.html +6 -6
  330. mage_ai/server/frontend_dist/terminal.html +6 -6
  331. mage_ai/server/frontend_dist/test.html +3 -3
  332. mage_ai/server/frontend_dist/triggers.html +6 -6
  333. mage_ai/server/frontend_dist/version-control.html +6 -6
  334. mage_ai/server/frontend_dist_base_path_template/404.html +8 -8
  335. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
  336. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
  337. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
  338. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
  339. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
  340. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
  341. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
  342. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
  343. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
  344. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
  345. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
  346. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
  347. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
  348. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
  349. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
  350. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
  351. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
  352. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
  353. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
  354. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
  355. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
  356. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
  357. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
  358. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
  359. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
  360. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
  361. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
  362. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
  363. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
  364. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
  365. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
  366. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
  367. mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
  368. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
  369. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
  370. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
  371. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
  372. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
  373. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
  374. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
  375. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
  376. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
  377. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
  378. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
  379. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
  380. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
  381. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
  382. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
  383. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
  384. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
  385. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
  386. mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
  387. mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
  388. mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
  389. mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
  390. mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
  391. mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
  392. mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
  393. mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
  394. mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
  395. mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
  396. mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
  397. mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
  398. mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
  399. mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
  400. mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
  401. mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
  402. mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
  403. mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
  404. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
  405. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
  406. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
  407. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
  408. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
  409. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
  410. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
  411. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
  412. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
  413. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
  414. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
  415. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
  416. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
  417. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
  418. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
  419. mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
  420. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
  421. mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
  422. mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
  423. mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
  424. mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
  425. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
  426. mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
  427. mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
  428. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
  429. mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
  430. mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
  431. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
  432. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
  433. mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
  434. mage_ai/server/frontend_dist_base_path_template/sign-in.html +15 -15
  435. mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
  436. mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
  437. mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
  438. mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
  439. mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
  440. mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
  441. mage_ai/server/kernel_output_parser.py +4 -1
  442. mage_ai/server/scheduler_manager.py +12 -1
  443. mage_ai/server/server.py +69 -42
  444. mage_ai/server/utils/custom_output.py +284 -0
  445. mage_ai/server/utils/execute_custom_code.py +245 -0
  446. mage_ai/server/utils/output_display.py +123 -289
  447. mage_ai/server/websocket_server.py +116 -69
  448. mage_ai/services/aws/ecs/ecs.py +1 -0
  449. mage_ai/services/k8s/config.py +27 -4
  450. mage_ai/services/k8s/job_manager.py +6 -1
  451. mage_ai/services/k8s/utils.py +97 -0
  452. mage_ai/services/ssh/aws/emr/utils.py +8 -8
  453. mage_ai/settings/keys/auth.py +1 -0
  454. mage_ai/settings/platform/__init__.py +159 -38
  455. mage_ai/settings/platform/constants.py +5 -0
  456. mage_ai/settings/platform/utils.py +53 -10
  457. mage_ai/settings/repo.py +26 -12
  458. mage_ai/settings/server.py +128 -37
  459. mage_ai/shared/array.py +24 -1
  460. mage_ai/shared/complex.py +45 -0
  461. mage_ai/shared/config.py +2 -1
  462. mage_ai/shared/custom_logger.py +11 -0
  463. mage_ai/shared/dates.py +10 -6
  464. mage_ai/shared/files.py +63 -8
  465. mage_ai/shared/hash.py +33 -9
  466. mage_ai/shared/io.py +9 -5
  467. mage_ai/shared/models.py +82 -24
  468. mage_ai/shared/outputs.py +87 -0
  469. mage_ai/shared/parsers.py +144 -13
  470. mage_ai/shared/path_fixer.py +11 -7
  471. mage_ai/shared/singletons/__init__.py +0 -0
  472. mage_ai/shared/singletons/base.py +47 -0
  473. mage_ai/shared/singletons/memory.py +38 -0
  474. mage_ai/shared/strings.py +34 -1
  475. mage_ai/shared/yaml.py +24 -0
  476. mage_ai/streaming/sinks/oracledb.py +57 -0
  477. mage_ai/streaming/sinks/sink_factory.py +4 -0
  478. mage_ai/system/__init__.py +0 -0
  479. mage_ai/system/constants.py +14 -0
  480. mage_ai/system/memory/__init__.py +0 -0
  481. mage_ai/system/memory/constants.py +1 -0
  482. mage_ai/system/memory/manager.py +174 -0
  483. mage_ai/system/memory/presenters.py +158 -0
  484. mage_ai/system/memory/process.py +216 -0
  485. mage_ai/system/memory/samples.py +13 -0
  486. mage_ai/system/memory/utils.py +656 -0
  487. mage_ai/system/memory/wrappers.py +177 -0
  488. mage_ai/system/models.py +58 -0
  489. mage_ai/system/storage/__init__.py +0 -0
  490. mage_ai/system/storage/utils.py +29 -0
  491. mage_ai/tests/api/endpoints/mixins.py +2 -2
  492. mage_ai/tests/api/endpoints/test_blocks.py +2 -1
  493. mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
  494. mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
  495. mage_ai/tests/api/endpoints/test_projects.py +2 -1
  496. mage_ai/tests/api/operations/base/mixins.py +1 -1
  497. mage_ai/tests/api/operations/base/test_base.py +27 -27
  498. mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
  499. mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
  500. mage_ai/tests/api/operations/test_syncs.py +6 -4
  501. mage_ai/tests/api/resources/test_pipeline_resource.py +11 -4
  502. mage_ai/tests/authentication/oauth/test_utils.py +1 -1
  503. mage_ai/tests/authentication/providers/test_oidc.py +59 -0
  504. mage_ai/tests/base_test.py +2 -2
  505. mage_ai/tests/data/__init__.py +0 -0
  506. mage_ai/tests/data/models/__init__.py +0 -0
  507. mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
  508. mage_ai/tests/data_preparation/git/test_git.py +4 -1
  509. mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
  510. mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
  511. mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
  512. mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
  513. mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
  514. mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
  515. mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
  516. mage_ai/tests/data_preparation/models/block/test_global_data_product.py +5 -2
  517. mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
  518. mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
  519. mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
  520. mage_ai/tests/data_preparation/models/test_block.py +115 -120
  521. mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
  522. mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
  523. mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
  524. mage_ai/tests/data_preparation/models/test_project.py +4 -1
  525. mage_ai/tests/data_preparation/models/test_utils.py +80 -0
  526. mage_ai/tests/data_preparation/models/test_variable.py +242 -69
  527. mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
  528. mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
  529. mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
  530. mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
  531. mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
  532. mage_ai/tests/factory.py +64 -43
  533. mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
  534. mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
  535. mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
  536. mage_ai/tests/orchestration/triggers/test_global_data_product.py +141 -138
  537. mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
  538. mage_ai/tests/server/test_server.py +19 -0
  539. mage_ai/tests/services/k8s/test_job_manager.py +27 -6
  540. mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
  541. mage_ai/tests/test_shared.py +61 -0
  542. mage_ai/usage_statistics/logger.py +7 -2
  543. mage_ai/utils/code.py +33 -19
  544. mage_ai/version_control/branch/utils.py +2 -1
  545. mage_ai/version_control/models.py +3 -2
  546. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/METADATA +6 -3
  547. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/RECORD +555 -454
  548. mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
  549. mage_ai/server/frontend_dist/_next/static/_krrrgup_C-dPOpX36S8I/_buildManifest.js +0 -1
  550. mage_ai/server/frontend_dist/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
  551. mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
  552. mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
  553. mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
  554. mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
  555. mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
  556. mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
  557. mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
  558. mage_ai/server/frontend_dist/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
  559. mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
  560. mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
  561. mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
  562. mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
  563. mage_ai/server/frontend_dist/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
  564. mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
  565. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
  566. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
  567. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
  568. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
  569. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
  570. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
  571. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
  572. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
  573. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
  574. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
  575. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
  576. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
  577. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
  578. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
  579. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
  580. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
  581. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
  582. mage_ai/server/frontend_dist_base_path_template/_next/static/KLL5mirre9d7_ZeEpaw3s/_buildManifest.js +0 -1
  583. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
  584. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
  585. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
  586. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
  587. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
  588. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
  589. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
  590. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
  591. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
  592. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
  593. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
  594. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
  595. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
  596. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
  597. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
  598. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
  599. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
  600. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
  601. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
  602. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
  603. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
  604. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
  605. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
  606. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
  607. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
  608. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
  609. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
  610. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
  611. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
  612. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
  613. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
  614. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
  615. mage_ai/shared/memory.py +0 -90
  616. mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
  617. /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
  618. /mage_ai/server/frontend_dist/_next/static/{_krrrgup_C-dPOpX36S8I → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
  619. /mage_ai/server/frontend_dist_base_path_template/_next/static/{KLL5mirre9d7_ZeEpaw3s → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
  620. /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
  621. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
  622. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
  623. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
  624. {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
@@ -62,29 +62,38 @@ from mage_ai.usage_statistics.logger import UsageStatisticLogger
62
62
 
63
63
 
64
64
  @safe_db_query
65
- def query_pipeline_schedules(pipeline_uuids: List[str]):
65
+ def query_pipeline_schedules(
66
+ pipeline_uuids: List[str],
67
+ context_data: Dict = None,
68
+ repo_path: str = None,
69
+ ):
66
70
  a = aliased(PipelineSchedule, name='a')
67
71
  result = (
68
- PipelineSchedule.
69
- select(*[
70
- a.created_at,
71
- a.id,
72
- a.name,
73
- a.pipeline_uuid,
74
- a.schedule_interval,
75
- a.schedule_type,
76
- a.start_time,
77
- a.status,
78
- a.updated_at,
79
- ]).
80
- filter(
72
+ PipelineSchedule.select(
73
+ *[
74
+ a.created_at,
75
+ a.id,
76
+ a.name,
77
+ a.pipeline_uuid,
78
+ a.schedule_interval,
79
+ a.schedule_type,
80
+ a.start_time,
81
+ a.status,
82
+ a.updated_at,
83
+ ]
84
+ ).filter(
81
85
  a.pipeline_uuid.in_(pipeline_uuids),
82
86
  or_(
83
- a.repo_path.in_(Project().repo_path_for_database_query(
84
- 'pipeline_schedules',
85
- )),
87
+ a.repo_path.in_(
88
+ Project(
89
+ context_data=context_data,
90
+ repo_path=repo_path,
91
+ ).repo_path_for_database_query(
92
+ 'pipeline_schedules',
93
+ )
94
+ ),
86
95
  a.repo_path.is_(None),
87
- )
96
+ ),
88
97
  )
89
98
  ).all()
90
99
  return group_by(lambda x: x.pipeline_uuid, result)
@@ -94,6 +103,7 @@ class PipelineResource(BaseResource):
94
103
  @classmethod
95
104
  @safe_db_query
96
105
  async def collection(self, query, meta, user, **kwargs):
106
+ context_data = kwargs.get('context_data')
97
107
  limit = (meta or {}).get(META_KEY_LIMIT, None)
98
108
  if limit is not None:
99
109
  limit = int(limit)
@@ -119,6 +129,16 @@ class PipelineResource(BaseResource):
119
129
  if include_schedules:
120
130
  include_schedules = include_schedules[0]
121
131
 
132
+ repo_path = query.get('repo_path', [None])
133
+ if repo_path:
134
+ repo_path = repo_path[0]
135
+ if not repo_path:
136
+ repo_path = get_repo_path(
137
+ context_data=context_data,
138
+ root_project=False,
139
+ user=user,
140
+ )
141
+
122
142
  search_query = query.get('search', [None])
123
143
  if search_query:
124
144
  search_query = search_query[0]
@@ -148,9 +168,12 @@ class PipelineResource(BaseResource):
148
168
 
149
169
  history_by_pipeline_uuid = {}
150
170
  if from_history_days is not None and is_number(from_history_days):
151
- timestamp_start = (datetime.utcnow() - timedelta(
152
- hours=24 * int(from_history_days),
153
- )).timestamp()
171
+ timestamp_start = (
172
+ datetime.utcnow()
173
+ - timedelta(
174
+ hours=24 * int(from_history_days),
175
+ )
176
+ ).timestamp()
154
177
  history = load_pipelines_detail(timestamp_start=timestamp_start)
155
178
  history = sorted(
156
179
  history,
@@ -174,21 +197,29 @@ class PipelineResource(BaseResource):
174
197
 
175
198
  if NO_TAGS_QUERY in tags:
176
199
  pipeline_uuids_with_tags = set(cache.get_all_pipeline_uuids_with_tags())
177
- all_pipeline_uuids = set(Pipeline.get_all_pipelines(get_repo_path()))
178
- pipeline_uuids_without_tags = list(all_pipeline_uuids - pipeline_uuids_with_tags)
200
+ all_pipeline_uuids = set(
201
+ Pipeline.get_all_pipelines(repo_path=repo_path)
202
+ )
203
+ pipeline_uuids_without_tags = list(
204
+ all_pipeline_uuids - pipeline_uuids_with_tags
205
+ )
179
206
  pipeline_uuids = pipeline_uuids + pipeline_uuids_without_tags
180
207
  else:
181
- pipeline_uuids = Pipeline.get_all_pipelines(get_repo_path())
208
+ pipeline_uuids = Pipeline.get_all_pipelines(repo_path=repo_path)
182
209
 
183
210
  if search_query:
184
- pipeline_uuids = list(filter(
185
- lambda x: search_query.lower() in x.lower() or
186
- search_query.lower() in x.lower().split(' '),
187
- pipeline_uuids,
188
- ))
211
+ pipeline_uuids = list(
212
+ filter(
213
+ lambda x: search_query.lower() in x.lower()
214
+ or search_query.lower() in x.lower().split(' '),
215
+ pipeline_uuids,
216
+ )
217
+ )
189
218
 
190
219
  total_count = len(pipeline_uuids)
191
- await UsageStatisticLogger().pipelines_impression(lambda: total_count)
220
+ await UsageStatisticLogger(
221
+ context_data=context_data, repo_path=repo_path,
222
+ ).pipelines_impression(lambda: total_count)
192
223
 
193
224
  if not sorts:
194
225
  pipeline_uuids = sorted(pipeline_uuids, reverse=reverse_sort)
@@ -199,17 +230,20 @@ class PipelineResource(BaseResource):
199
230
  if offset:
200
231
  pipeline_uuids = pipeline_uuids[offset:]
201
232
  if limit:
202
- pipeline_uuids = pipeline_uuids[:(limit + 1)]
233
+ pipeline_uuids = pipeline_uuids[: (limit + 1)]
203
234
  offset_limit_applied = True
204
235
 
205
236
  cache = await PipelineCache.initialize_cache()
206
237
 
207
238
  async def get_pipeline(uuid: str) -> Pipeline:
208
239
  try:
209
- return await Pipeline.get_async(uuid)
240
+ return await Pipeline.get_async(uuid, repo_path=repo_path)
210
241
  except Exception as err:
211
242
  err_message = f'Error loading pipeline {uuid}: {err}.'
212
- if err.__class__.__name__ == 'OSError' and 'Too many open files' in err.strerror:
243
+ if (
244
+ err.__class__.__name__ == 'OSError'
245
+ and 'Too many open files' in err.strerror
246
+ ):
213
247
  raise Exception(err_message)
214
248
  else:
215
249
  print(err_message)
@@ -217,10 +251,13 @@ class PipelineResource(BaseResource):
217
251
 
218
252
  def get_pipeline_with_config(uuid, config: Dict) -> Pipeline:
219
253
  try:
220
- return Pipeline(uuid, config=config)
254
+ return Pipeline(uuid, config=config, repo_path=repo_path)
221
255
  except Exception as err:
222
256
  err_message = f'Error loading pipeline sync {uuid}: {err}.'
223
- if err.__class__.__name__ == 'OSError' and 'Too many open files' in err.strerror:
257
+ if (
258
+ err.__class__.__name__ == 'OSError'
259
+ and 'Too many open files' in err.strerror
260
+ ):
224
261
  raise Exception(err_message)
225
262
  else:
226
263
  print(err_message)
@@ -233,13 +270,16 @@ class PipelineResource(BaseResource):
233
270
  for pipeline_dict in cache.get_models(types=pipeline_types):
234
271
  pipeline_uuid_from_cache = pipeline_dict['pipeline']['uuid']
235
272
  if pipeline_uuid_from_cache in pipeline_uuids:
236
- pipelines.append(Pipeline(
237
- pipeline_uuid_from_cache,
238
- config=pipeline_dict['pipeline'],
239
- ))
273
+ pipelines.append(
274
+ Pipeline(
275
+ pipeline_uuid_from_cache,
276
+ config=pipeline_dict['pipeline'],
277
+ repo_path=repo_path,
278
+ )
279
+ )
240
280
  else:
241
281
  for uuid in pipeline_uuids:
242
- pipeline_dict = cache.get_model(dict(uuid=uuid))
282
+ pipeline_dict = cache.get_model(dict(uuid=uuid, repo_path=repo_path))
243
283
  if pipeline_dict and pipeline_dict.get('pipeline'):
244
284
  pipeline = get_pipeline_with_config(uuid, pipeline_dict['pipeline'])
245
285
  if pipeline:
@@ -247,10 +287,13 @@ class PipelineResource(BaseResource):
247
287
  else:
248
288
  # Add pipeline with type "invalid" so pipeline with invalid config
249
289
  # can still be displayed in UI and visible to user
250
- pipelines.append(Pipeline(
251
- uuid,
252
- config=dict(type='invalid'),
253
- ))
290
+ pipelines.append(
291
+ Pipeline(
292
+ uuid,
293
+ config=dict(type='invalid'),
294
+ repo_path=repo_path,
295
+ )
296
+ )
254
297
  else:
255
298
  pipeline_uuids_miss.append(uuid)
256
299
 
@@ -263,7 +306,11 @@ class PipelineResource(BaseResource):
263
306
 
264
307
  mapping = {}
265
308
  if include_schedules:
266
- mapping = query_pipeline_schedules(pipeline_uuids)
309
+ mapping = query_pipeline_schedules(
310
+ pipeline_uuids,
311
+ context_data=context_data,
312
+ repo_path=repo_path,
313
+ )
267
314
 
268
315
  filtered_pipelines = []
269
316
  for pipeline in pipelines:
@@ -273,15 +320,23 @@ class PipelineResource(BaseResource):
273
320
  pipeline.schedules = schedules
274
321
 
275
322
  if pipeline_statuses and (
276
- (PipelineStatus.ACTIVE in pipeline_statuses and
277
- any(s.status == ScheduleStatus.ACTIVE
278
- for s in pipeline.schedules)) or
279
- (PipelineStatus.INACTIVE in pipeline_statuses and
280
- len(pipeline.schedules) > 0 and
281
- all(s.status == ScheduleStatus.INACTIVE
282
- for s in pipeline.schedules)) or
283
- (PipelineStatus.NO_SCHEDULES in pipeline_statuses and
284
- len(pipeline.schedules) == 0)
323
+ (
324
+ PipelineStatus.ACTIVE in pipeline_statuses
325
+ and any(
326
+ s.status == ScheduleStatus.ACTIVE for s in pipeline.schedules
327
+ )
328
+ )
329
+ or (
330
+ PipelineStatus.INACTIVE in pipeline_statuses
331
+ and len(pipeline.schedules) > 0
332
+ and all(
333
+ s.status == ScheduleStatus.INACTIVE for s in pipeline.schedules
334
+ )
335
+ )
336
+ or (
337
+ PipelineStatus.NO_SCHEDULES in pipeline_statuses
338
+ and len(pipeline.schedules) == 0
339
+ )
285
340
  ):
286
341
  filtered_pipelines.append(pipeline)
287
342
 
@@ -294,6 +349,7 @@ class PipelineResource(BaseResource):
294
349
  pipeline.history = history_by_pipeline_uuid.get(pipeline.uuid)
295
350
 
296
351
  if sorts:
352
+
297
353
  def _sort_key(p, sorts=sorts, reverse_sort=reverse_sort):
298
354
  bools = []
299
355
  vals = []
@@ -301,24 +357,35 @@ class PipelineResource(BaseResource):
301
357
  if 'blocks' == k.lower():
302
358
  val = len(p.blocks_by_uuid)
303
359
  vals.append(val)
304
- bools.append(val is None if not reverse_sort else val is not None)
360
+ bools.append(
361
+ val is None if not reverse_sort else val is not None
362
+ )
305
363
  elif 'triggers' == k.lower():
306
364
  val = len(p.schedules)
307
365
  vals.append(val)
308
- bools.append(val is None if not reverse_sort else val is not None)
366
+ bools.append(
367
+ val is None if not reverse_sort else val is not None
368
+ )
309
369
  elif 'status' == k.lower():
310
370
  if len(p.schedules) == 0:
311
371
  val = 'no_schedules'
312
- elif find(lambda s: s.status == 'active', p.schedules) is not None:
372
+ elif (
373
+ find(lambda s: s.status == 'active', p.schedules)
374
+ is not None
375
+ ):
313
376
  val = 'active'
314
377
  else:
315
378
  val = 'inactive'
316
379
  vals.append(val)
317
- bools.append(val is None if not reverse_sort else val is not None)
380
+ bools.append(
381
+ val is None if not reverse_sort else val is not None
382
+ )
318
383
  elif hasattr(p, k):
319
384
  val = getattr(p, k)
320
385
  vals.append(val)
321
- bools.append(val is None if not reverse_sort else val is not None)
386
+ bools.append(
387
+ val is None if not reverse_sort else val is not None
388
+ )
322
389
  else:
323
390
  bools.append(False)
324
391
 
@@ -353,12 +420,15 @@ class PipelineResource(BaseResource):
353
420
  'results': len(arr),
354
421
  'next': has_next,
355
422
  }
356
-
423
+ for p in arr:
424
+ p.context_data = context_data
357
425
  return result_set
358
426
 
359
427
  @classmethod
360
428
  @safe_db_query
361
429
  async def create(self, payload, user, **kwargs):
430
+ context_data = kwargs.get('context_data')
431
+
362
432
  clone_pipeline_uuid = payload.get('clone_pipeline_uuid')
363
433
  template_uuid = payload.get('custom_template_uuid')
364
434
  name = payload.get('name')
@@ -368,18 +438,22 @@ class PipelineResource(BaseResource):
368
438
  llm_payload = payload.get('llm')
369
439
  pipeline = None
370
440
 
441
+ repo_path = get_repo_path(context_data=context_data, user=user)
371
442
  if template_uuid:
372
- custom_template = CustomPipelineTemplate.load(template_uuid=template_uuid)
443
+ custom_template = CustomPipelineTemplate.load(
444
+ repo_path,
445
+ template_uuid=template_uuid,
446
+ )
373
447
  pipeline = custom_template.create_pipeline(name)
374
448
  elif clone_pipeline_uuid is not None:
375
- source = Pipeline.get(clone_pipeline_uuid)
449
+ source = Pipeline.get(clone_pipeline_uuid, repo_path=repo_path)
376
450
  pipeline = await Pipeline.duplicate(source, name)
377
451
  else:
378
452
  pipeline = Pipeline.create(
379
453
  name,
380
454
  description=description,
381
455
  pipeline_type=pipeline_type,
382
- repo_path=get_repo_path(),
456
+ repo_path=repo_path,
383
457
  tags=tags,
384
458
  )
385
459
 
@@ -403,25 +477,37 @@ class PipelineResource(BaseResource):
403
477
  block_payload['configuration'] = configuration
404
478
 
405
479
  block_uuid = f'{pipeline.uuid}_block_{block_number}'
406
- block_resource = await BlockResource.create(merge_dict(
407
- dict(
408
- name=block_uuid,
409
- type=block_payload.get('block_type'),
480
+ block_resource = await BlockResource.create(
481
+ merge_dict(
482
+ dict(
483
+ name=block_uuid,
484
+ type=block_payload.get('block_type'),
485
+ ),
486
+ ignore_keys(
487
+ block_payload,
488
+ [
489
+ 'block_type',
490
+ 'upstream_blocks',
491
+ ],
492
+ ),
493
+ ),
494
+ user,
495
+ **merge_dict(
496
+ kwargs,
497
+ dict(
498
+ parent_model=pipeline,
499
+ ),
410
500
  ),
411
- ignore_keys(block_payload, [
412
- 'block_type',
413
- 'upstream_blocks',
414
- ]),
415
- ), user, **merge_dict(kwargs, dict(
416
- parent_model=pipeline,
417
- )))
501
+ )
418
502
 
419
503
  upstream_block_uuids = block_payload.get('upstream_blocks')
420
504
 
421
505
  pipeline.add_block(
422
506
  block_resource.model,
423
507
  None,
424
- priority=len(upstream_block_uuids) if upstream_block_uuids else 0,
508
+ priority=len(upstream_block_uuids)
509
+ if upstream_block_uuids
510
+ else 0,
425
511
  widget=False,
426
512
  )
427
513
 
@@ -435,11 +521,17 @@ class PipelineResource(BaseResource):
435
521
 
436
522
  if upstream_block_uuids and len(upstream_block_uuids) >= 1:
437
523
  block = config['block']
438
- arr = [f'{pipeline.uuid}_block_{bn}' for bn in upstream_block_uuids]
524
+ arr = [
525
+ f'{pipeline.uuid}_block_{bn}'
526
+ for bn in upstream_block_uuids
527
+ ]
439
528
  block.update(dict(upstream_blocks=arr))
440
529
 
441
530
  if pipeline:
442
- await UsageStatisticLogger().pipeline_create(
531
+ await UsageStatisticLogger(
532
+ context_data=context_data,
533
+ repo_path=repo_path,
534
+ ).pipeline_create(
443
535
  pipeline,
444
536
  clone_pipeline_uuid=clone_pipeline_uuid,
445
537
  llm_payload=llm_payload,
@@ -466,19 +558,26 @@ class PipelineResource(BaseResource):
466
558
 
467
559
  self.on_create_callback = _on_create_callback
468
560
 
561
+ pipeline.context_data = context_data
469
562
  return self(pipeline, user, **kwargs)
470
563
 
471
564
  @classmethod
472
565
  @safe_db_query
473
- async def __fetch_model(self, pipeline_uuid: str, **kqwargs):
566
+ async def __fetch_model(self, pipeline_uuid: str, repo_path: str, **kwargs):
474
567
  all_projects = project_platform_activated()
475
568
 
476
569
  if all_projects:
477
570
  return await get_pipeline_from_platform_async(
478
571
  pipeline_uuid,
572
+ repo_path=repo_path,
573
+ context_data=kwargs.get('context_data'),
479
574
  )
480
575
 
481
- return await Pipeline.get_async(pipeline_uuid, all_projects=all_projects)
576
+ return await Pipeline.get_async(
577
+ pipeline_uuid,
578
+ all_projects=all_projects,
579
+ repo_path=repo_path,
580
+ )
482
581
 
483
582
  @classmethod
484
583
  @safe_db_query
@@ -488,23 +587,33 @@ class PipelineResource(BaseResource):
488
587
  **kwargs,
489
588
  ):
490
589
  pipeline_uuid = urllib.parse.unquote(pk)
491
- return await self.__fetch_model(pipeline_uuid, **kwargs)
590
+ user = kwargs.get('user')
591
+ repo_path = get_repo_path(context_data=kwargs.get('context_data'), user=user)
592
+ return await self.__fetch_model(pipeline_uuid, repo_path, **kwargs)
492
593
 
493
594
  @classmethod
494
595
  @safe_db_query
495
596
  async def member(self, pk, user, **kwargs):
496
- pipeline = await self.__fetch_model(pk, **kwargs)
597
+ context_data = kwargs.get('context_data')
598
+
599
+ repo_path = get_repo_path(context_data=context_data, user=user)
600
+ pipeline = await self.__fetch_model(pk, repo_path, **kwargs)
497
601
 
498
602
  api_operation_action = kwargs.get('api_operation_action', None)
499
603
  if api_operation_action != DELETE:
500
604
  kernel_name = PIPELINE_TO_KERNEL_NAME[pipeline.type]
501
605
  switch_active_kernel(
502
606
  kernel_name,
503
- emr_config=pipeline.executor_config if kernel_name == KernelName.PYSPARK else None,
607
+ emr_config=pipeline.executor_config
608
+ if kernel_name == KernelName.PYSPARK
609
+ else None,
504
610
  )
505
611
 
506
612
  if api_operation_action == DETAIL:
507
- if Project(pipeline.repo_config).is_feature_enabled(
613
+ if Project(
614
+ context_data=context_data,
615
+ repo_config=pipeline.repo_config,
616
+ ).is_feature_enabled(
508
617
  FeatureUUID.OPERATION_HISTORY,
509
618
  ):
510
619
  record_detail_pipeline(
@@ -531,6 +640,7 @@ class PipelineResource(BaseResource):
531
640
  if mapping.get(pipeline.uuid):
532
641
  pipeline.schedules = mapping[pipeline.uuid] or []
533
642
 
643
+ pipeline.context_data = context_data
534
644
  return self(pipeline, user, **kwargs)
535
645
 
536
646
  @safe_db_query
@@ -557,26 +667,30 @@ class PipelineResource(BaseResource):
557
667
 
558
668
  block_cache = await BlockCache.initialize_cache()
559
669
  for block in pipeline.blocks_by_uuid.values():
560
- block_cache.remove_pipeline(block, pipeline.uuid, pipeline.repo_path)
670
+ block_cache.remove_pipeline(
671
+ block, pipeline.uuid, pipeline.repo_path
672
+ )
561
673
 
562
674
  self.on_delete_callback = _on_delete_callback
563
675
  return self.model.delete()
564
676
 
565
677
  @safe_db_query
566
678
  async def update(self, payload, **kwargs):
679
+ context_data = kwargs.get('context_data')
567
680
  if 'add_upstream_for_block_uuid' in payload:
568
681
  block_uuid = payload['add_upstream_for_block_uuid']
569
682
  block = self.model.get_block(block_uuid, widget=False)
570
683
  if BlockType.DBT == block.type and block.language == BlockLanguage.SQL:
571
684
  upstream_dbt_blocks_by_uuid = {
572
- block.uuid: block
573
- for block in block.upstream_dbt_blocks()
685
+ block.uuid: block for block in block.upstream_dbt_blocks()
574
686
  }
575
687
  self.model.blocks_by_uuid.update(upstream_dbt_blocks_by_uuid)
576
688
  self.model.validate('A cycle was formed while adding a block')
577
689
  self.model.save()
578
690
  return self
579
691
 
692
+ repo_path = get_repo_path(context_data=context_data, user=self.current_user)
693
+
580
694
  query = kwargs.get('query', {})
581
695
  update_content = query.get('update_content', [False])
582
696
  if update_content:
@@ -590,23 +704,34 @@ class PipelineResource(BaseResource):
590
704
  if 'pipeline_uuid' not in llm_payload:
591
705
  llm_payload['pipeline_uuid'] = self.model.uuid
592
706
 
593
- llm_resource = await LlmResource.create(llm_payload, self.current_user, **kwargs)
707
+ llm_resource = await LlmResource.create(
708
+ llm_payload, self.current_user, **kwargs
709
+ )
594
710
  llm_response = llm_resource.model.get('response')
595
711
 
596
712
  pipeline_doc = None
597
713
  block_docs = []
598
714
  blocks = self.model.blocks_by_uuid.values()
599
715
 
600
- async def _add_markdown_block(block_doc: str, block_uuid: str, priority: int):
601
- return await BlockResource.create(dict(
602
- content=block_doc.strip() if block_doc else block_doc,
603
- language=BlockLanguage.MARKDOWN,
604
- name=f'Documentation for {block_uuid}',
605
- priority=priority,
606
- type=BlockType.MARKDOWN,
607
- ), self.current_user, **merge_dict(kwargs, dict(
608
- parent_model=self.model,
609
- )))
716
+ async def _add_markdown_block(
717
+ block_doc: str, block_uuid: str, priority: int
718
+ ):
719
+ return await BlockResource.create(
720
+ dict(
721
+ content=block_doc.strip() if block_doc else block_doc,
722
+ language=BlockLanguage.MARKDOWN,
723
+ name=f'Documentation for {block_uuid}',
724
+ priority=priority,
725
+ type=BlockType.MARKDOWN,
726
+ ),
727
+ self.current_user,
728
+ **merge_dict(
729
+ kwargs,
730
+ dict(
731
+ parent_model=self.model,
732
+ ),
733
+ ),
734
+ )
610
735
 
611
736
  if LLMUseCase.GENERATE_DOC_FOR_BLOCK == llm_use_case:
612
737
  block_doc = llm_response.get('block_doc')
@@ -650,11 +775,15 @@ class PipelineResource(BaseResource):
650
775
  switch_active_kernel(
651
776
  kernel_name,
652
777
  emr_config=self.model.executor_config
653
- if kernel_name == KernelName.PYSPARK else None,
778
+ if kernel_name == KernelName.PYSPARK
779
+ else None,
654
780
  )
655
781
  except Exception as e:
656
782
  pipeline_type_updated = payload.get('type')
657
- if pipeline_type_updated is not None and pipeline_type_updated != pipeline_type:
783
+ if (
784
+ pipeline_type_updated is not None
785
+ and pipeline_type_updated != pipeline_type
786
+ ):
658
787
  await self.model.update(dict(type=pipeline_type))
659
788
 
660
789
  raise e
@@ -664,9 +793,9 @@ class PipelineResource(BaseResource):
664
793
  trigger_configs_by_name = get_trigger_configs_by_name(pipeline_uuid)
665
794
  triggers_in_code_to_update = []
666
795
  schedules = (
667
- PipelineSchedule.
668
- query.
669
- filter(PipelineSchedule.pipeline_uuid == pipeline_uuid)
796
+ PipelineSchedule.query.filter(
797
+ PipelineSchedule.pipeline_uuid == pipeline_uuid
798
+ )
670
799
  ).all()
671
800
  for schedule in schedules:
672
801
  trigger_config = trigger_configs_by_name.get(schedule.name)
@@ -689,29 +818,30 @@ class PipelineResource(BaseResource):
689
818
  ):
690
819
  if pipeline_runs is not None:
691
820
  pipeline_run_ids = [run.get('id') for run in pipeline_runs]
692
- pipeline_runs_to_cancel = (
693
- PipelineRun.
694
- query.
695
- filter(PipelineRun.id.in_(pipeline_run_ids))
821
+ pipeline_runs_to_cancel = PipelineRun.query.filter(
822
+ PipelineRun.id.in_(pipeline_run_ids)
696
823
  )
697
824
  elif pipeline_schedule_id is not None:
698
- pipeline_runs_to_cancel = PipelineRun.in_progress_runs([pipeline_schedule_id])
825
+ pipeline_runs_to_cancel = PipelineRun.in_progress_runs(
826
+ [pipeline_schedule_id]
827
+ )
699
828
  else:
700
- pipeline_runs_to_cancel = (
701
- PipelineRun.
702
- query.
703
- filter(PipelineRun.pipeline_uuid == pipeline_uuid).
704
- filter(PipelineRun.status.in_([
705
- PipelineRun.PipelineRunStatus.INITIAL,
706
- PipelineRun.PipelineRunStatus.RUNNING,
707
- ]))
829
+ pipeline_runs_to_cancel = PipelineRun.query.filter(
830
+ PipelineRun.pipeline_uuid == pipeline_uuid
831
+ ).filter(
832
+ PipelineRun.status.in_(
833
+ [
834
+ PipelineRun.PipelineRunStatus.INITIAL,
835
+ PipelineRun.PipelineRunStatus.RUNNING,
836
+ ]
837
+ )
708
838
  )
709
839
  for pipeline_run in pipeline_runs_to_cancel:
710
840
  PipelineScheduler(pipeline_run).stop()
711
841
 
712
842
  def retry_pipeline_runs(pipeline_runs):
713
843
  for run in pipeline_runs:
714
- retry_pipeline_run(run)
844
+ retry_pipeline_run(run, repo_path)
715
845
 
716
846
  @safe_db_query
717
847
  def query_incomplete_block_runs(pipeline_uuid: str):
@@ -725,15 +855,18 @@ class PipelineResource(BaseResource):
725
855
  a.pipeline_uuid,
726
856
  ]
727
857
  result = (
728
- PipelineRun.
729
- select(*columns).
730
- join(b, a.id == b.pipeline_run_id).
731
- filter(a.pipeline_uuid == pipeline_uuid).
732
- filter(a.status == PipelineRun.PipelineRunStatus.FAILED).
733
- filter(b.status.not_in([
734
- BlockRun.BlockRunStatus.COMPLETED,
735
- BlockRun.BlockRunStatus.CONDITION_FAILED,
736
- ]))
858
+ PipelineRun.select(*columns)
859
+ .join(b, a.id == b.pipeline_run_id)
860
+ .filter(a.pipeline_uuid == pipeline_uuid)
861
+ .filter(a.status == PipelineRun.PipelineRunStatus.FAILED)
862
+ .filter(
863
+ b.status.not_in(
864
+ [
865
+ BlockRun.BlockRunStatus.COMPLETED,
866
+ BlockRun.BlockRunStatus.CONDITION_FAILED,
867
+ ]
868
+ )
869
+ )
737
870
  ).all()
738
871
 
739
872
  return result