mage-ai 0.9.12__py3-none-any.whl → 0.9.14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mage-ai might be problematic. Click here for more details.

Files changed (325) hide show
  1. mage_ai/ai/generator.py +1 -1
  2. mage_ai/ai/generator_cmds.py +1 -1
  3. mage_ai/ai/llm_pipeline_wizard.py +31 -17
  4. mage_ai/api/policies/BlockPolicy.py +1 -0
  5. mage_ai/api/policies/PipelineRunPolicy.py +1 -0
  6. mage_ai/api/policies/PipelineSchedulePolicy.py +12 -0
  7. mage_ai/api/policies/TagPolicy.py +18 -0
  8. mage_ai/api/presenters/BlockRunPresenter.py +4 -1
  9. mage_ai/api/presenters/PipelineRunPresenter.py +2 -0
  10. mage_ai/api/presenters/PipelineSchedulePresenter.py +9 -2
  11. mage_ai/api/presenters/TagPresenter.py +3 -0
  12. mage_ai/api/resources/BackfillResource.py +6 -3
  13. mage_ai/api/resources/BlockResource.py +2 -0
  14. mage_ai/api/resources/BlockRunResource.py +18 -12
  15. mage_ai/api/resources/DatabaseResource.py +23 -0
  16. mage_ai/api/resources/GitBranchResource.py +8 -1
  17. mage_ai/api/resources/GlobalDataProductResource.py +5 -2
  18. mage_ai/api/resources/PipelineResource.py +30 -14
  19. mage_ai/api/resources/PipelineScheduleResource.py +157 -1
  20. mage_ai/api/resources/SearchResultResource.py +1 -0
  21. mage_ai/api/resources/TagResource.py +16 -3
  22. mage_ai/data_integrations/sources/constants.py +1 -0
  23. mage_ai/data_preparation/git/__init__.py +18 -0
  24. mage_ai/data_preparation/models/block/__init__.py +34 -11
  25. mage_ai/data_preparation/models/block/dbt/utils/__init__.py +37 -10
  26. mage_ai/data_preparation/models/block/integration/__init__.py +1 -1
  27. mage_ai/data_preparation/models/block/sql/__init__.py +44 -2
  28. mage_ai/data_preparation/models/custom_templates/utils.py +14 -4
  29. mage_ai/data_preparation/models/global_data_product/__init__.py +8 -0
  30. mage_ai/data_preparation/models/pipeline.py +4 -0
  31. mage_ai/data_preparation/models/variable.py +5 -2
  32. mage_ai/data_preparation/repo_manager.py +28 -16
  33. mage_ai/io/clickhouse.py +9 -6
  34. mage_ai/io/redshift.py +9 -10
  35. mage_ai/io/utils.py +3 -1
  36. mage_ai/orchestration/db/migrations/README.md +39 -2
  37. mage_ai/orchestration/db/migrations/env.py +1 -0
  38. mage_ai/orchestration/db/migrations/versions/386bcfebd48d_create_tag_and_tagassociation_tables.py +59 -0
  39. mage_ai/orchestration/db/models/schedules.py +29 -2
  40. mage_ai/orchestration/db/models/tags.py +57 -0
  41. mage_ai/orchestration/pipeline_scheduler.py +7 -7
  42. mage_ai/orchestration/triggers/global_data_product.py +41 -40
  43. mage_ai/orchestration/triggers/utils.py +11 -1
  44. mage_ai/server/constants.py +1 -1
  45. mage_ai/server/frontend_dist/404.html +2 -2
  46. mage_ai/server/frontend_dist/404.html.html +2 -2
  47. mage_ai/server/frontend_dist/_next/static/CNjkRIWPaAu1yJUmIaX_q/_buildManifest.js +1 -0
  48. mage_ai/server/frontend_dist/_next/static/chunks/1005-a2f0e3ee378ef02b.js +1 -0
  49. mage_ai/server/frontend_dist/_next/static/chunks/1424-fca78f21a81a7183.js +1 -0
  50. mage_ai/server/frontend_dist/_next/static/chunks/1484-87d4d4a698ac63dc.js +1 -0
  51. mage_ai/server/frontend_dist/_next/static/chunks/2485-39885e5335888821.js +1 -0
  52. mage_ai/server/frontend_dist/_next/static/chunks/2786-f71862671f66d948.js +1 -0
  53. mage_ai/server/frontend_dist/_next/static/chunks/3391-6f0a0a5c254cd7f2.js +1 -0
  54. mage_ai/server/frontend_dist/_next/static/chunks/3881-131cf690e54c23a3.js +1 -0
  55. mage_ai/server/frontend_dist/_next/static/chunks/437-331193bd9b2fe777.js +1 -0
  56. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/4822-045cc7cdd7c95163.js → frontend_dist/_next/static/chunks/4822-ee62acb1927c8150.js} +1 -1
  57. mage_ai/server/frontend_dist/_next/static/chunks/547-bd961ea93f3eb98b.js +1 -0
  58. mage_ai/server/frontend_dist/_next/static/chunks/6299-fcb702d0f3d3291b.js +1 -0
  59. mage_ai/server/frontend_dist/_next/static/chunks/6422-0cdd6e596dcd43b6.js +1 -0
  60. mage_ai/server/frontend_dist/_next/static/chunks/{6786-77c7e36678abb2c6.js → 6786-55e1bca3c897d5ee.js} +1 -1
  61. mage_ai/server/frontend_dist/_next/static/chunks/7496-7e4dd11e3f3b8e79.js +1 -0
  62. mage_ai/server/frontend_dist/_next/static/chunks/7722-76c724a66240561b.js +1 -0
  63. mage_ai/server/frontend_dist/_next/static/chunks/7815-8b68b0eb665fcd2d.js +1 -0
  64. mage_ai/server/frontend_dist/_next/static/chunks/8190-d38ed1133030797d.js +1 -0
  65. mage_ai/server/frontend_dist/_next/static/chunks/8952-9d6fa18fa9378989.js +1 -0
  66. mage_ai/server/frontend_dist/_next/static/chunks/9605-9332e1686c46da7d.js +1 -0
  67. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9dae6fa5126cf001.js +1 -0
  68. mage_ai/server/frontend_dist/_next/static/chunks/pages/files-2dc2a0dfc0ff620d.js +1 -0
  69. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-6d3d53624debede6.js +1 -0
  70. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-2756fe6d9decae4a.js +1 -0
  71. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/settings-6577159a0af52a78.js +1 -0
  72. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-6815a3ece7dc1678.js +1 -0
  73. mage_ai/server/frontend_dist/_next/static/chunks/pages/overview-7f790238fe70d9ab.js +1 -0
  74. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipeline-runs-33c7f2550add0719.js +1 -0
  75. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-07e4a3a674e83578.js +1 -0
  76. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-b8b4bed1e8e50068.js +1 -0
  77. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-54cb4936accdd2d9.js +1 -0
  78. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-d331e98ad103b13e.js +1 -0
  79. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-d87d0c21758a2d7c.js +1 -0
  80. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-68e1b861ef4fc0d7.js +1 -0
  81. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-9fa0c4ce1c921a41.js +1 -0
  82. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-d5ef19ca1f9931de.js +1 -0
  83. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-4a2671811a153411.js +1 -0
  84. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-8872a6e00280f58c.js +1 -0
  85. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/syncs-484581ae34a1c596.js +1 -0
  86. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-ce11db27e4af7869.js +1 -0
  87. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers-624a2d7cbe6303b4.js +1 -0
  88. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-1169f4eecf752033.js +1 -0
  89. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/profile-d2224a447987fa69.js → frontend_dist/_next/static/chunks/pages/settings/account/profile-c3ff06a12baa40a2.js} +1 -1
  90. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-60b4398d7ae00206.js +1 -0
  91. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-dd827816bf4a7908.js → frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-05b53444f0e7449b.js} +1 -1
  92. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/sign-in-097aa0bef1e34cdb.js → frontend_dist/_next/static/chunks/pages/sign-in-a5e9561a6c0d2e38.js} +1 -1
  93. mage_ai/server/frontend_dist/_next/static/chunks/pages/templates/{[...slug]-78a07e7fe8973811.js → [...slug]-c21b269750441494.js} +1 -1
  94. mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/templates-19df643f52679d5a.js → frontend_dist/_next/static/chunks/pages/templates-ce2a29d477a6ce94.js} +1 -1
  95. mage_ai/server/frontend_dist/_next/static/chunks/pages/triggers-71bbb30dd9b57f80.js +1 -0
  96. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-1dc780d52fbd1573.js +1 -0
  97. mage_ai/server/frontend_dist/files.html +2 -2
  98. mage_ai/server/frontend_dist/global-data-products/[...slug].html +2 -2
  99. mage_ai/server/frontend_dist/global-data-products.html +2 -2
  100. mage_ai/server/frontend_dist/index.html +2 -2
  101. mage_ai/server/frontend_dist/manage/settings.html +2 -2
  102. mage_ai/server/frontend_dist/manage/users/[user].html +2 -2
  103. mage_ai/server/frontend_dist/manage/users/new.html +2 -2
  104. mage_ai/server/frontend_dist/manage/users.html +2 -2
  105. mage_ai/server/frontend_dist/manage.html +2 -2
  106. mage_ai/server/frontend_dist/overview.html +2 -2
  107. mage_ai/server/frontend_dist/pipeline-runs.html +2 -2
  108. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +2 -2
  109. mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +2 -2
  110. mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +2 -2
  111. mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +2 -2
  112. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +2 -2
  113. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
  114. mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +2 -2
  115. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +2 -2
  116. mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +2 -2
  117. mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +2 -2
  118. mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +2 -2
  119. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +2 -2
  120. mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +2 -2
  121. mage_ai/server/frontend_dist/pipelines/[pipeline].html +2 -2
  122. mage_ai/server/frontend_dist/pipelines.html +2 -2
  123. mage_ai/server/frontend_dist/settings/account/profile.html +2 -2
  124. mage_ai/server/frontend_dist/settings/workspace/preferences.html +2 -2
  125. mage_ai/server/frontend_dist/settings/workspace/sync-data.html +2 -2
  126. mage_ai/server/frontend_dist/settings/workspace/users.html +2 -2
  127. mage_ai/server/frontend_dist/settings.html +2 -2
  128. mage_ai/server/frontend_dist/sign-in.html +11 -11
  129. mage_ai/server/frontend_dist/templates/[...slug].html +2 -2
  130. mage_ai/server/frontend_dist/templates.html +2 -2
  131. mage_ai/server/frontend_dist/terminal.html +2 -2
  132. mage_ai/server/frontend_dist/test.html +3 -3
  133. mage_ai/server/frontend_dist/triggers.html +2 -2
  134. mage_ai/server/frontend_dist/version-control.html +2 -2
  135. mage_ai/server/frontend_dist_base_path_template/404.html +2 -2
  136. mage_ai/server/frontend_dist_base_path_template/404.html.html +2 -2
  137. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1005-a2f0e3ee378ef02b.js +1 -0
  138. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1424-fca78f21a81a7183.js +1 -0
  139. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1484-87d4d4a698ac63dc.js +1 -0
  140. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2485-39885e5335888821.js +1 -0
  141. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2786-f71862671f66d948.js +1 -0
  142. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3391-6f0a0a5c254cd7f2.js +1 -0
  143. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3881-131cf690e54c23a3.js +1 -0
  144. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/437-331193bd9b2fe777.js +1 -0
  145. mage_ai/server/{frontend_dist/_next/static/chunks/4822-045cc7cdd7c95163.js → frontend_dist_base_path_template/_next/static/chunks/4822-ee62acb1927c8150.js} +1 -1
  146. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/547-bd961ea93f3eb98b.js +1 -0
  147. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/6299-fcb702d0f3d3291b.js +1 -0
  148. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/6422-0cdd6e596dcd43b6.js +1 -0
  149. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{6786-77c7e36678abb2c6.js → 6786-55e1bca3c897d5ee.js} +1 -1
  150. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7496-7e4dd11e3f3b8e79.js +1 -0
  151. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7722-76c724a66240561b.js +1 -0
  152. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7815-8b68b0eb665fcd2d.js +1 -0
  153. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8190-d38ed1133030797d.js +1 -0
  154. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8952-9d6fa18fa9378989.js +1 -0
  155. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9605-9332e1686c46da7d.js +1 -0
  156. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-9dae6fa5126cf001.js +1 -0
  157. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/files-2dc2a0dfc0ff620d.js +1 -0
  158. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-6d3d53624debede6.js +1 -0
  159. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-2756fe6d9decae4a.js +1 -0
  160. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/settings-6577159a0af52a78.js +1 -0
  161. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-6815a3ece7dc1678.js +1 -0
  162. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/overview-7f790238fe70d9ab.js +1 -0
  163. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-33c7f2550add0719.js +1 -0
  164. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-07e4a3a674e83578.js +1 -0
  165. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-b8b4bed1e8e50068.js +1 -0
  166. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-54cb4936accdd2d9.js +1 -0
  167. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-d331e98ad103b13e.js +1 -0
  168. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-d87d0c21758a2d7c.js +1 -0
  169. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-68e1b861ef4fc0d7.js +1 -0
  170. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-9fa0c4ce1c921a41.js +1 -0
  171. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-d5ef19ca1f9931de.js +1 -0
  172. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs-4a2671811a153411.js +1 -0
  173. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-8872a6e00280f58c.js +1 -0
  174. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/syncs-484581ae34a1c596.js +1 -0
  175. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-ce11db27e4af7869.js +1 -0
  176. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers-624a2d7cbe6303b4.js +1 -0
  177. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines-1169f4eecf752033.js +1 -0
  178. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/account/profile-d2224a447987fa69.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/profile-c3ff06a12baa40a2.js} +1 -1
  179. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-60b4398d7ae00206.js +1 -0
  180. mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-dd827816bf4a7908.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-05b53444f0e7449b.js} +1 -1
  181. mage_ai/server/{frontend_dist/_next/static/chunks/pages/sign-in-097aa0bef1e34cdb.js → frontend_dist_base_path_template/_next/static/chunks/pages/sign-in-a5e9561a6c0d2e38.js} +1 -1
  182. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/templates/{[...slug]-78a07e7fe8973811.js → [...slug]-c21b269750441494.js} +1 -1
  183. mage_ai/server/{frontend_dist/_next/static/chunks/pages/templates-19df643f52679d5a.js → frontend_dist_base_path_template/_next/static/chunks/pages/templates-ce2a29d477a6ce94.js} +1 -1
  184. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/triggers-71bbb30dd9b57f80.js +1 -0
  185. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-1dc780d52fbd1573.js +1 -0
  186. mage_ai/server/frontend_dist_base_path_template/_next/static/yJvL-3bfsNF3WCStZ10Dm/_buildManifest.js +1 -0
  187. mage_ai/server/frontend_dist_base_path_template/files.html +2 -2
  188. mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +2 -2
  189. mage_ai/server/frontend_dist_base_path_template/global-data-products.html +2 -2
  190. mage_ai/server/frontend_dist_base_path_template/index.html +2 -2
  191. mage_ai/server/frontend_dist_base_path_template/manage/settings.html +2 -2
  192. mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +2 -2
  193. mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +2 -2
  194. mage_ai/server/frontend_dist_base_path_template/manage/users.html +2 -2
  195. mage_ai/server/frontend_dist_base_path_template/manage.html +2 -2
  196. mage_ai/server/frontend_dist_base_path_template/overview.html +2 -2
  197. mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +2 -2
  198. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +2 -2
  199. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +2 -2
  200. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +2 -2
  201. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +2 -2
  202. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +2 -2
  203. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
  204. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +2 -2
  205. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +2 -2
  206. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +2 -2
  207. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +2 -2
  208. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +2 -2
  209. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +2 -2
  210. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +2 -2
  211. mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +2 -2
  212. mage_ai/server/frontend_dist_base_path_template/pipelines.html +2 -2
  213. mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +2 -2
  214. mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +2 -2
  215. mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +2 -2
  216. mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +2 -2
  217. mage_ai/server/frontend_dist_base_path_template/settings.html +2 -2
  218. mage_ai/server/frontend_dist_base_path_template/sign-in.html +11 -11
  219. mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +2 -2
  220. mage_ai/server/frontend_dist_base_path_template/templates.html +2 -2
  221. mage_ai/server/frontend_dist_base_path_template/terminal.html +2 -2
  222. mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
  223. mage_ai/server/frontend_dist_base_path_template/triggers.html +2 -2
  224. mage_ai/server/frontend_dist_base_path_template/version-control.html +2 -2
  225. mage_ai/server/server.py +53 -27
  226. mage_ai/services/spark/spark.py +75 -2
  227. mage_ai/streaming/constants.py +1 -0
  228. mage_ai/streaming/sources/kafka.py +17 -9
  229. mage_ai/tests/data_preparation/models/custom_templates/__init__.py +0 -0
  230. mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +36 -0
  231. mage_ai/tests/data_preparation/models/test_block.py +48 -0
  232. mage_ai/tests/data_preparation/test_repo_manager.py +19 -1
  233. mage_ai/tests/server/test_server.py +20 -2
  234. mage_ai/tests/services/spark/__init__.py +0 -0
  235. mage_ai/tests/services/spark/test_spark.py +52 -0
  236. {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/METADATA +1 -1
  237. {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/RECORD +245 -233
  238. mage_ai/server/frontend_dist/_next/static/RdDEYzOW6lfmOEilKsz4V/_buildManifest.js +0 -1
  239. mage_ai/server/frontend_dist/_next/static/chunks/1005-2f8ef0e28c917767.js +0 -1
  240. mage_ai/server/frontend_dist/_next/static/chunks/1424-a2cd1bfc708a323b.js +0 -1
  241. mage_ai/server/frontend_dist/_next/static/chunks/1484-a07f74ae5c40141c.js +0 -1
  242. mage_ai/server/frontend_dist/_next/static/chunks/2485-b569baad92049d6b.js +0 -1
  243. mage_ai/server/frontend_dist/_next/static/chunks/2786-acce6ea0d9b4898e.js +0 -1
  244. mage_ai/server/frontend_dist/_next/static/chunks/3654-14ee3eacc42b118a.js +0 -1
  245. mage_ai/server/frontend_dist/_next/static/chunks/3881-0eb04f7f7a244347.js +0 -1
  246. mage_ai/server/frontend_dist/_next/static/chunks/437-d43ccff3a064a528.js +0 -1
  247. mage_ai/server/frontend_dist/_next/static/chunks/547-4ad2cdae967055b6.js +0 -1
  248. mage_ai/server/frontend_dist/_next/static/chunks/6299-cf188c1b7a1bc33c.js +0 -1
  249. mage_ai/server/frontend_dist/_next/static/chunks/7496-ab1be388ae09d362.js +0 -1
  250. mage_ai/server/frontend_dist/_next/static/chunks/7722-a74e6f977993e75e.js +0 -1
  251. mage_ai/server/frontend_dist/_next/static/chunks/7815-fbd99c8f259d9e8b.js +0 -1
  252. mage_ai/server/frontend_dist/_next/static/chunks/8952-98e54b14c1af492f.js +0 -1
  253. mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-74805cf1296fd50f.js +0 -1
  254. mage_ai/server/frontend_dist/_next/static/chunks/pages/files-bcdaa82adcca891d.js +0 -1
  255. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-abdbda772d94c386.js +0 -1
  256. mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-01f8ef10e87ec236.js +0 -1
  257. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/settings-950fbcb72961eb3c.js +0 -1
  258. mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-c6fd44191c812edf.js +0 -1
  259. mage_ai/server/frontend_dist/_next/static/chunks/pages/overview-6e46ed84cf3f4e9d.js +0 -1
  260. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipeline-runs-1d52bba72442c53f.js +0 -1
  261. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-8c33956a28fb4fa8.js +0 -1
  262. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-b28d151c32c2a469.js +0 -1
  263. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1708e5576cd26d1.js +0 -1
  264. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-0f3468d52020dcbf.js +0 -1
  265. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-489d02937d5f4695.js +0 -1
  266. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-544475d8e841638f.js +0 -1
  267. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-e117a99f334b6022.js +0 -1
  268. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-f1ef481c7beb8aef.js +0 -1
  269. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-ab52e3174c3dcf64.js +0 -1
  270. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-3e290f7d76c68b3f.js +0 -1
  271. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/syncs-1ac4dc91ec394c6d.js +0 -1
  272. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-4753de69f7e366c5.js +0 -1
  273. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers-cf285c43759e9b2a.js +0 -1
  274. mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-3f86e7f1c8d63beb.js +0 -1
  275. mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-b2354688508b0755.js +0 -1
  276. mage_ai/server/frontend_dist/_next/static/chunks/pages/triggers-cacb24e3632cfdfc.js +0 -1
  277. mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5a6e3a62fef551f1.js +0 -1
  278. mage_ai/server/frontend_dist_base_path_template/_next/static/4dbzHNz2XwS_s6Z0qsIGO/_buildManifest.js +0 -1
  279. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1005-2f8ef0e28c917767.js +0 -1
  280. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1424-a2cd1bfc708a323b.js +0 -1
  281. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1484-a07f74ae5c40141c.js +0 -1
  282. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2485-b569baad92049d6b.js +0 -1
  283. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2786-acce6ea0d9b4898e.js +0 -1
  284. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3654-14ee3eacc42b118a.js +0 -1
  285. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3881-0eb04f7f7a244347.js +0 -1
  286. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/437-d43ccff3a064a528.js +0 -1
  287. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/547-4ad2cdae967055b6.js +0 -1
  288. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/6299-cf188c1b7a1bc33c.js +0 -1
  289. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7496-ab1be388ae09d362.js +0 -1
  290. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7722-a74e6f977993e75e.js +0 -1
  291. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7815-fbd99c8f259d9e8b.js +0 -1
  292. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8952-98e54b14c1af492f.js +0 -1
  293. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-74805cf1296fd50f.js +0 -1
  294. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/files-bcdaa82adcca891d.js +0 -1
  295. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-abdbda772d94c386.js +0 -1
  296. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-01f8ef10e87ec236.js +0 -1
  297. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/settings-950fbcb72961eb3c.js +0 -1
  298. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-c6fd44191c812edf.js +0 -1
  299. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/overview-6e46ed84cf3f4e9d.js +0 -1
  300. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-1d52bba72442c53f.js +0 -1
  301. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-8c33956a28fb4fa8.js +0 -1
  302. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-b28d151c32c2a469.js +0 -1
  303. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1708e5576cd26d1.js +0 -1
  304. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-0f3468d52020dcbf.js +0 -1
  305. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-489d02937d5f4695.js +0 -1
  306. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-544475d8e841638f.js +0 -1
  307. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-e117a99f334b6022.js +0 -1
  308. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-f1ef481c7beb8aef.js +0 -1
  309. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs-ab52e3174c3dcf64.js +0 -1
  310. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-3e290f7d76c68b3f.js +0 -1
  311. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/syncs-1ac4dc91ec394c6d.js +0 -1
  312. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-4753de69f7e366c5.js +0 -1
  313. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers-cf285c43759e9b2a.js +0 -1
  314. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines-3f86e7f1c8d63beb.js +0 -1
  315. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-b2354688508b0755.js +0 -1
  316. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/triggers-cacb24e3632cfdfc.js +0 -1
  317. mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-5a6e3a62fef551f1.js +0 -1
  318. /mage_ai/server/frontend_dist/_next/static/{RdDEYzOW6lfmOEilKsz4V → CNjkRIWPaAu1yJUmIaX_q}/_middlewareManifest.js +0 -0
  319. /mage_ai/server/frontend_dist/_next/static/{RdDEYzOW6lfmOEilKsz4V → CNjkRIWPaAu1yJUmIaX_q}/_ssgManifest.js +0 -0
  320. /mage_ai/server/frontend_dist_base_path_template/_next/static/{4dbzHNz2XwS_s6Z0qsIGO → yJvL-3bfsNF3WCStZ10Dm}/_middlewareManifest.js +0 -0
  321. /mage_ai/server/frontend_dist_base_path_template/_next/static/{4dbzHNz2XwS_s6Z0qsIGO → yJvL-3bfsNF3WCStZ10Dm}/_ssgManifest.js +0 -0
  322. {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/LICENSE +0 -0
  323. {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/WHEEL +0 -0
  324. {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/entry_points.txt +0 -0
  325. {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/top_level.txt +0 -0
@@ -1,14 +1,16 @@
1
+ import importlib
1
2
  import uuid
2
3
 
3
4
  from sqlalchemy.orm import selectinload
4
5
 
5
6
  from mage_ai.api.resources.DatabaseResource import DatabaseResource
6
- from mage_ai.orchestration.db import safe_db_query
7
+ from mage_ai.orchestration.db import db_connection, safe_db_query
7
8
  from mage_ai.orchestration.db.models.schedules import (
8
9
  EventMatcher,
9
10
  PipelineSchedule,
10
11
  pipeline_schedule_event_matcher_association_table,
11
12
  )
13
+ from mage_ai.orchestration.db.models.tags import Tag, TagAssociation, TagAssociationWithTag
12
14
  from mage_ai.settings.repo import get_repo_path
13
15
  from mage_ai.shared.hash import merge_dict
14
16
 
@@ -17,6 +19,10 @@ class PipelineScheduleResource(DatabaseResource):
17
19
  datetime_keys = ['start_time']
18
20
  model_class = PipelineSchedule
19
21
 
22
+ def __init__(self, *args, **kwargs):
23
+ super().__init__(*args, **kwargs)
24
+ self.tag_associations_updated = None
25
+
20
26
  @classmethod
21
27
  @safe_db_query
22
28
  def collection(self, query_arg, meta, user, **kwargs):
@@ -26,7 +32,35 @@ class PipelineScheduleResource(DatabaseResource):
26
32
  if global_data_product_uuid:
27
33
  global_data_product_uuid = global_data_product_uuid[0]
28
34
 
35
+ tag_names = query_arg.get('tag[]', [])
36
+ if tag_names:
37
+ if isinstance(tag_names, str):
38
+ tag_names = tag_names.split(',')
39
+
29
40
  query = PipelineSchedule.repo_query
41
+
42
+ if len(tag_names) >= 1:
43
+ tag_associations = (
44
+ TagAssociation.
45
+ select(
46
+ Tag.name,
47
+ TagAssociation.taggable_id,
48
+ TagAssociation.taggable_type,
49
+ ).
50
+ join(
51
+ Tag,
52
+ Tag.id == TagAssociation.tag_id,
53
+ ).
54
+ filter(
55
+ Tag.name.in_(tag_names),
56
+ TagAssociation.taggable_type == self.model_class.__name__,
57
+ ).
58
+ all()
59
+ )
60
+ query = query.filter(
61
+ PipelineSchedule.id.in_([ta.taggable_id for ta in tag_associations]),
62
+ )
63
+
30
64
  if global_data_product_uuid or pipeline:
31
65
  query = (
32
66
  query.
@@ -108,6 +142,128 @@ class PipelineScheduleResource(DatabaseResource):
108
142
  ps = [p for p in PipelineSchedule.query.filter(PipelineSchedule.id.in_(new_ids))]
109
143
  em.update(pipeline_schedules=ps)
110
144
 
145
+ tag_names = payload.pop('tags', None)
146
+ if tag_names is not None:
147
+ # 1. Fetch all tag associations
148
+ # 2. Delete any tag associations that don’t have a tag in tag_names
149
+ tag_associations_to_keep = []
150
+ tag_associations_to_delete = []
151
+ for ta in self.tag_associations:
152
+ if ta.name in tag_names:
153
+ tag_associations_to_keep.append(ta)
154
+ else:
155
+ tag_associations_to_delete.append(ta)
156
+
157
+ if len(tag_associations_to_delete) >= 1:
158
+ delete_query = TagAssociation.__table__.delete().where(
159
+ TagAssociation.id.in_(
160
+ [ta.id for ta in tag_associations_to_delete],
161
+ ),
162
+ )
163
+ db_connection.session.execute(delete_query)
164
+
165
+ # 3. Fetch all tags in tag_names that aren’t in tag associations
166
+ existing_tags = Tag.query.filter(
167
+ Tag.name.in_(tag_names),
168
+ Tag.name.not_in([ta.name for ta in tag_associations_to_keep]),
169
+ ).all()
170
+ existing_tag_pks = []
171
+ existing_tag_names = []
172
+ for tag in existing_tags:
173
+ existing_tag_pks.append(tag.id)
174
+ existing_tag_names.append(tag.name)
175
+
176
+ # 4. Create new tags
177
+ tag_names_to_keep = [ta.name for ta in tag_associations_to_keep]
178
+ tag_names_to_create = \
179
+ [tag_name for tag_name in tag_names if tag_name not in (
180
+ existing_tag_names + tag_names_to_keep
181
+ )]
182
+
183
+ new_tags = [Tag(name=tag_name) for tag_name in tag_names_to_create]
184
+ db_connection.session.bulk_save_objects(
185
+ new_tags,
186
+ return_defaults=True,
187
+ )
188
+
189
+ # 5. Create tag associations
190
+ tag_names_to_use = existing_tag_names.copy()
191
+ tag_ids_to_use = existing_tag_pks.copy()
192
+ for tag in new_tags:
193
+ tag_names_to_use.append(tag.name)
194
+ tag_ids_to_use.append(tag.id)
195
+
196
+ new_tag_associations = [TagAssociation(
197
+ tag_id=tag_id,
198
+ taggable_id=self.model.id,
199
+ taggable_type=self.model.__class__.__name__,
200
+ ) for tag_id in tag_ids_to_use]
201
+ db_connection.session.bulk_save_objects(
202
+ new_tag_associations,
203
+ return_defaults=True,
204
+ )
205
+
206
+ tag_associations_updated = []
207
+ for tag_name, new_tag_association in zip(tag_names_to_use, new_tag_associations):
208
+ taw = TagAssociationWithTag(
209
+ id=new_tag_association.id,
210
+ name=tag_name,
211
+ tag_id=new_tag_association.tag_id,
212
+ taggable_id=new_tag_association.taggable_id,
213
+ taggable_type=new_tag_association.taggable_type,
214
+ )
215
+ tag_associations_updated.append(taw)
216
+
217
+ self.tag_associations_updated = tag_associations_updated + tag_associations_to_keep
218
+
111
219
  super().update(payload)
112
220
 
113
221
  return self
222
+
223
+ def get_tag_associations(self):
224
+ if self.tag_associations_updated is None:
225
+ return self.tag_associations
226
+ else:
227
+ return self.tag_associations_updated
228
+
229
+
230
+ def __load_tag_associations(resource):
231
+ pipeline_schedule_ids = [r.id for r in resource.result_set()]
232
+ result = (
233
+ TagAssociation.
234
+ select(
235
+ TagAssociation.id,
236
+ TagAssociation.tag_id,
237
+ TagAssociation.taggable_id,
238
+ TagAssociation.taggable_type,
239
+ Tag.name,
240
+ ).
241
+ join(
242
+ Tag,
243
+ Tag.id == TagAssociation.tag_id,
244
+ ).
245
+ filter(
246
+ TagAssociation.taggable_id.in_(pipeline_schedule_ids),
247
+ TagAssociation.taggable_type == resource.model.__class__.__name__,
248
+ ).
249
+ all()
250
+ )
251
+ TagResource = getattr(
252
+ importlib.import_module('mage_ai.api.resources.TagResource'),
253
+ 'TagResource',
254
+ )
255
+
256
+ return TagResource.build_result_set(result, resource.current_user)
257
+
258
+
259
+ def __select_tag_associations(resource, arr):
260
+ def _func(res):
261
+ return resource.id == res.taggable_id
262
+ return list(filter(_func, arr))
263
+
264
+
265
+ PipelineScheduleResource.register_collective_loader(
266
+ 'tag_associations',
267
+ load=__load_tag_associations,
268
+ select=__select_tag_associations,
269
+ )
@@ -36,6 +36,7 @@ def filter_results(result: Dict) -> bool:
36
36
 
37
37
  if block_type in [
38
38
  BlockType.CALLBACK,
39
+ BlockType.CHART,
39
40
  BlockType.CONDITIONAL,
40
41
  BlockType.EXTENSION,
41
42
  ]:
@@ -1,15 +1,28 @@
1
- from mage_ai.api.resources.GenericResource import GenericResource
1
+ from mage_ai.api.resources.DatabaseResource import DatabaseResource
2
2
  from mage_ai.cache.tag import TagCache
3
+ from mage_ai.orchestration.db import safe_db_query
4
+ from mage_ai.orchestration.db.models.tags import Tag
5
+ from mage_ai.shared.array import unique_by
3
6
 
4
7
 
5
- class TagResource(GenericResource):
8
+ class TagResource(DatabaseResource):
9
+ model_class = Tag
10
+
6
11
  @classmethod
12
+ @safe_db_query
7
13
  async def collection(self, query, meta, user, **kwargs):
8
14
  cache = await TagCache.initialize_cache()
9
15
  tag_uuids = cache.get_tags().keys()
16
+ tags = sorted(
17
+ unique_by(
18
+ list(Tag.query.all()) + [Tag(name=uuid) for uuid in tag_uuids],
19
+ lambda x: x.name,
20
+ ),
21
+ key=lambda x: x.name,
22
+ )
10
23
 
11
24
  return self.build_result_set(
12
- [dict(uuid=uuid) for uuid in tag_uuids],
25
+ tags,
13
26
  user,
14
27
  **kwargs,
15
28
  )
@@ -38,6 +38,7 @@ SOURCES = sorted([
38
38
  dict(name='Pipedrive'),
39
39
  dict(name='Postmark'),
40
40
  dict(name='Salesforce'),
41
+ dict(name='Sftp'),
41
42
  dict(name='Stripe'),
42
43
  dict(name='Twitter Ads'),
43
44
  dict(name='Zendesk'),
@@ -434,6 +434,24 @@ class Git:
434
434
  for url in remote.urls:
435
435
  if url.lower().startswith('https'):
436
436
  repository_names.append('/'.join(url.split('/')[-2:]).replace('.git', ''))
437
+
438
+ # Remove the token from the URL
439
+ # e.g. https://[user]:[token]@[netloc]
440
+ parts = url.split('@')
441
+
442
+ parts_arr = []
443
+ if len(parts) >= 2:
444
+ # https://[user]:[token]
445
+ parts2 = parts[0].split(':')
446
+ # ['https', '', 'user', 'token']
447
+ parts2[len(parts2) - 1] = '[token]'
448
+ parts_arr.append(':'.join(parts2))
449
+ parts_arr += parts[1:]
450
+ else:
451
+ parts_arr += parts
452
+
453
+ url = '@'.join(parts_arr)
454
+
437
455
  urls.append(url)
438
456
  except GitCommandError as err:
439
457
  print('WARNING (mage_ai.data_preparation.git.remotes):')
@@ -17,6 +17,7 @@ import pandas as pd
17
17
  import simplejson
18
18
  from jinja2 import Template
19
19
 
20
+ import mage_ai.data_preparation.decorators
20
21
  from mage_ai.cache.block import BlockCache
21
22
  from mage_ai.data_cleaner.shared.utils import is_geo_dataframe, is_spark_dataframe
22
23
  from mage_ai.data_preparation.logging.logger import DictLogger
@@ -70,6 +71,7 @@ from mage_ai.shared.utils import clean_name as clean_name_orig
70
71
  from mage_ai.shared.utils import is_spark_env
71
72
 
72
73
  PYTHON_COMMAND = 'python3'
74
+ BLOCK_EXISTS_ERROR = '[ERR_BLOCK_EXISTS]'
73
75
 
74
76
 
75
77
  async def run_blocks(
@@ -497,6 +499,7 @@ class Block:
497
499
  pipeline=None,
498
500
  priority: int = None,
499
501
  replicated_block: str = None,
502
+ require_unique_name: bool = False,
500
503
  upstream_block_uuids: List[str] = None,
501
504
  config: Dict = None,
502
505
  widget: bool = False,
@@ -513,9 +516,14 @@ class Block:
513
516
  uuid = clean_name(name)
514
517
  language = language or BlockLanguage.PYTHON
515
518
 
516
- # Don’t create a file if block is replicated from another block
519
+ # Don’t create a file if block is replicated from another block.
520
+
521
+ # Only create a file on the filesystem if the block type isn’t a global data product
522
+ # because global data products reference a data product which already has its
523
+ # own files.
517
524
  if not replicated_block and \
518
- (BlockType.DBT != block_type or BlockLanguage.YAML == language):
525
+ (BlockType.DBT != block_type or BlockLanguage.YAML == language) and \
526
+ BlockType.GLOBAL_DATA_PRODUCT != block_type:
519
527
 
520
528
  block_directory = self.file_directory_name(block_type)
521
529
  block_dir_path = os.path.join(repo_path, block_directory)
@@ -527,16 +535,20 @@ class Block:
527
535
  file_extension = BLOCK_LANGUAGE_TO_FILE_EXTENSION[language]
528
536
  file_path = os.path.join(block_dir_path, f'{uuid}.{file_extension}')
529
537
  if os.path.exists(file_path):
530
- if pipeline is not None and pipeline.has_block(
538
+ if (pipeline is not None and pipeline.has_block(
531
539
  uuid,
532
540
  block_type=block_type,
533
541
  extension_uuid=extension_uuid,
534
- ):
535
- raise Exception(f'Block {uuid} already exists. Please use a different name.')
536
- elif BlockType.GLOBAL_DATA_PRODUCT != block_type:
537
- # Only create a file on the filesystem if the block type isn’t a global data product
538
- # because global data products reference a data product which already has its
539
- # own files.
542
+ )) or require_unique_name:
543
+ """
544
+ The BLOCK_EXISTS_ERROR constant is used on the frontend to identify when
545
+ a user is trying to create a new block with an existing block name, and
546
+ link them to the existing block file so the user can choose to add the
547
+ existing block to their pipeline.
548
+ """
549
+ raise Exception(f'{BLOCK_EXISTS_ERROR} Block {uuid} already exists. \
550
+ Please use a different name.')
551
+ else:
540
552
  load_template(
541
553
  block_type,
542
554
  config,
@@ -1099,6 +1111,8 @@ class Block:
1099
1111
  ) -> List:
1100
1112
  if logging_tags is None:
1101
1113
  logging_tags = dict()
1114
+ if input_vars is None:
1115
+ input_vars = list()
1102
1116
 
1103
1117
  decorated_functions = []
1104
1118
  test_functions = []
@@ -1166,9 +1180,18 @@ class Block:
1166
1180
  block_uuid = self.replicated_block
1167
1181
  block_file_path = self.replicated_block_object.file_path
1168
1182
  spec = importlib.util.spec_from_file_location(
1169
- block_uuid, block_file_path,
1183
+ block_uuid,
1184
+ block_file_path,
1170
1185
  )
1171
1186
  module = importlib.util.module_from_spec(spec)
1187
+ # Set the decorators in the module in case they are not defined in the block
1188
+ # code
1189
+ setattr(
1190
+ module,
1191
+ self.type,
1192
+ getattr(mage_ai.data_preparation.decorators, self.type),
1193
+ )
1194
+ module.test = mage_ai.data_preparation.decorators.test
1172
1195
  spec.loader.exec_module(module)
1173
1196
  block_function_updated = getattr(module, block_function.__name__)
1174
1197
  self.module = module
@@ -2261,7 +2284,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
2261
2284
  f'Block {new_uuid} already exists in pipeline. Please use a different name.'
2262
2285
  )
2263
2286
 
2264
- if not self.replicated_block:
2287
+ if not self.replicated_block and BlockType.GLOBAL_DATA_PRODUCT != self.type:
2265
2288
  if os.path.exists(new_file_path):
2266
2289
  raise Exception(f'Block {new_uuid} already exists. Please use a different name.')
2267
2290
 
@@ -94,6 +94,7 @@ def parse_attributes(block) -> Dict:
94
94
  config = model_config(block.content)
95
95
  if config.get('alias'):
96
96
  table_name = config['alias']
97
+ database = config.get('database', None)
97
98
 
98
99
  full_path = os.path.join(get_repo_path(), 'dbt', file_path)
99
100
 
@@ -131,6 +132,7 @@ def parse_attributes(block) -> Dict:
131
132
  snapshot = first_folder_name and first_folder_name in snapshot_paths
132
133
 
133
134
  return dict(
135
+ database=database,
134
136
  dbt_project=dbt_project,
135
137
  dbt_project_full_path=dbt_project_full_path,
136
138
  file_extension=file_extension,
@@ -456,7 +458,11 @@ def get_profile(block, profile_target: str = None) -> Dict:
456
458
  return load_profile(profile_name, profiles_full_path, profile_target)
457
459
 
458
460
 
459
- def config_file_loader_and_configuration(block, profile_target: str) -> Dict:
461
+ def config_file_loader_and_configuration(
462
+ block,
463
+ profile_target: str,
464
+ **kwargs,
465
+ ) -> Dict:
460
466
  profile = get_profile(block, profile_target)
461
467
 
462
468
  if not profile:
@@ -492,12 +498,15 @@ def config_file_loader_and_configuration(block, profile_target: str) -> Dict:
492
498
  )
493
499
  elif DataSource.BIGQUERY == profile_type:
494
500
  keyfile = profile.get('keyfile')
495
- database = profile.get('project')
501
+ database = kwargs.get('database') or profile.get('project')
496
502
  schema = profile.get('dataset')
497
503
 
498
- config_file_loader = ConfigFileLoader(config=dict(
504
+ config_file_loader_kwargs = dict(
499
505
  GOOGLE_SERVICE_ACC_KEY_FILEPATH=keyfile,
500
- ))
506
+ )
507
+ if profile.get('location'):
508
+ config_file_loader_kwargs['GOOGLE_LOCATION'] = profile.get('location')
509
+ config_file_loader = ConfigFileLoader(config=config_file_loader_kwargs)
501
510
  configuration = dict(
502
511
  data_provider=profile_type,
503
512
  data_provider_database=database,
@@ -573,11 +582,11 @@ def config_file_loader_and_configuration(block, profile_target: str) -> Dict:
573
582
  SNOWFLAKE_ROLE=profile.get('role'),
574
583
  )
575
584
 
576
- if 'password' in profile:
585
+ if profile.get('password', None):
577
586
  config['SNOWFLAKE_PASSWORD'] = profile['password']
578
- if 'private_key_passphrase' in profile:
587
+ if profile.get('private_key_passphrase', None):
579
588
  config['SNOWFLAKE_PRIVATE_KEY_PASSPHRASE'] = profile['private_key_passphrase']
580
- if 'private_key_path' in profile:
589
+ if profile.get('private_key_path', None):
581
590
  config['SNOWFLAKE_PRIVATE_KEY_PATH'] = profile['private_key_path']
582
591
 
583
592
  config_file_loader = ConfigFileLoader(config=config)
@@ -927,10 +936,12 @@ def execute_query(
927
936
  profile_target: str,
928
937
  query_string: str,
929
938
  limit: int = None,
939
+ database: str = None,
930
940
  ) -> DataFrame:
931
941
  config_file_loader, configuration = config_file_loader_and_configuration(
932
942
  block,
933
943
  profile_target,
944
+ database=database,
934
945
  )
935
946
 
936
947
  data_provider = configuration['data_provider']
@@ -1304,12 +1315,15 @@ def fetch_model_data(
1304
1315
  # If the model SQL file contains a config with schema, change the schema to use that.
1305
1316
  # https://docs.getdbt.com/reference/resource-configs/schema
1306
1317
  config = model_config(block.content)
1318
+ config_database = config.get('database')
1307
1319
  config_schema = config.get('schema')
1320
+
1321
+ # settings from the dbt_project.yml
1322
+ model_configurations = get_model_configurations_from_dbt_project_settings(block)
1323
+
1308
1324
  if config_schema:
1309
1325
  schema = f'{schema}_{config_schema}'
1310
1326
  else:
1311
- # settings from the dbt_project.yml
1312
- model_configurations = get_model_configurations_from_dbt_project_settings(block)
1313
1327
  model_configuration_schema = None
1314
1328
  if model_configurations:
1315
1329
  model_configuration_schema = (model_configurations.get('schema') or
@@ -1318,9 +1332,22 @@ def fetch_model_data(
1318
1332
  if model_configuration_schema:
1319
1333
  schema = f"{schema}_{model_configuration_schema}"
1320
1334
 
1335
+ database = None
1336
+ if config_database:
1337
+ database = config_database
1338
+ elif model_configurations:
1339
+ database = (model_configurations.get('database') or
1340
+ model_configurations.get('+database'))
1341
+
1321
1342
  query_string = f'SELECT * FROM {schema}.{table_name}'
1322
1343
 
1323
- return execute_query(block, profile_target, query_string, limit)
1344
+ return execute_query(
1345
+ block,
1346
+ profile_target,
1347
+ query_string,
1348
+ limit,
1349
+ database=database,
1350
+ )
1324
1351
 
1325
1352
 
1326
1353
  def upstream_blocks_from_sources(block: Block) -> List[Block]:
@@ -122,7 +122,7 @@ class IntegrationBlock(Block):
122
122
  proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
123
123
 
124
124
  for line in proc.stdout:
125
- f.write(filter_out_config_values(line.decode(), config)),
125
+ f.write(line.decode()),
126
126
  print_log_from_line(
127
127
  line,
128
128
  config=config,
@@ -46,6 +46,44 @@ def execute_sql_code(
46
46
  config_file_loader: Any = None,
47
47
  configuration: Dict = None,
48
48
  ) -> List[Any]:
49
+ """
50
+ Execute SQL code within the given block's data context.
51
+
52
+ Args:
53
+ block (Block): The block containing the SQL execution context.
54
+ query (str): The SQL query to execute.
55
+ dynamic_block_index (int, optional): Index of the dynamic block, if applicable.
56
+ dynamic_upstream_block_uuids (List[str], optional): List of upstream block UUIDs for
57
+ dynamic execution.
58
+ execution_partition (str, optional): The partition for execution.
59
+ from_notebook (bool, optional): Indicates if execution is from a notebook.
60
+ global_vars (Dict, optional): Global variables to be used in the execution.
61
+ config_file_loader (Any, optional): Configuration file loader for data sources.
62
+ configuration (Dict, optional): Configuration settings for the block. If not provided, the
63
+ configuration from the block's context will be used. The configuration dictionary may
64
+ contain the following parameters:
65
+
66
+ - `use_raw_sql` (bool): If True, execute the query as raw SQL. Default is False.
67
+ - `data_provider` (str): The data provider for the execution, e.g., 'bigquery',
68
+ 'clickhouse', etc.
69
+ - `data_provider_database` (str): The database name for the data provider.
70
+ - `data_provider_schema` (str): The schema name for the data provider.
71
+ - `export_write_policy` (str): The write policy for exporting data. Default is
72
+ ExportWritePolicy.APPEND.
73
+ - `limit` (int): The maximum number of rows to return in notebook.
74
+ Default is QUERY_ROW_LIMIT.
75
+ - `limit_in_pipeline_run` (int): Limit rows when running the block in the pipeline run.
76
+ Default is QUERY_ROW_LIMIT.
77
+ - Other provider-specific parameters may also be present.
78
+ Returns:
79
+ List[Any]: A list containing the query execution results.
80
+
81
+ Note:
82
+ This method executes the provided SQL query within the context of the given block.
83
+ It supports various data sources such as BigQuery, ClickHouse, Druid, MSSQL, MySQL,
84
+ PostgreSQL, Redshift, Snowflake, and Trino, applying relevant configurations and
85
+ returning the query execution results.
86
+ """
49
87
  configuration = configuration if configuration else block.configuration
50
88
  use_raw_sql = configuration.get('use_raw_sql')
51
89
 
@@ -68,10 +106,12 @@ def execute_sql_code(
68
106
  should_query = block.type in PREVIEWABLE_BLOCK_TYPES
69
107
 
70
108
  limit = int(configuration.get('limit') or QUERY_ROW_LIMIT)
109
+ # Limit rows when running the block in the pipeline run
110
+ limit_in_pipeline_run = int(configuration.get('limit_in_pipeline_run') or QUERY_ROW_LIMIT)
71
111
  if from_notebook:
72
112
  limit = min(limit, QUERY_ROW_LIMIT)
73
113
  else:
74
- limit = QUERY_ROW_LIMIT
114
+ limit = min(limit_in_pipeline_run, QUERY_ROW_LIMIT)
75
115
 
76
116
  create_upstream_block_tables_kwargs = dict(
77
117
  configuration=configuration,
@@ -280,7 +320,9 @@ def execute_sql_code(
280
320
  if should_query:
281
321
  return [
282
322
  loader.load(
283
- f'SELECT * FROM {table_name}',
323
+ # Add the limit directly in the SELECT statement
324
+ # since io.mssql doesn't support enforcing limit
325
+ f'SELECT TOP {limit} * FROM {table_name}',
284
326
  limit=limit,
285
327
  verbose=False,
286
328
  ),
@@ -56,13 +56,23 @@ def group_and_hydrate_files(
56
56
  file_dicts: List[Dict],
57
57
  custom_template_class,
58
58
  ) -> List:
59
- groups = group_by(lambda x: os.path.join(*x.get('parent_names', [])), file_dicts)
59
+ def _func(x):
60
+ arr = ['']
60
61
 
61
- arr = []
62
+ if x:
63
+ parent_names = x.get('parent_names', []) or []
64
+ if parent_names and len(parent_names) >= 1:
65
+ arr = [str(parent_name) for parent_name in parent_names]
66
+
67
+ return os.path.join(*arr)
68
+
69
+ groups = group_by(_func, file_dicts)
70
+
71
+ custom_templates = []
62
72
 
63
73
  for template_uuid, _ in groups.items():
64
74
  custom_template = custom_template_class.load(template_uuid=template_uuid)
65
75
  if custom_template:
66
- arr.append(custom_template)
76
+ custom_templates.append(custom_template)
67
77
 
68
- return arr
78
+ return custom_templates
@@ -133,6 +133,8 @@ class GlobalDataProduct:
133
133
  else:
134
134
  return d
135
135
 
136
+ return None
137
+
136
138
  def is_outdated_after(self, now: datetime = None, return_values: bool = False) -> bool:
137
139
  values = {}
138
140
  outdated_starting_at = self.outdated_starting_at or {}
@@ -174,6 +176,10 @@ class GlobalDataProduct:
174
176
  def next_run_at(self, pipeline_run: 'PipelineRun') -> datetime:
175
177
  execution_date = pipeline_run.execution_date
176
178
  outdated_at_delta = self.get_outdated_at_delta()
179
+
180
+ if not outdated_at_delta:
181
+ return None
182
+
177
183
  if execution_date and outdated_at_delta:
178
184
  execution_date += outdated_at_delta
179
185
 
@@ -189,6 +195,8 @@ class GlobalDataProduct:
189
195
  now = datetime.utcnow().replace(tzinfo=timezone.utc)
190
196
 
191
197
  execution_date = self.next_run_at(pipeline_run)
198
+ if not execution_date:
199
+ return [False, False]
192
200
 
193
201
  outdated = execution_date and now >= execution_date
194
202
 
@@ -885,6 +885,10 @@ class Pipeline:
885
885
  if block_data.get('has_callback') is not None:
886
886
  block.update(extract(block_data, ['has_callback']))
887
887
 
888
+ color = block_data.get('color')
889
+ if color is not None and color != block.color:
890
+ block.update(extract(block_data, ['color']))
891
+
888
892
  configuration = block_data.get('configuration')
889
893
  if configuration:
890
894
  if configuration.get('dynamic') and not is_dynamic_block(block):
@@ -430,11 +430,14 @@ class Variable:
430
430
  df_output = data.copy()
431
431
  # Clean up data types since parquet doesn't support mixed data types
432
432
  for c in df_output.columns:
433
- c_dtype = df_output[c].dtype
433
+ df_col = df_output[c]
434
+ if type(df_col) is pd.DataFrame:
435
+ raise Exception(f'Please do not use duplicate column name: "{c}"')
436
+ c_dtype = df_col.dtype
434
437
  if not is_object_dtype(c_dtype):
435
438
  column_types[c] = str(c_dtype)
436
439
  else:
437
- series_non_null = df_output[c].dropna()
440
+ series_non_null = df_col.dropna()
438
441
  if len(series_non_null) > 0:
439
442
  coltype = type(series_non_null.iloc[0])
440
443
  if is_object_dtype(series_non_null.dtype):