mage-ai 0.9.12__py3-none-any.whl → 0.9.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mage-ai might be problematic. Click here for more details.
- mage_ai/ai/generator.py +1 -1
- mage_ai/ai/generator_cmds.py +1 -1
- mage_ai/ai/llm_pipeline_wizard.py +31 -17
- mage_ai/api/policies/BlockPolicy.py +1 -0
- mage_ai/api/policies/PipelineRunPolicy.py +1 -0
- mage_ai/api/policies/PipelineSchedulePolicy.py +12 -0
- mage_ai/api/policies/TagPolicy.py +18 -0
- mage_ai/api/presenters/BlockRunPresenter.py +4 -1
- mage_ai/api/presenters/PipelineRunPresenter.py +2 -0
- mage_ai/api/presenters/PipelineSchedulePresenter.py +9 -2
- mage_ai/api/presenters/TagPresenter.py +3 -0
- mage_ai/api/resources/BackfillResource.py +6 -3
- mage_ai/api/resources/BlockResource.py +2 -0
- mage_ai/api/resources/BlockRunResource.py +18 -12
- mage_ai/api/resources/DatabaseResource.py +23 -0
- mage_ai/api/resources/GitBranchResource.py +8 -1
- mage_ai/api/resources/GlobalDataProductResource.py +5 -2
- mage_ai/api/resources/PipelineResource.py +30 -14
- mage_ai/api/resources/PipelineScheduleResource.py +157 -1
- mage_ai/api/resources/SearchResultResource.py +1 -0
- mage_ai/api/resources/TagResource.py +16 -3
- mage_ai/data_integrations/sources/constants.py +1 -0
- mage_ai/data_preparation/git/__init__.py +18 -0
- mage_ai/data_preparation/models/block/__init__.py +34 -11
- mage_ai/data_preparation/models/block/dbt/utils/__init__.py +37 -10
- mage_ai/data_preparation/models/block/integration/__init__.py +1 -1
- mage_ai/data_preparation/models/block/sql/__init__.py +44 -2
- mage_ai/data_preparation/models/custom_templates/utils.py +14 -4
- mage_ai/data_preparation/models/global_data_product/__init__.py +8 -0
- mage_ai/data_preparation/models/pipeline.py +4 -0
- mage_ai/data_preparation/models/variable.py +5 -2
- mage_ai/data_preparation/repo_manager.py +28 -16
- mage_ai/io/clickhouse.py +9 -6
- mage_ai/io/redshift.py +9 -10
- mage_ai/io/utils.py +3 -1
- mage_ai/orchestration/db/migrations/README.md +39 -2
- mage_ai/orchestration/db/migrations/env.py +1 -0
- mage_ai/orchestration/db/migrations/versions/386bcfebd48d_create_tag_and_tagassociation_tables.py +59 -0
- mage_ai/orchestration/db/models/schedules.py +29 -2
- mage_ai/orchestration/db/models/tags.py +57 -0
- mage_ai/orchestration/pipeline_scheduler.py +7 -7
- mage_ai/orchestration/triggers/global_data_product.py +41 -40
- mage_ai/orchestration/triggers/utils.py +11 -1
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +2 -2
- mage_ai/server/frontend_dist/404.html.html +2 -2
- mage_ai/server/frontend_dist/_next/static/CNjkRIWPaAu1yJUmIaX_q/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1005-a2f0e3ee378ef02b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1424-fca78f21a81a7183.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1484-87d4d4a698ac63dc.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/2485-39885e5335888821.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/2786-f71862671f66d948.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3391-6f0a0a5c254cd7f2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3881-131cf690e54c23a3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/437-331193bd9b2fe777.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/4822-045cc7cdd7c95163.js → frontend_dist/_next/static/chunks/4822-ee62acb1927c8150.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/547-bd961ea93f3eb98b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/6299-fcb702d0f3d3291b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/6422-0cdd6e596dcd43b6.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{6786-77c7e36678abb2c6.js → 6786-55e1bca3c897d5ee.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7496-7e4dd11e3f3b8e79.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7722-76c724a66240561b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7815-8b68b0eb665fcd2d.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8190-d38ed1133030797d.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8952-9d6fa18fa9378989.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/9605-9332e1686c46da7d.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9dae6fa5126cf001.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/files-2dc2a0dfc0ff620d.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-6d3d53624debede6.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-2756fe6d9decae4a.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/settings-6577159a0af52a78.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-6815a3ece7dc1678.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/overview-7f790238fe70d9ab.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipeline-runs-33c7f2550add0719.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-07e4a3a674e83578.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-b8b4bed1e8e50068.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-54cb4936accdd2d9.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-d331e98ad103b13e.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-d87d0c21758a2d7c.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-68e1b861ef4fc0d7.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-9fa0c4ce1c921a41.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-d5ef19ca1f9931de.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-4a2671811a153411.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-8872a6e00280f58c.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/syncs-484581ae34a1c596.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-ce11db27e4af7869.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers-624a2d7cbe6303b4.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-1169f4eecf752033.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/profile-d2224a447987fa69.js → frontend_dist/_next/static/chunks/pages/settings/account/profile-c3ff06a12baa40a2.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-60b4398d7ae00206.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-dd827816bf4a7908.js → frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-05b53444f0e7449b.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/sign-in-097aa0bef1e34cdb.js → frontend_dist/_next/static/chunks/pages/sign-in-a5e9561a6c0d2e38.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/templates/{[...slug]-78a07e7fe8973811.js → [...slug]-c21b269750441494.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/templates-19df643f52679d5a.js → frontend_dist/_next/static/chunks/pages/templates-ce2a29d477a6ce94.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/triggers-71bbb30dd9b57f80.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-1dc780d52fbd1573.js +1 -0
- mage_ai/server/frontend_dist/files.html +2 -2
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +2 -2
- mage_ai/server/frontend_dist/global-data-products.html +2 -2
- mage_ai/server/frontend_dist/index.html +2 -2
- mage_ai/server/frontend_dist/manage/settings.html +2 -2
- mage_ai/server/frontend_dist/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist/manage/users.html +2 -2
- mage_ai/server/frontend_dist/manage.html +2 -2
- mage_ai/server/frontend_dist/overview.html +2 -2
- mage_ai/server/frontend_dist/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist/pipelines.html +2 -2
- mage_ai/server/frontend_dist/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist/settings.html +2 -2
- mage_ai/server/frontend_dist/sign-in.html +11 -11
- mage_ai/server/frontend_dist/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist/templates.html +2 -2
- mage_ai/server/frontend_dist/terminal.html +2 -2
- mage_ai/server/frontend_dist/test.html +3 -3
- mage_ai/server/frontend_dist/triggers.html +2 -2
- mage_ai/server/frontend_dist/version-control.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/404.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/404.html.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1005-a2f0e3ee378ef02b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1424-fca78f21a81a7183.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1484-87d4d4a698ac63dc.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2485-39885e5335888821.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2786-f71862671f66d948.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3391-6f0a0a5c254cd7f2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3881-131cf690e54c23a3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/437-331193bd9b2fe777.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/4822-045cc7cdd7c95163.js → frontend_dist_base_path_template/_next/static/chunks/4822-ee62acb1927c8150.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/547-bd961ea93f3eb98b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/6299-fcb702d0f3d3291b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/6422-0cdd6e596dcd43b6.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{6786-77c7e36678abb2c6.js → 6786-55e1bca3c897d5ee.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7496-7e4dd11e3f3b8e79.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7722-76c724a66240561b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7815-8b68b0eb665fcd2d.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8190-d38ed1133030797d.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8952-9d6fa18fa9378989.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9605-9332e1686c46da7d.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-9dae6fa5126cf001.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/files-2dc2a0dfc0ff620d.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-6d3d53624debede6.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-2756fe6d9decae4a.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/settings-6577159a0af52a78.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-6815a3ece7dc1678.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/overview-7f790238fe70d9ab.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-33c7f2550add0719.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-07e4a3a674e83578.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-b8b4bed1e8e50068.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-54cb4936accdd2d9.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-d331e98ad103b13e.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-d87d0c21758a2d7c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-68e1b861ef4fc0d7.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-9fa0c4ce1c921a41.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-d5ef19ca1f9931de.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs-4a2671811a153411.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-8872a6e00280f58c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/syncs-484581ae34a1c596.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-ce11db27e4af7869.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers-624a2d7cbe6303b4.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines-1169f4eecf752033.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/account/profile-d2224a447987fa69.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/profile-c3ff06a12baa40a2.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-60b4398d7ae00206.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-dd827816bf4a7908.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-05b53444f0e7449b.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/sign-in-097aa0bef1e34cdb.js → frontend_dist_base_path_template/_next/static/chunks/pages/sign-in-a5e9561a6c0d2e38.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/templates/{[...slug]-78a07e7fe8973811.js → [...slug]-c21b269750441494.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/templates-19df643f52679d5a.js → frontend_dist_base_path_template/_next/static/chunks/pages/templates-ce2a29d477a6ce94.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/triggers-71bbb30dd9b57f80.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-1dc780d52fbd1573.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/yJvL-3bfsNF3WCStZ10Dm/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/files.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/index.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/overview.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +11 -11
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/templates.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/terminal.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/triggers.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/version-control.html +2 -2
- mage_ai/server/server.py +53 -27
- mage_ai/services/spark/spark.py +75 -2
- mage_ai/streaming/constants.py +1 -0
- mage_ai/streaming/sources/kafka.py +17 -9
- mage_ai/tests/data_preparation/models/custom_templates/__init__.py +0 -0
- mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +36 -0
- mage_ai/tests/data_preparation/models/test_block.py +48 -0
- mage_ai/tests/data_preparation/test_repo_manager.py +19 -1
- mage_ai/tests/server/test_server.py +20 -2
- mage_ai/tests/services/spark/__init__.py +0 -0
- mage_ai/tests/services/spark/test_spark.py +52 -0
- {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/METADATA +1 -1
- {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/RECORD +245 -233
- mage_ai/server/frontend_dist/_next/static/RdDEYzOW6lfmOEilKsz4V/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1005-2f8ef0e28c917767.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1424-a2cd1bfc708a323b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1484-a07f74ae5c40141c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2485-b569baad92049d6b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2786-acce6ea0d9b4898e.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3654-14ee3eacc42b118a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3881-0eb04f7f7a244347.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/437-d43ccff3a064a528.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/547-4ad2cdae967055b6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/6299-cf188c1b7a1bc33c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7496-ab1be388ae09d362.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7722-a74e6f977993e75e.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7815-fbd99c8f259d9e8b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8952-98e54b14c1af492f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-74805cf1296fd50f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/files-bcdaa82adcca891d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-abdbda772d94c386.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-01f8ef10e87ec236.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/settings-950fbcb72961eb3c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-c6fd44191c812edf.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/overview-6e46ed84cf3f4e9d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipeline-runs-1d52bba72442c53f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-8c33956a28fb4fa8.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-b28d151c32c2a469.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1708e5576cd26d1.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-0f3468d52020dcbf.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-489d02937d5f4695.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-544475d8e841638f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-e117a99f334b6022.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-f1ef481c7beb8aef.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-ab52e3174c3dcf64.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-3e290f7d76c68b3f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/syncs-1ac4dc91ec394c6d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-4753de69f7e366c5.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers-cf285c43759e9b2a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-3f86e7f1c8d63beb.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-b2354688508b0755.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/triggers-cacb24e3632cfdfc.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5a6e3a62fef551f1.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/4dbzHNz2XwS_s6Z0qsIGO/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1005-2f8ef0e28c917767.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1424-a2cd1bfc708a323b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1484-a07f74ae5c40141c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2485-b569baad92049d6b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2786-acce6ea0d9b4898e.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3654-14ee3eacc42b118a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3881-0eb04f7f7a244347.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/437-d43ccff3a064a528.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/547-4ad2cdae967055b6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/6299-cf188c1b7a1bc33c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7496-ab1be388ae09d362.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7722-a74e6f977993e75e.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7815-fbd99c8f259d9e8b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8952-98e54b14c1af492f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-74805cf1296fd50f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/files-bcdaa82adcca891d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-abdbda772d94c386.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-01f8ef10e87ec236.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/settings-950fbcb72961eb3c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-c6fd44191c812edf.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/overview-6e46ed84cf3f4e9d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-1d52bba72442c53f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-8c33956a28fb4fa8.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-b28d151c32c2a469.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1708e5576cd26d1.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-0f3468d52020dcbf.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-489d02937d5f4695.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-544475d8e841638f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-e117a99f334b6022.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-f1ef481c7beb8aef.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs-ab52e3174c3dcf64.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-3e290f7d76c68b3f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/syncs-1ac4dc91ec394c6d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-4753de69f7e366c5.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers-cf285c43759e9b2a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines-3f86e7f1c8d63beb.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-b2354688508b0755.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/triggers-cacb24e3632cfdfc.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-5a6e3a62fef551f1.js +0 -1
- /mage_ai/server/frontend_dist/_next/static/{RdDEYzOW6lfmOEilKsz4V → CNjkRIWPaAu1yJUmIaX_q}/_middlewareManifest.js +0 -0
- /mage_ai/server/frontend_dist/_next/static/{RdDEYzOW6lfmOEilKsz4V → CNjkRIWPaAu1yJUmIaX_q}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{4dbzHNz2XwS_s6Z0qsIGO → yJvL-3bfsNF3WCStZ10Dm}/_middlewareManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{4dbzHNz2XwS_s6Z0qsIGO → yJvL-3bfsNF3WCStZ10Dm}/_ssgManifest.js +0 -0
- {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/WHEEL +0 -0
- {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.12.dist-info → mage_ai-0.9.14.dist-info}/top_level.txt +0 -0
|
@@ -1,14 +1,16 @@
|
|
|
1
|
+
import importlib
|
|
1
2
|
import uuid
|
|
2
3
|
|
|
3
4
|
from sqlalchemy.orm import selectinload
|
|
4
5
|
|
|
5
6
|
from mage_ai.api.resources.DatabaseResource import DatabaseResource
|
|
6
|
-
from mage_ai.orchestration.db import safe_db_query
|
|
7
|
+
from mage_ai.orchestration.db import db_connection, safe_db_query
|
|
7
8
|
from mage_ai.orchestration.db.models.schedules import (
|
|
8
9
|
EventMatcher,
|
|
9
10
|
PipelineSchedule,
|
|
10
11
|
pipeline_schedule_event_matcher_association_table,
|
|
11
12
|
)
|
|
13
|
+
from mage_ai.orchestration.db.models.tags import Tag, TagAssociation, TagAssociationWithTag
|
|
12
14
|
from mage_ai.settings.repo import get_repo_path
|
|
13
15
|
from mage_ai.shared.hash import merge_dict
|
|
14
16
|
|
|
@@ -17,6 +19,10 @@ class PipelineScheduleResource(DatabaseResource):
|
|
|
17
19
|
datetime_keys = ['start_time']
|
|
18
20
|
model_class = PipelineSchedule
|
|
19
21
|
|
|
22
|
+
def __init__(self, *args, **kwargs):
|
|
23
|
+
super().__init__(*args, **kwargs)
|
|
24
|
+
self.tag_associations_updated = None
|
|
25
|
+
|
|
20
26
|
@classmethod
|
|
21
27
|
@safe_db_query
|
|
22
28
|
def collection(self, query_arg, meta, user, **kwargs):
|
|
@@ -26,7 +32,35 @@ class PipelineScheduleResource(DatabaseResource):
|
|
|
26
32
|
if global_data_product_uuid:
|
|
27
33
|
global_data_product_uuid = global_data_product_uuid[0]
|
|
28
34
|
|
|
35
|
+
tag_names = query_arg.get('tag[]', [])
|
|
36
|
+
if tag_names:
|
|
37
|
+
if isinstance(tag_names, str):
|
|
38
|
+
tag_names = tag_names.split(',')
|
|
39
|
+
|
|
29
40
|
query = PipelineSchedule.repo_query
|
|
41
|
+
|
|
42
|
+
if len(tag_names) >= 1:
|
|
43
|
+
tag_associations = (
|
|
44
|
+
TagAssociation.
|
|
45
|
+
select(
|
|
46
|
+
Tag.name,
|
|
47
|
+
TagAssociation.taggable_id,
|
|
48
|
+
TagAssociation.taggable_type,
|
|
49
|
+
).
|
|
50
|
+
join(
|
|
51
|
+
Tag,
|
|
52
|
+
Tag.id == TagAssociation.tag_id,
|
|
53
|
+
).
|
|
54
|
+
filter(
|
|
55
|
+
Tag.name.in_(tag_names),
|
|
56
|
+
TagAssociation.taggable_type == self.model_class.__name__,
|
|
57
|
+
).
|
|
58
|
+
all()
|
|
59
|
+
)
|
|
60
|
+
query = query.filter(
|
|
61
|
+
PipelineSchedule.id.in_([ta.taggable_id for ta in tag_associations]),
|
|
62
|
+
)
|
|
63
|
+
|
|
30
64
|
if global_data_product_uuid or pipeline:
|
|
31
65
|
query = (
|
|
32
66
|
query.
|
|
@@ -108,6 +142,128 @@ class PipelineScheduleResource(DatabaseResource):
|
|
|
108
142
|
ps = [p for p in PipelineSchedule.query.filter(PipelineSchedule.id.in_(new_ids))]
|
|
109
143
|
em.update(pipeline_schedules=ps)
|
|
110
144
|
|
|
145
|
+
tag_names = payload.pop('tags', None)
|
|
146
|
+
if tag_names is not None:
|
|
147
|
+
# 1. Fetch all tag associations
|
|
148
|
+
# 2. Delete any tag associations that don’t have a tag in tag_names
|
|
149
|
+
tag_associations_to_keep = []
|
|
150
|
+
tag_associations_to_delete = []
|
|
151
|
+
for ta in self.tag_associations:
|
|
152
|
+
if ta.name in tag_names:
|
|
153
|
+
tag_associations_to_keep.append(ta)
|
|
154
|
+
else:
|
|
155
|
+
tag_associations_to_delete.append(ta)
|
|
156
|
+
|
|
157
|
+
if len(tag_associations_to_delete) >= 1:
|
|
158
|
+
delete_query = TagAssociation.__table__.delete().where(
|
|
159
|
+
TagAssociation.id.in_(
|
|
160
|
+
[ta.id for ta in tag_associations_to_delete],
|
|
161
|
+
),
|
|
162
|
+
)
|
|
163
|
+
db_connection.session.execute(delete_query)
|
|
164
|
+
|
|
165
|
+
# 3. Fetch all tags in tag_names that aren’t in tag associations
|
|
166
|
+
existing_tags = Tag.query.filter(
|
|
167
|
+
Tag.name.in_(tag_names),
|
|
168
|
+
Tag.name.not_in([ta.name for ta in tag_associations_to_keep]),
|
|
169
|
+
).all()
|
|
170
|
+
existing_tag_pks = []
|
|
171
|
+
existing_tag_names = []
|
|
172
|
+
for tag in existing_tags:
|
|
173
|
+
existing_tag_pks.append(tag.id)
|
|
174
|
+
existing_tag_names.append(tag.name)
|
|
175
|
+
|
|
176
|
+
# 4. Create new tags
|
|
177
|
+
tag_names_to_keep = [ta.name for ta in tag_associations_to_keep]
|
|
178
|
+
tag_names_to_create = \
|
|
179
|
+
[tag_name for tag_name in tag_names if tag_name not in (
|
|
180
|
+
existing_tag_names + tag_names_to_keep
|
|
181
|
+
)]
|
|
182
|
+
|
|
183
|
+
new_tags = [Tag(name=tag_name) for tag_name in tag_names_to_create]
|
|
184
|
+
db_connection.session.bulk_save_objects(
|
|
185
|
+
new_tags,
|
|
186
|
+
return_defaults=True,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
# 5. Create tag associations
|
|
190
|
+
tag_names_to_use = existing_tag_names.copy()
|
|
191
|
+
tag_ids_to_use = existing_tag_pks.copy()
|
|
192
|
+
for tag in new_tags:
|
|
193
|
+
tag_names_to_use.append(tag.name)
|
|
194
|
+
tag_ids_to_use.append(tag.id)
|
|
195
|
+
|
|
196
|
+
new_tag_associations = [TagAssociation(
|
|
197
|
+
tag_id=tag_id,
|
|
198
|
+
taggable_id=self.model.id,
|
|
199
|
+
taggable_type=self.model.__class__.__name__,
|
|
200
|
+
) for tag_id in tag_ids_to_use]
|
|
201
|
+
db_connection.session.bulk_save_objects(
|
|
202
|
+
new_tag_associations,
|
|
203
|
+
return_defaults=True,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
tag_associations_updated = []
|
|
207
|
+
for tag_name, new_tag_association in zip(tag_names_to_use, new_tag_associations):
|
|
208
|
+
taw = TagAssociationWithTag(
|
|
209
|
+
id=new_tag_association.id,
|
|
210
|
+
name=tag_name,
|
|
211
|
+
tag_id=new_tag_association.tag_id,
|
|
212
|
+
taggable_id=new_tag_association.taggable_id,
|
|
213
|
+
taggable_type=new_tag_association.taggable_type,
|
|
214
|
+
)
|
|
215
|
+
tag_associations_updated.append(taw)
|
|
216
|
+
|
|
217
|
+
self.tag_associations_updated = tag_associations_updated + tag_associations_to_keep
|
|
218
|
+
|
|
111
219
|
super().update(payload)
|
|
112
220
|
|
|
113
221
|
return self
|
|
222
|
+
|
|
223
|
+
def get_tag_associations(self):
|
|
224
|
+
if self.tag_associations_updated is None:
|
|
225
|
+
return self.tag_associations
|
|
226
|
+
else:
|
|
227
|
+
return self.tag_associations_updated
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def __load_tag_associations(resource):
|
|
231
|
+
pipeline_schedule_ids = [r.id for r in resource.result_set()]
|
|
232
|
+
result = (
|
|
233
|
+
TagAssociation.
|
|
234
|
+
select(
|
|
235
|
+
TagAssociation.id,
|
|
236
|
+
TagAssociation.tag_id,
|
|
237
|
+
TagAssociation.taggable_id,
|
|
238
|
+
TagAssociation.taggable_type,
|
|
239
|
+
Tag.name,
|
|
240
|
+
).
|
|
241
|
+
join(
|
|
242
|
+
Tag,
|
|
243
|
+
Tag.id == TagAssociation.tag_id,
|
|
244
|
+
).
|
|
245
|
+
filter(
|
|
246
|
+
TagAssociation.taggable_id.in_(pipeline_schedule_ids),
|
|
247
|
+
TagAssociation.taggable_type == resource.model.__class__.__name__,
|
|
248
|
+
).
|
|
249
|
+
all()
|
|
250
|
+
)
|
|
251
|
+
TagResource = getattr(
|
|
252
|
+
importlib.import_module('mage_ai.api.resources.TagResource'),
|
|
253
|
+
'TagResource',
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
return TagResource.build_result_set(result, resource.current_user)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def __select_tag_associations(resource, arr):
|
|
260
|
+
def _func(res):
|
|
261
|
+
return resource.id == res.taggable_id
|
|
262
|
+
return list(filter(_func, arr))
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
PipelineScheduleResource.register_collective_loader(
|
|
266
|
+
'tag_associations',
|
|
267
|
+
load=__load_tag_associations,
|
|
268
|
+
select=__select_tag_associations,
|
|
269
|
+
)
|
|
@@ -1,15 +1,28 @@
|
|
|
1
|
-
from mage_ai.api.resources.
|
|
1
|
+
from mage_ai.api.resources.DatabaseResource import DatabaseResource
|
|
2
2
|
from mage_ai.cache.tag import TagCache
|
|
3
|
+
from mage_ai.orchestration.db import safe_db_query
|
|
4
|
+
from mage_ai.orchestration.db.models.tags import Tag
|
|
5
|
+
from mage_ai.shared.array import unique_by
|
|
3
6
|
|
|
4
7
|
|
|
5
|
-
class TagResource(
|
|
8
|
+
class TagResource(DatabaseResource):
|
|
9
|
+
model_class = Tag
|
|
10
|
+
|
|
6
11
|
@classmethod
|
|
12
|
+
@safe_db_query
|
|
7
13
|
async def collection(self, query, meta, user, **kwargs):
|
|
8
14
|
cache = await TagCache.initialize_cache()
|
|
9
15
|
tag_uuids = cache.get_tags().keys()
|
|
16
|
+
tags = sorted(
|
|
17
|
+
unique_by(
|
|
18
|
+
list(Tag.query.all()) + [Tag(name=uuid) for uuid in tag_uuids],
|
|
19
|
+
lambda x: x.name,
|
|
20
|
+
),
|
|
21
|
+
key=lambda x: x.name,
|
|
22
|
+
)
|
|
10
23
|
|
|
11
24
|
return self.build_result_set(
|
|
12
|
-
|
|
25
|
+
tags,
|
|
13
26
|
user,
|
|
14
27
|
**kwargs,
|
|
15
28
|
)
|
|
@@ -434,6 +434,24 @@ class Git:
|
|
|
434
434
|
for url in remote.urls:
|
|
435
435
|
if url.lower().startswith('https'):
|
|
436
436
|
repository_names.append('/'.join(url.split('/')[-2:]).replace('.git', ''))
|
|
437
|
+
|
|
438
|
+
# Remove the token from the URL
|
|
439
|
+
# e.g. https://[user]:[token]@[netloc]
|
|
440
|
+
parts = url.split('@')
|
|
441
|
+
|
|
442
|
+
parts_arr = []
|
|
443
|
+
if len(parts) >= 2:
|
|
444
|
+
# https://[user]:[token]
|
|
445
|
+
parts2 = parts[0].split(':')
|
|
446
|
+
# ['https', '', 'user', 'token']
|
|
447
|
+
parts2[len(parts2) - 1] = '[token]'
|
|
448
|
+
parts_arr.append(':'.join(parts2))
|
|
449
|
+
parts_arr += parts[1:]
|
|
450
|
+
else:
|
|
451
|
+
parts_arr += parts
|
|
452
|
+
|
|
453
|
+
url = '@'.join(parts_arr)
|
|
454
|
+
|
|
437
455
|
urls.append(url)
|
|
438
456
|
except GitCommandError as err:
|
|
439
457
|
print('WARNING (mage_ai.data_preparation.git.remotes):')
|
|
@@ -17,6 +17,7 @@ import pandas as pd
|
|
|
17
17
|
import simplejson
|
|
18
18
|
from jinja2 import Template
|
|
19
19
|
|
|
20
|
+
import mage_ai.data_preparation.decorators
|
|
20
21
|
from mage_ai.cache.block import BlockCache
|
|
21
22
|
from mage_ai.data_cleaner.shared.utils import is_geo_dataframe, is_spark_dataframe
|
|
22
23
|
from mage_ai.data_preparation.logging.logger import DictLogger
|
|
@@ -70,6 +71,7 @@ from mage_ai.shared.utils import clean_name as clean_name_orig
|
|
|
70
71
|
from mage_ai.shared.utils import is_spark_env
|
|
71
72
|
|
|
72
73
|
PYTHON_COMMAND = 'python3'
|
|
74
|
+
BLOCK_EXISTS_ERROR = '[ERR_BLOCK_EXISTS]'
|
|
73
75
|
|
|
74
76
|
|
|
75
77
|
async def run_blocks(
|
|
@@ -497,6 +499,7 @@ class Block:
|
|
|
497
499
|
pipeline=None,
|
|
498
500
|
priority: int = None,
|
|
499
501
|
replicated_block: str = None,
|
|
502
|
+
require_unique_name: bool = False,
|
|
500
503
|
upstream_block_uuids: List[str] = None,
|
|
501
504
|
config: Dict = None,
|
|
502
505
|
widget: bool = False,
|
|
@@ -513,9 +516,14 @@ class Block:
|
|
|
513
516
|
uuid = clean_name(name)
|
|
514
517
|
language = language or BlockLanguage.PYTHON
|
|
515
518
|
|
|
516
|
-
# Don’t create a file if block is replicated from another block
|
|
519
|
+
# Don’t create a file if block is replicated from another block.
|
|
520
|
+
|
|
521
|
+
# Only create a file on the filesystem if the block type isn’t a global data product
|
|
522
|
+
# because global data products reference a data product which already has its
|
|
523
|
+
# own files.
|
|
517
524
|
if not replicated_block and \
|
|
518
|
-
(BlockType.DBT != block_type or BlockLanguage.YAML == language)
|
|
525
|
+
(BlockType.DBT != block_type or BlockLanguage.YAML == language) and \
|
|
526
|
+
BlockType.GLOBAL_DATA_PRODUCT != block_type:
|
|
519
527
|
|
|
520
528
|
block_directory = self.file_directory_name(block_type)
|
|
521
529
|
block_dir_path = os.path.join(repo_path, block_directory)
|
|
@@ -527,16 +535,20 @@ class Block:
|
|
|
527
535
|
file_extension = BLOCK_LANGUAGE_TO_FILE_EXTENSION[language]
|
|
528
536
|
file_path = os.path.join(block_dir_path, f'{uuid}.{file_extension}')
|
|
529
537
|
if os.path.exists(file_path):
|
|
530
|
-
if pipeline is not None and pipeline.has_block(
|
|
538
|
+
if (pipeline is not None and pipeline.has_block(
|
|
531
539
|
uuid,
|
|
532
540
|
block_type=block_type,
|
|
533
541
|
extension_uuid=extension_uuid,
|
|
534
|
-
):
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
542
|
+
)) or require_unique_name:
|
|
543
|
+
"""
|
|
544
|
+
The BLOCK_EXISTS_ERROR constant is used on the frontend to identify when
|
|
545
|
+
a user is trying to create a new block with an existing block name, and
|
|
546
|
+
link them to the existing block file so the user can choose to add the
|
|
547
|
+
existing block to their pipeline.
|
|
548
|
+
"""
|
|
549
|
+
raise Exception(f'{BLOCK_EXISTS_ERROR} Block {uuid} already exists. \
|
|
550
|
+
Please use a different name.')
|
|
551
|
+
else:
|
|
540
552
|
load_template(
|
|
541
553
|
block_type,
|
|
542
554
|
config,
|
|
@@ -1099,6 +1111,8 @@ class Block:
|
|
|
1099
1111
|
) -> List:
|
|
1100
1112
|
if logging_tags is None:
|
|
1101
1113
|
logging_tags = dict()
|
|
1114
|
+
if input_vars is None:
|
|
1115
|
+
input_vars = list()
|
|
1102
1116
|
|
|
1103
1117
|
decorated_functions = []
|
|
1104
1118
|
test_functions = []
|
|
@@ -1166,9 +1180,18 @@ class Block:
|
|
|
1166
1180
|
block_uuid = self.replicated_block
|
|
1167
1181
|
block_file_path = self.replicated_block_object.file_path
|
|
1168
1182
|
spec = importlib.util.spec_from_file_location(
|
|
1169
|
-
block_uuid,
|
|
1183
|
+
block_uuid,
|
|
1184
|
+
block_file_path,
|
|
1170
1185
|
)
|
|
1171
1186
|
module = importlib.util.module_from_spec(spec)
|
|
1187
|
+
# Set the decorators in the module in case they are not defined in the block
|
|
1188
|
+
# code
|
|
1189
|
+
setattr(
|
|
1190
|
+
module,
|
|
1191
|
+
self.type,
|
|
1192
|
+
getattr(mage_ai.data_preparation.decorators, self.type),
|
|
1193
|
+
)
|
|
1194
|
+
module.test = mage_ai.data_preparation.decorators.test
|
|
1172
1195
|
spec.loader.exec_module(module)
|
|
1173
1196
|
block_function_updated = getattr(module, block_function.__name__)
|
|
1174
1197
|
self.module = module
|
|
@@ -2261,7 +2284,7 @@ df = get_variable('{self.pipeline.uuid}', '{block_uuid}', 'df')
|
|
|
2261
2284
|
f'Block {new_uuid} already exists in pipeline. Please use a different name.'
|
|
2262
2285
|
)
|
|
2263
2286
|
|
|
2264
|
-
if not self.replicated_block:
|
|
2287
|
+
if not self.replicated_block and BlockType.GLOBAL_DATA_PRODUCT != self.type:
|
|
2265
2288
|
if os.path.exists(new_file_path):
|
|
2266
2289
|
raise Exception(f'Block {new_uuid} already exists. Please use a different name.')
|
|
2267
2290
|
|
|
@@ -94,6 +94,7 @@ def parse_attributes(block) -> Dict:
|
|
|
94
94
|
config = model_config(block.content)
|
|
95
95
|
if config.get('alias'):
|
|
96
96
|
table_name = config['alias']
|
|
97
|
+
database = config.get('database', None)
|
|
97
98
|
|
|
98
99
|
full_path = os.path.join(get_repo_path(), 'dbt', file_path)
|
|
99
100
|
|
|
@@ -131,6 +132,7 @@ def parse_attributes(block) -> Dict:
|
|
|
131
132
|
snapshot = first_folder_name and first_folder_name in snapshot_paths
|
|
132
133
|
|
|
133
134
|
return dict(
|
|
135
|
+
database=database,
|
|
134
136
|
dbt_project=dbt_project,
|
|
135
137
|
dbt_project_full_path=dbt_project_full_path,
|
|
136
138
|
file_extension=file_extension,
|
|
@@ -456,7 +458,11 @@ def get_profile(block, profile_target: str = None) -> Dict:
|
|
|
456
458
|
return load_profile(profile_name, profiles_full_path, profile_target)
|
|
457
459
|
|
|
458
460
|
|
|
459
|
-
def config_file_loader_and_configuration(
|
|
461
|
+
def config_file_loader_and_configuration(
|
|
462
|
+
block,
|
|
463
|
+
profile_target: str,
|
|
464
|
+
**kwargs,
|
|
465
|
+
) -> Dict:
|
|
460
466
|
profile = get_profile(block, profile_target)
|
|
461
467
|
|
|
462
468
|
if not profile:
|
|
@@ -492,12 +498,15 @@ def config_file_loader_and_configuration(block, profile_target: str) -> Dict:
|
|
|
492
498
|
)
|
|
493
499
|
elif DataSource.BIGQUERY == profile_type:
|
|
494
500
|
keyfile = profile.get('keyfile')
|
|
495
|
-
database = profile.get('project')
|
|
501
|
+
database = kwargs.get('database') or profile.get('project')
|
|
496
502
|
schema = profile.get('dataset')
|
|
497
503
|
|
|
498
|
-
|
|
504
|
+
config_file_loader_kwargs = dict(
|
|
499
505
|
GOOGLE_SERVICE_ACC_KEY_FILEPATH=keyfile,
|
|
500
|
-
)
|
|
506
|
+
)
|
|
507
|
+
if profile.get('location'):
|
|
508
|
+
config_file_loader_kwargs['GOOGLE_LOCATION'] = profile.get('location')
|
|
509
|
+
config_file_loader = ConfigFileLoader(config=config_file_loader_kwargs)
|
|
501
510
|
configuration = dict(
|
|
502
511
|
data_provider=profile_type,
|
|
503
512
|
data_provider_database=database,
|
|
@@ -573,11 +582,11 @@ def config_file_loader_and_configuration(block, profile_target: str) -> Dict:
|
|
|
573
582
|
SNOWFLAKE_ROLE=profile.get('role'),
|
|
574
583
|
)
|
|
575
584
|
|
|
576
|
-
if 'password'
|
|
585
|
+
if profile.get('password', None):
|
|
577
586
|
config['SNOWFLAKE_PASSWORD'] = profile['password']
|
|
578
|
-
if 'private_key_passphrase'
|
|
587
|
+
if profile.get('private_key_passphrase', None):
|
|
579
588
|
config['SNOWFLAKE_PRIVATE_KEY_PASSPHRASE'] = profile['private_key_passphrase']
|
|
580
|
-
if 'private_key_path'
|
|
589
|
+
if profile.get('private_key_path', None):
|
|
581
590
|
config['SNOWFLAKE_PRIVATE_KEY_PATH'] = profile['private_key_path']
|
|
582
591
|
|
|
583
592
|
config_file_loader = ConfigFileLoader(config=config)
|
|
@@ -927,10 +936,12 @@ def execute_query(
|
|
|
927
936
|
profile_target: str,
|
|
928
937
|
query_string: str,
|
|
929
938
|
limit: int = None,
|
|
939
|
+
database: str = None,
|
|
930
940
|
) -> DataFrame:
|
|
931
941
|
config_file_loader, configuration = config_file_loader_and_configuration(
|
|
932
942
|
block,
|
|
933
943
|
profile_target,
|
|
944
|
+
database=database,
|
|
934
945
|
)
|
|
935
946
|
|
|
936
947
|
data_provider = configuration['data_provider']
|
|
@@ -1304,12 +1315,15 @@ def fetch_model_data(
|
|
|
1304
1315
|
# If the model SQL file contains a config with schema, change the schema to use that.
|
|
1305
1316
|
# https://docs.getdbt.com/reference/resource-configs/schema
|
|
1306
1317
|
config = model_config(block.content)
|
|
1318
|
+
config_database = config.get('database')
|
|
1307
1319
|
config_schema = config.get('schema')
|
|
1320
|
+
|
|
1321
|
+
# settings from the dbt_project.yml
|
|
1322
|
+
model_configurations = get_model_configurations_from_dbt_project_settings(block)
|
|
1323
|
+
|
|
1308
1324
|
if config_schema:
|
|
1309
1325
|
schema = f'{schema}_{config_schema}'
|
|
1310
1326
|
else:
|
|
1311
|
-
# settings from the dbt_project.yml
|
|
1312
|
-
model_configurations = get_model_configurations_from_dbt_project_settings(block)
|
|
1313
1327
|
model_configuration_schema = None
|
|
1314
1328
|
if model_configurations:
|
|
1315
1329
|
model_configuration_schema = (model_configurations.get('schema') or
|
|
@@ -1318,9 +1332,22 @@ def fetch_model_data(
|
|
|
1318
1332
|
if model_configuration_schema:
|
|
1319
1333
|
schema = f"{schema}_{model_configuration_schema}"
|
|
1320
1334
|
|
|
1335
|
+
database = None
|
|
1336
|
+
if config_database:
|
|
1337
|
+
database = config_database
|
|
1338
|
+
elif model_configurations:
|
|
1339
|
+
database = (model_configurations.get('database') or
|
|
1340
|
+
model_configurations.get('+database'))
|
|
1341
|
+
|
|
1321
1342
|
query_string = f'SELECT * FROM {schema}.{table_name}'
|
|
1322
1343
|
|
|
1323
|
-
return execute_query(
|
|
1344
|
+
return execute_query(
|
|
1345
|
+
block,
|
|
1346
|
+
profile_target,
|
|
1347
|
+
query_string,
|
|
1348
|
+
limit,
|
|
1349
|
+
database=database,
|
|
1350
|
+
)
|
|
1324
1351
|
|
|
1325
1352
|
|
|
1326
1353
|
def upstream_blocks_from_sources(block: Block) -> List[Block]:
|
|
@@ -122,7 +122,7 @@ class IntegrationBlock(Block):
|
|
|
122
122
|
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
123
123
|
|
|
124
124
|
for line in proc.stdout:
|
|
125
|
-
f.write(
|
|
125
|
+
f.write(line.decode()),
|
|
126
126
|
print_log_from_line(
|
|
127
127
|
line,
|
|
128
128
|
config=config,
|
|
@@ -46,6 +46,44 @@ def execute_sql_code(
|
|
|
46
46
|
config_file_loader: Any = None,
|
|
47
47
|
configuration: Dict = None,
|
|
48
48
|
) -> List[Any]:
|
|
49
|
+
"""
|
|
50
|
+
Execute SQL code within the given block's data context.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
block (Block): The block containing the SQL execution context.
|
|
54
|
+
query (str): The SQL query to execute.
|
|
55
|
+
dynamic_block_index (int, optional): Index of the dynamic block, if applicable.
|
|
56
|
+
dynamic_upstream_block_uuids (List[str], optional): List of upstream block UUIDs for
|
|
57
|
+
dynamic execution.
|
|
58
|
+
execution_partition (str, optional): The partition for execution.
|
|
59
|
+
from_notebook (bool, optional): Indicates if execution is from a notebook.
|
|
60
|
+
global_vars (Dict, optional): Global variables to be used in the execution.
|
|
61
|
+
config_file_loader (Any, optional): Configuration file loader for data sources.
|
|
62
|
+
configuration (Dict, optional): Configuration settings for the block. If not provided, the
|
|
63
|
+
configuration from the block's context will be used. The configuration dictionary may
|
|
64
|
+
contain the following parameters:
|
|
65
|
+
|
|
66
|
+
- `use_raw_sql` (bool): If True, execute the query as raw SQL. Default is False.
|
|
67
|
+
- `data_provider` (str): The data provider for the execution, e.g., 'bigquery',
|
|
68
|
+
'clickhouse', etc.
|
|
69
|
+
- `data_provider_database` (str): The database name for the data provider.
|
|
70
|
+
- `data_provider_schema` (str): The schema name for the data provider.
|
|
71
|
+
- `export_write_policy` (str): The write policy for exporting data. Default is
|
|
72
|
+
ExportWritePolicy.APPEND.
|
|
73
|
+
- `limit` (int): The maximum number of rows to return in notebook.
|
|
74
|
+
Default is QUERY_ROW_LIMIT.
|
|
75
|
+
- `limit_in_pipeline_run` (int): Limit rows when running the block in the pipeline run.
|
|
76
|
+
Default is QUERY_ROW_LIMIT.
|
|
77
|
+
- Other provider-specific parameters may also be present.
|
|
78
|
+
Returns:
|
|
79
|
+
List[Any]: A list containing the query execution results.
|
|
80
|
+
|
|
81
|
+
Note:
|
|
82
|
+
This method executes the provided SQL query within the context of the given block.
|
|
83
|
+
It supports various data sources such as BigQuery, ClickHouse, Druid, MSSQL, MySQL,
|
|
84
|
+
PostgreSQL, Redshift, Snowflake, and Trino, applying relevant configurations and
|
|
85
|
+
returning the query execution results.
|
|
86
|
+
"""
|
|
49
87
|
configuration = configuration if configuration else block.configuration
|
|
50
88
|
use_raw_sql = configuration.get('use_raw_sql')
|
|
51
89
|
|
|
@@ -68,10 +106,12 @@ def execute_sql_code(
|
|
|
68
106
|
should_query = block.type in PREVIEWABLE_BLOCK_TYPES
|
|
69
107
|
|
|
70
108
|
limit = int(configuration.get('limit') or QUERY_ROW_LIMIT)
|
|
109
|
+
# Limit rows when running the block in the pipeline run
|
|
110
|
+
limit_in_pipeline_run = int(configuration.get('limit_in_pipeline_run') or QUERY_ROW_LIMIT)
|
|
71
111
|
if from_notebook:
|
|
72
112
|
limit = min(limit, QUERY_ROW_LIMIT)
|
|
73
113
|
else:
|
|
74
|
-
limit = QUERY_ROW_LIMIT
|
|
114
|
+
limit = min(limit_in_pipeline_run, QUERY_ROW_LIMIT)
|
|
75
115
|
|
|
76
116
|
create_upstream_block_tables_kwargs = dict(
|
|
77
117
|
configuration=configuration,
|
|
@@ -280,7 +320,9 @@ def execute_sql_code(
|
|
|
280
320
|
if should_query:
|
|
281
321
|
return [
|
|
282
322
|
loader.load(
|
|
283
|
-
|
|
323
|
+
# Add the limit directly in the SELECT statement
|
|
324
|
+
# since io.mssql doesn't support enforcing limit
|
|
325
|
+
f'SELECT TOP {limit} * FROM {table_name}',
|
|
284
326
|
limit=limit,
|
|
285
327
|
verbose=False,
|
|
286
328
|
),
|
|
@@ -56,13 +56,23 @@ def group_and_hydrate_files(
|
|
|
56
56
|
file_dicts: List[Dict],
|
|
57
57
|
custom_template_class,
|
|
58
58
|
) -> List:
|
|
59
|
-
|
|
59
|
+
def _func(x):
|
|
60
|
+
arr = ['']
|
|
60
61
|
|
|
61
|
-
|
|
62
|
+
if x:
|
|
63
|
+
parent_names = x.get('parent_names', []) or []
|
|
64
|
+
if parent_names and len(parent_names) >= 1:
|
|
65
|
+
arr = [str(parent_name) for parent_name in parent_names]
|
|
66
|
+
|
|
67
|
+
return os.path.join(*arr)
|
|
68
|
+
|
|
69
|
+
groups = group_by(_func, file_dicts)
|
|
70
|
+
|
|
71
|
+
custom_templates = []
|
|
62
72
|
|
|
63
73
|
for template_uuid, _ in groups.items():
|
|
64
74
|
custom_template = custom_template_class.load(template_uuid=template_uuid)
|
|
65
75
|
if custom_template:
|
|
66
|
-
|
|
76
|
+
custom_templates.append(custom_template)
|
|
67
77
|
|
|
68
|
-
return
|
|
78
|
+
return custom_templates
|
|
@@ -133,6 +133,8 @@ class GlobalDataProduct:
|
|
|
133
133
|
else:
|
|
134
134
|
return d
|
|
135
135
|
|
|
136
|
+
return None
|
|
137
|
+
|
|
136
138
|
def is_outdated_after(self, now: datetime = None, return_values: bool = False) -> bool:
|
|
137
139
|
values = {}
|
|
138
140
|
outdated_starting_at = self.outdated_starting_at or {}
|
|
@@ -174,6 +176,10 @@ class GlobalDataProduct:
|
|
|
174
176
|
def next_run_at(self, pipeline_run: 'PipelineRun') -> datetime:
|
|
175
177
|
execution_date = pipeline_run.execution_date
|
|
176
178
|
outdated_at_delta = self.get_outdated_at_delta()
|
|
179
|
+
|
|
180
|
+
if not outdated_at_delta:
|
|
181
|
+
return None
|
|
182
|
+
|
|
177
183
|
if execution_date and outdated_at_delta:
|
|
178
184
|
execution_date += outdated_at_delta
|
|
179
185
|
|
|
@@ -189,6 +195,8 @@ class GlobalDataProduct:
|
|
|
189
195
|
now = datetime.utcnow().replace(tzinfo=timezone.utc)
|
|
190
196
|
|
|
191
197
|
execution_date = self.next_run_at(pipeline_run)
|
|
198
|
+
if not execution_date:
|
|
199
|
+
return [False, False]
|
|
192
200
|
|
|
193
201
|
outdated = execution_date and now >= execution_date
|
|
194
202
|
|
|
@@ -885,6 +885,10 @@ class Pipeline:
|
|
|
885
885
|
if block_data.get('has_callback') is not None:
|
|
886
886
|
block.update(extract(block_data, ['has_callback']))
|
|
887
887
|
|
|
888
|
+
color = block_data.get('color')
|
|
889
|
+
if color is not None and color != block.color:
|
|
890
|
+
block.update(extract(block_data, ['color']))
|
|
891
|
+
|
|
888
892
|
configuration = block_data.get('configuration')
|
|
889
893
|
if configuration:
|
|
890
894
|
if configuration.get('dynamic') and not is_dynamic_block(block):
|
|
@@ -430,11 +430,14 @@ class Variable:
|
|
|
430
430
|
df_output = data.copy()
|
|
431
431
|
# Clean up data types since parquet doesn't support mixed data types
|
|
432
432
|
for c in df_output.columns:
|
|
433
|
-
|
|
433
|
+
df_col = df_output[c]
|
|
434
|
+
if type(df_col) is pd.DataFrame:
|
|
435
|
+
raise Exception(f'Please do not use duplicate column name: "{c}"')
|
|
436
|
+
c_dtype = df_col.dtype
|
|
434
437
|
if not is_object_dtype(c_dtype):
|
|
435
438
|
column_types[c] = str(c_dtype)
|
|
436
439
|
else:
|
|
437
|
-
series_non_null =
|
|
440
|
+
series_non_null = df_col.dropna()
|
|
438
441
|
if len(series_non_null) > 0:
|
|
439
442
|
coltype = type(series_non_null.iloc[0])
|
|
440
443
|
if is_object_dtype(series_non_null.dtype):
|