mage-ai 0.9.69__py3-none-any.whl → 0.9.70__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mage-ai might be problematic. Click here for more details.
- mage_ai/api/policies/BackfillPolicy.py +1 -0
- mage_ai/api/policies/PipelinePolicy.py +1 -0
- mage_ai/api/policies/WorkspacePolicy.py +1 -0
- mage_ai/api/presenters/BackfillPresenter.py +1 -0
- mage_ai/api/resources/GitBranchResource.py +56 -23
- mage_ai/api/resources/GitCustomBranchResource.py +29 -1
- mage_ai/api/resources/OauthResource.py +1 -1
- mage_ai/api/resources/PipelineResource.py +11 -5
- mage_ai/api/resources/PipelineRunResource.py +41 -4
- mage_ai/api/resources/PipelineScheduleResource.py +4 -0
- mage_ai/api/resources/PullRequestResource.py +6 -4
- mage_ai/api/resources/WorkspaceResource.py +5 -4
- mage_ai/cache/block_action_object/__init__.py +1 -1
- mage_ai/cluster_manager/kubernetes/workload_manager.py +52 -1
- mage_ai/cluster_manager/workspace/base.py +6 -0
- mage_ai/cluster_manager/workspace/kubernetes.py +22 -1
- mage_ai/command_center/applications/utils.py +2 -2
- mage_ai/command_center/presenters/text.py +1 -1
- mage_ai/data_preparation/executors/k8s_block_executor.py +30 -7
- mage_ai/data_preparation/executors/k8s_pipeline_executor.py +30 -7
- mage_ai/data_preparation/executors/streaming_pipeline_executor.py +1 -1
- mage_ai/data_preparation/git/__init__.py +50 -22
- mage_ai/data_preparation/git/api.py +62 -7
- mage_ai/data_preparation/git/utils.py +45 -21
- mage_ai/data_preparation/models/block/__init__.py +24 -5
- mage_ai/data_preparation/models/block/dynamic/utils.py +9 -4
- mage_ai/data_preparation/models/block/global_data_product/__init__.py +25 -2
- mage_ai/data_preparation/models/block/remote/__init__.py +0 -0
- mage_ai/data_preparation/models/block/remote/models.py +58 -0
- mage_ai/data_preparation/models/block/utils.py +38 -0
- mage_ai/data_preparation/models/constants.py +2 -0
- mage_ai/data_preparation/models/global_data_product/__init__.py +12 -0
- mage_ai/data_preparation/models/pipeline.py +9 -5
- mage_ai/data_preparation/models/triggers/__init__.py +4 -2
- mage_ai/data_preparation/storage/local_storage.py +12 -6
- mage_ai/orchestration/db/migrations/versions/42a14d6143f1_update_token_column_type.py +54 -0
- mage_ai/orchestration/db/models/oauth.py +10 -9
- mage_ai/orchestration/db/models/schedules.py +21 -0
- mage_ai/orchestration/notification/sender.py +37 -15
- mage_ai/orchestration/pipeline_scheduler_original.py +32 -25
- mage_ai/orchestration/triggers/api.py +29 -1
- mage_ai/orchestration/triggers/global_data_product.py +9 -4
- mage_ai/orchestration/triggers/utils.py +10 -1
- mage_ai/orchestration/utils/resources.py +3 -0
- mage_ai/server/api/downloads.py +4 -1
- mage_ai/server/api/runs.py +151 -0
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +6 -6
- mage_ai/server/frontend_dist/_next/static/{_krrrgup_C-dPOpX36S8I → RhDiJSkcjCsh4xxX4BFBk}/_buildManifest.js +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1557-b3502f3f1aa92ac7.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7966-b9b85ba10667e654.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7181b086c93784d2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/triggers-9cba3211434a8966.js → frontend_dist/_next/static/chunks/pages/triggers-a599c6ac89be8c8d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +1 -0
- mage_ai/server/frontend_dist/block-layout.html +2 -2
- mage_ai/server/frontend_dist/compute.html +2 -2
- mage_ai/server/frontend_dist/files.html +2 -2
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +2 -2
- mage_ai/server/frontend_dist/global-data-products.html +2 -2
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist/global-hooks.html +2 -2
- mage_ai/server/frontend_dist/index.html +2 -2
- mage_ai/server/frontend_dist/manage/files.html +2 -2
- mage_ai/server/frontend_dist/manage/settings.html +2 -2
- mage_ai/server/frontend_dist/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist/manage/users.html +2 -2
- mage_ai/server/frontend_dist/manage.html +2 -2
- mage_ai/server/frontend_dist/oauth.html +2 -2
- mage_ai/server/frontend_dist/overview.html +2 -2
- mage_ai/server/frontend_dist/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist/pipelines.html +2 -2
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist/platform/global-hooks.html +2 -2
- mage_ai/server/frontend_dist/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist/settings/platform/preferences.html +2 -2
- mage_ai/server/frontend_dist/settings/platform/settings.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/roles.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist/settings.html +2 -2
- mage_ai/server/frontend_dist/sign-in.html +6 -6
- mage_ai/server/frontend_dist/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist/templates.html +2 -2
- mage_ai/server/frontend_dist/terminal.html +2 -2
- mage_ai/server/frontend_dist/test.html +2 -2
- mage_ai/server/frontend_dist/triggers.html +2 -2
- mage_ai/server/frontend_dist/version-control.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/404.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/_next/static/{KLL5mirre9d7_ZeEpaw3s → TdpLLFome13qvM0gXvpHs}/_buildManifest.js +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-b3502f3f1aa92ac7.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-b9b85ba10667e654.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7181b086c93784d2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/triggers-9cba3211434a8966.js → frontend_dist_base_path_template/_next/static/chunks/pages/triggers-a599c6ac89be8c8d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/compute.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/files.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/index.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/oauth.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/overview.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/templates.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/terminal.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/test.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/triggers.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/version-control.html +2 -2
- mage_ai/server/scheduler_manager.py +3 -1
- mage_ai/server/server.py +35 -12
- mage_ai/server/utils/output_display.py +2 -2
- mage_ai/services/aws/ecs/ecs.py +1 -0
- mage_ai/services/k8s/config.py +4 -4
- mage_ai/services/k8s/utils.py +97 -0
- mage_ai/shared/parsers.py +6 -1
- mage_ai/tests/api/operations/base/mixins.py +1 -1
- mage_ai/tests/api/resources/test_pipeline_resource.py +2 -2
- mage_ai/tests/authentication/oauth/test_utils.py +1 -1
- mage_ai/tests/data_preparation/models/block/test_global_data_product.py +2 -0
- mage_ai/tests/orchestration/triggers/test_global_data_product.py +138 -136
- mage_ai/tests/server/test_server.py +19 -0
- mage_ai/tests/services/k8s/test_job_manager.py +9 -6
- mage_ai/version_control/branch/utils.py +2 -1
- mage_ai/version_control/models.py +3 -2
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.70.dist-info}/METADATA +2 -2
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.70.dist-info}/RECORD +217 -212
- mage_ai/server/frontend_dist/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
- /mage_ai/server/frontend_dist/_next/static/{_krrrgup_C-dPOpX36S8I → RhDiJSkcjCsh4xxX4BFBk}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{KLL5mirre9d7_ZeEpaw3s → TdpLLFome13qvM0gXvpHs}/_ssgManifest.js +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.70.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.70.dist-info}/WHEEL +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.70.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.70.dist-info}/top_level.txt +0 -0
|
@@ -49,6 +49,7 @@ class BackfillPresenter(BasePresenter):
|
|
|
49
49
|
):
|
|
50
50
|
pipeline_run_dates = preview_run_dates(self.model)
|
|
51
51
|
data['total_run_count'] = len(pipeline_run_dates)
|
|
52
|
+
data['run_status_counts'] = self.model.pipeline_run_status_counts
|
|
52
53
|
if include_preview_runs:
|
|
53
54
|
start_idx = offset
|
|
54
55
|
end_idx = start_idx + limit
|
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
import os
|
|
2
|
-
from typing import Dict, List
|
|
2
|
+
from typing import Dict, List, Tuple
|
|
3
3
|
|
|
4
4
|
from mage_ai.api.errors import ApiError
|
|
5
5
|
from mage_ai.api.resources.GenericResource import GenericResource
|
|
6
6
|
from mage_ai.authentication.oauth.constants import ProviderName
|
|
7
7
|
from mage_ai.data_preparation.git import REMOTE_NAME, Git, api
|
|
8
8
|
from mage_ai.data_preparation.git.clients.base import Client as GitClient
|
|
9
|
-
from mage_ai.data_preparation.git.utils import
|
|
9
|
+
from mage_ai.data_preparation.git.utils import (
|
|
10
|
+
get_oauth_access_token_for_user,
|
|
11
|
+
get_provider_from_remote_url,
|
|
12
|
+
)
|
|
10
13
|
from mage_ai.data_preparation.preferences import get_preferences
|
|
14
|
+
from mage_ai.server.logger import Logger
|
|
11
15
|
from mage_ai.shared.path_fixer import remove_base_repo_path
|
|
12
16
|
from mage_ai.shared.strings import capitalize_remove_underscore_lower
|
|
13
17
|
|
|
18
|
+
logger = Logger().new_server_logger(__name__)
|
|
19
|
+
|
|
14
20
|
|
|
15
21
|
def build_file_object(obj):
|
|
16
22
|
arr = []
|
|
@@ -53,7 +59,7 @@ class GitBranchResource(GenericResource):
|
|
|
53
59
|
if remote_url:
|
|
54
60
|
provider = get_provider_from_remote_url(remote_url)
|
|
55
61
|
|
|
56
|
-
access_token =
|
|
62
|
+
access_token = get_oauth_access_token_for_user(user, provider=provider)
|
|
57
63
|
if access_token:
|
|
58
64
|
branches = GitClient.get_client_for_provider(provider)(
|
|
59
65
|
access_token.token
|
|
@@ -64,7 +70,15 @@ class GitBranchResource(GenericResource):
|
|
|
64
70
|
arr += [dict(name=branch) for branch in git_manager.branches]
|
|
65
71
|
|
|
66
72
|
if include_remote_branches:
|
|
67
|
-
|
|
73
|
+
try:
|
|
74
|
+
git_manager.fetch()
|
|
75
|
+
mage_remote = git_manager.origin
|
|
76
|
+
arr += [
|
|
77
|
+
dict(name=ref.name)
|
|
78
|
+
for ref in mage_remote.refs
|
|
79
|
+
]
|
|
80
|
+
except Exception:
|
|
81
|
+
logger.warning('Failed to fetch remote branches')
|
|
68
82
|
|
|
69
83
|
return self.build_result_set(
|
|
70
84
|
arr,
|
|
@@ -123,32 +137,27 @@ class GitBranchResource(GenericResource):
|
|
|
123
137
|
**kwargs,
|
|
124
138
|
)
|
|
125
139
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
files = payload.get('files', None)
|
|
136
|
-
message = payload.get('message', None)
|
|
137
|
-
|
|
140
|
+
@classmethod
|
|
141
|
+
def get_oauth_config(
|
|
142
|
+
cls,
|
|
143
|
+
remote_url: str = None,
|
|
144
|
+
remote_name: str = None,
|
|
145
|
+
user=None,
|
|
146
|
+
) -> Tuple[str, str, str, Dict]:
|
|
147
|
+
git_manager = cls.get_git_manager(user=user)
|
|
138
148
|
url = None
|
|
139
|
-
remote_url = query.get('remote_url', None)
|
|
140
149
|
if remote_url:
|
|
141
|
-
url = remote_url
|
|
142
|
-
elif
|
|
143
|
-
remote = git_manager.repo.remotes[
|
|
150
|
+
url = remote_url
|
|
151
|
+
elif remote_name:
|
|
152
|
+
remote = git_manager.repo.remotes[remote_name]
|
|
144
153
|
url = list(remote.urls)[0]
|
|
145
154
|
|
|
146
155
|
provider = ProviderName.GITHUB
|
|
147
156
|
if url:
|
|
148
157
|
provider = get_provider_from_remote_url(url)
|
|
149
158
|
|
|
150
|
-
access_token =
|
|
151
|
-
|
|
159
|
+
access_token = get_oauth_access_token_for_user(
|
|
160
|
+
user, provider=provider
|
|
152
161
|
)
|
|
153
162
|
http_access_token = git_manager.get_access_token()
|
|
154
163
|
|
|
@@ -159,7 +168,7 @@ class GitBranchResource(GenericResource):
|
|
|
159
168
|
user_from_api = api.get_user(token, provider=provider)
|
|
160
169
|
# Default to mage user email if no email is returned from API
|
|
161
170
|
email = user_from_api.get(
|
|
162
|
-
'email',
|
|
171
|
+
'email', user.email if user else None
|
|
163
172
|
)
|
|
164
173
|
config_overwrite = dict(
|
|
165
174
|
username=user_from_api.get('username'),
|
|
@@ -168,6 +177,30 @@ class GitBranchResource(GenericResource):
|
|
|
168
177
|
elif http_access_token:
|
|
169
178
|
token = http_access_token
|
|
170
179
|
|
|
180
|
+
return token, provider, url, config_overwrite
|
|
181
|
+
|
|
182
|
+
async def update(self, payload, **kwargs):
|
|
183
|
+
query = kwargs.get('query') or {}
|
|
184
|
+
|
|
185
|
+
git_manager = self.get_git_manager(user=self.current_user)
|
|
186
|
+
action_type = payload.get('action_type')
|
|
187
|
+
action_payload = payload.get('action_payload', dict())
|
|
188
|
+
action_remote = action_payload.get('remote', None)
|
|
189
|
+
action_branch = action_payload.get('branch', None)
|
|
190
|
+
|
|
191
|
+
files = payload.get('files', None)
|
|
192
|
+
message = payload.get('message', None)
|
|
193
|
+
|
|
194
|
+
remote_url = query.get('remote_url', None)
|
|
195
|
+
if remote_url:
|
|
196
|
+
remote_url = remote_url[0]
|
|
197
|
+
|
|
198
|
+
token, provider, url, config_overwrite = self.get_oauth_config(
|
|
199
|
+
remote_url=remote_url,
|
|
200
|
+
remote_name=action_remote,
|
|
201
|
+
user=self.current_user,
|
|
202
|
+
)
|
|
203
|
+
|
|
171
204
|
# Recreate git manager with updated config
|
|
172
205
|
git_manager = self.get_git_manager(
|
|
173
206
|
user=self.current_user, config_overwrite=config_overwrite
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from typing import Dict, List
|
|
2
2
|
|
|
3
3
|
from mage_ai.api.resources.GitBranchResource import GitBranchResource
|
|
4
|
-
from mage_ai.data_preparation.git import REMOTE_NAME, Git
|
|
4
|
+
from mage_ai.data_preparation.git import REMOTE_NAME, Git, api
|
|
5
5
|
from mage_ai.data_preparation.sync import AuthType
|
|
6
6
|
|
|
7
7
|
|
|
@@ -17,6 +17,34 @@ class GitCustomBranchResource(GitBranchResource):
|
|
|
17
17
|
user=user,
|
|
18
18
|
)
|
|
19
19
|
|
|
20
|
+
@classmethod
|
|
21
|
+
def create(cls, payload, user, **kwargs):
|
|
22
|
+
branch = payload.get('name')
|
|
23
|
+
remote = payload.get('remote')
|
|
24
|
+
|
|
25
|
+
token, _, url, config_overwrite = cls.get_oauth_config(
|
|
26
|
+
remote_name=remote,
|
|
27
|
+
user=user,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
git_manager = cls.get_git_manager(
|
|
31
|
+
user=user, config_overwrite=config_overwrite
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
if remote:
|
|
35
|
+
api.switch_branch(
|
|
36
|
+
remote,
|
|
37
|
+
url,
|
|
38
|
+
branch,
|
|
39
|
+
token,
|
|
40
|
+
config_overwrite=config_overwrite,
|
|
41
|
+
user=user,
|
|
42
|
+
)
|
|
43
|
+
else:
|
|
44
|
+
git_manager.switch_branch(branch)
|
|
45
|
+
|
|
46
|
+
return cls(dict(name=git_manager.current_branch), user, **kwargs)
|
|
47
|
+
|
|
20
48
|
@classmethod
|
|
21
49
|
async def member(self, pk, user, **kwargs):
|
|
22
50
|
resource = await GitBranchResource.member(pk, user, **kwargs)
|
|
@@ -20,7 +20,7 @@ from mage_ai.authentication.oauth.utils import (
|
|
|
20
20
|
refresh_token_for_client,
|
|
21
21
|
)
|
|
22
22
|
from mage_ai.authentication.providers.constants import NAME_TO_PROVIDER
|
|
23
|
-
from mage_ai.data_preparation.git.
|
|
23
|
+
from mage_ai.data_preparation.git.utils import get_oauth_client_id
|
|
24
24
|
from mage_ai.orchestration.db import safe_db_query
|
|
25
25
|
from mage_ai.orchestration.db.models.oauth import Oauth2AccessToken, Oauth2Application
|
|
26
26
|
|
|
@@ -119,6 +119,12 @@ class PipelineResource(BaseResource):
|
|
|
119
119
|
if include_schedules:
|
|
120
120
|
include_schedules = include_schedules[0]
|
|
121
121
|
|
|
122
|
+
repo_path = query.get('repo_path', [None])
|
|
123
|
+
if repo_path:
|
|
124
|
+
repo_path = repo_path[0]
|
|
125
|
+
if not repo_path:
|
|
126
|
+
repo_path = get_repo_path(root_project=False)
|
|
127
|
+
|
|
122
128
|
search_query = query.get('search', [None])
|
|
123
129
|
if search_query:
|
|
124
130
|
search_query = search_query[0]
|
|
@@ -174,11 +180,11 @@ class PipelineResource(BaseResource):
|
|
|
174
180
|
|
|
175
181
|
if NO_TAGS_QUERY in tags:
|
|
176
182
|
pipeline_uuids_with_tags = set(cache.get_all_pipeline_uuids_with_tags())
|
|
177
|
-
all_pipeline_uuids = set(Pipeline.get_all_pipelines(
|
|
183
|
+
all_pipeline_uuids = set(Pipeline.get_all_pipelines(repo_path))
|
|
178
184
|
pipeline_uuids_without_tags = list(all_pipeline_uuids - pipeline_uuids_with_tags)
|
|
179
185
|
pipeline_uuids = pipeline_uuids + pipeline_uuids_without_tags
|
|
180
186
|
else:
|
|
181
|
-
pipeline_uuids = Pipeline.get_all_pipelines(
|
|
187
|
+
pipeline_uuids = Pipeline.get_all_pipelines(repo_path)
|
|
182
188
|
|
|
183
189
|
if search_query:
|
|
184
190
|
pipeline_uuids = list(filter(
|
|
@@ -206,7 +212,7 @@ class PipelineResource(BaseResource):
|
|
|
206
212
|
|
|
207
213
|
async def get_pipeline(uuid: str) -> Pipeline:
|
|
208
214
|
try:
|
|
209
|
-
return await Pipeline.get_async(uuid)
|
|
215
|
+
return await Pipeline.get_async(uuid, repo_path=repo_path)
|
|
210
216
|
except Exception as err:
|
|
211
217
|
err_message = f'Error loading pipeline {uuid}: {err}.'
|
|
212
218
|
if err.__class__.__name__ == 'OSError' and 'Too many open files' in err.strerror:
|
|
@@ -217,7 +223,7 @@ class PipelineResource(BaseResource):
|
|
|
217
223
|
|
|
218
224
|
def get_pipeline_with_config(uuid, config: Dict) -> Pipeline:
|
|
219
225
|
try:
|
|
220
|
-
return Pipeline(uuid, config=config)
|
|
226
|
+
return Pipeline(uuid, config=config, repo_path=repo_path)
|
|
221
227
|
except Exception as err:
|
|
222
228
|
err_message = f'Error loading pipeline sync {uuid}: {err}.'
|
|
223
229
|
if err.__class__.__name__ == 'OSError' and 'Too many open files' in err.strerror:
|
|
@@ -239,7 +245,7 @@ class PipelineResource(BaseResource):
|
|
|
239
245
|
))
|
|
240
246
|
else:
|
|
241
247
|
for uuid in pipeline_uuids:
|
|
242
|
-
pipeline_dict = cache.get_model(dict(uuid=uuid))
|
|
248
|
+
pipeline_dict = cache.get_model(dict(uuid=uuid, repo_path=repo_path))
|
|
243
249
|
if pipeline_dict and pipeline_dict.get('pipeline'):
|
|
244
250
|
pipeline = get_pipeline_with_config(uuid, pipeline_dict['pipeline'])
|
|
245
251
|
if pipeline:
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from sqlalchemy import and_, desc, func
|
|
1
2
|
from sqlalchemy.orm import selectinload
|
|
2
3
|
|
|
3
4
|
from mage_ai.api.operations.constants import META_KEY_LIMIT, META_KEY_OFFSET
|
|
@@ -5,7 +6,10 @@ from mage_ai.api.resources.DatabaseResource import DatabaseResource
|
|
|
5
6
|
from mage_ai.api.utils import get_query_timestamps
|
|
6
7
|
from mage_ai.cache.tag import TagCache
|
|
7
8
|
from mage_ai.data_preparation.models.block.utils import get_all_descendants
|
|
8
|
-
from mage_ai.data_preparation.models.constants import
|
|
9
|
+
from mage_ai.data_preparation.models.constants import (
|
|
10
|
+
PIPELINE_RUN_STATUS_LAST_RUN_FAILED,
|
|
11
|
+
PipelineType,
|
|
12
|
+
)
|
|
9
13
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
10
14
|
from mage_ai.data_preparation.models.triggers import (
|
|
11
15
|
ScheduleInterval,
|
|
@@ -94,9 +98,38 @@ class PipelineRunResource(DatabaseResource):
|
|
|
94
98
|
|
|
95
99
|
repo_pipeline_schedule_ids = [s.id for s in PipelineSchedule.repo_query]
|
|
96
100
|
|
|
101
|
+
if status == PIPELINE_RUN_STATUS_LAST_RUN_FAILED:
|
|
102
|
+
"""
|
|
103
|
+
The initial query below is used to get the last pipeline run retry
|
|
104
|
+
(or individual pipeline run if there are no retries) for each
|
|
105
|
+
grouping of pipeline runs with the same execution_date,
|
|
106
|
+
pipeline_uuid, and pipeline_schedule_id.
|
|
107
|
+
"""
|
|
108
|
+
latest_pipeline_runs = PipelineRun.select(
|
|
109
|
+
PipelineRun.id,
|
|
110
|
+
func.row_number()
|
|
111
|
+
.over(
|
|
112
|
+
partition_by=(
|
|
113
|
+
PipelineRun.execution_date,
|
|
114
|
+
PipelineRun.pipeline_schedule_id,
|
|
115
|
+
PipelineRun.pipeline_uuid,
|
|
116
|
+
),
|
|
117
|
+
order_by=desc(PipelineRun.id))
|
|
118
|
+
.label('row_number')
|
|
119
|
+
).cte(name='latest_pipeline_runs')
|
|
120
|
+
query = (PipelineRun.select(
|
|
121
|
+
PipelineRun,
|
|
122
|
+
)
|
|
123
|
+
.join(latest_pipeline_runs, and_(
|
|
124
|
+
PipelineRun.id == latest_pipeline_runs.c.id,
|
|
125
|
+
latest_pipeline_runs.c.row_number == 1,
|
|
126
|
+
))
|
|
127
|
+
)
|
|
128
|
+
else:
|
|
129
|
+
query = PipelineRun.query
|
|
130
|
+
|
|
97
131
|
results = (
|
|
98
|
-
|
|
99
|
-
.query
|
|
132
|
+
query
|
|
100
133
|
.filter(PipelineRun.pipeline_schedule_id.in_(repo_pipeline_schedule_ids))
|
|
101
134
|
.options(selectinload(PipelineRun.block_runs))
|
|
102
135
|
.options(selectinload(PipelineRun.pipeline_schedule))
|
|
@@ -120,8 +153,12 @@ class PipelineRunResource(DatabaseResource):
|
|
|
120
153
|
results = results.filter(PipelineRun.pipeline_uuid.in_(pipeline_uuids))
|
|
121
154
|
if pipeline_uuids_with_tags:
|
|
122
155
|
results = results.filter(PipelineRun.pipeline_uuid.in_(pipeline_uuids_with_tags))
|
|
123
|
-
|
|
156
|
+
|
|
157
|
+
if status == PIPELINE_RUN_STATUS_LAST_RUN_FAILED:
|
|
158
|
+
results = results.filter(PipelineRun.status == PipelineRun.PipelineRunStatus.FAILED)
|
|
159
|
+
elif status is not None:
|
|
124
160
|
results = results.filter(PipelineRun.status == status)
|
|
161
|
+
|
|
125
162
|
if statuses:
|
|
126
163
|
results = results.filter(PipelineRun.status.in_(statuses))
|
|
127
164
|
if start_timestamp is not None:
|
|
@@ -403,6 +403,8 @@ class PipelineScheduleResource(DatabaseResource):
|
|
|
403
403
|
if updated_status == ScheduleStatus.ACTIVE and self.model.status == ScheduleStatus.INACTIVE:
|
|
404
404
|
payload['last_enabled_at'] = datetime.now(tz=pytz.UTC)
|
|
405
405
|
|
|
406
|
+
old_name = self.model.name
|
|
407
|
+
|
|
406
408
|
resource = super().update(payload)
|
|
407
409
|
updated_model = resource.model
|
|
408
410
|
|
|
@@ -424,11 +426,13 @@ class PipelineScheduleResource(DatabaseResource):
|
|
|
424
426
|
update_only_if_exists = (
|
|
425
427
|
not pipeline.should_save_trigger_in_code_automatically()
|
|
426
428
|
)
|
|
429
|
+
old_trigger_name = old_name if old_name != updated_model.name else None
|
|
427
430
|
|
|
428
431
|
add_or_update_trigger_for_pipeline_and_persist(
|
|
429
432
|
trigger,
|
|
430
433
|
pipeline.uuid,
|
|
431
434
|
update_only_if_exists=update_only_if_exists,
|
|
435
|
+
old_trigger_name=old_trigger_name,
|
|
432
436
|
)
|
|
433
437
|
|
|
434
438
|
return self
|
|
@@ -3,9 +3,11 @@ from typing import Dict
|
|
|
3
3
|
from mage_ai.api.errors import ApiError
|
|
4
4
|
from mage_ai.api.resources.GenericResource import GenericResource
|
|
5
5
|
from mage_ai.authentication.oauth.constants import ProviderName
|
|
6
|
-
from mage_ai.data_preparation.git import api
|
|
7
6
|
from mage_ai.data_preparation.git.clients.base import Client as GitClient
|
|
8
|
-
from mage_ai.data_preparation.git.utils import
|
|
7
|
+
from mage_ai.data_preparation.git.utils import (
|
|
8
|
+
get_oauth_access_token_for_user,
|
|
9
|
+
get_provider_from_remote_url,
|
|
10
|
+
)
|
|
9
11
|
|
|
10
12
|
|
|
11
13
|
def pull_request_to_dict(pr) -> Dict:
|
|
@@ -40,7 +42,7 @@ class PullRequestResource(GenericResource):
|
|
|
40
42
|
if remote_url:
|
|
41
43
|
provider = get_provider_from_remote_url(remote_url)
|
|
42
44
|
|
|
43
|
-
access_token =
|
|
45
|
+
access_token = get_oauth_access_token_for_user(user, provider=provider)
|
|
44
46
|
|
|
45
47
|
if access_token:
|
|
46
48
|
arr = GitClient.get_client_for_provider(provider)(
|
|
@@ -84,7 +86,7 @@ class PullRequestResource(GenericResource):
|
|
|
84
86
|
remote_url = payload.get('remote_url')
|
|
85
87
|
provider = get_provider_from_remote_url(remote_url)
|
|
86
88
|
|
|
87
|
-
access_token =
|
|
89
|
+
access_token = get_oauth_access_token_for_user(user, provider=provider)
|
|
88
90
|
if not access_token:
|
|
89
91
|
error.update(
|
|
90
92
|
dict(
|
|
@@ -123,14 +123,15 @@ class WorkspaceResource(GenericResource):
|
|
|
123
123
|
error = ApiError.RESOURCE_ERROR.copy()
|
|
124
124
|
try:
|
|
125
125
|
action = payload.pop('action')
|
|
126
|
-
args = ignore_keys(payload, ['name', 'cluster_type'])
|
|
127
126
|
if action == 'stop':
|
|
128
|
-
workspace.stop(**
|
|
127
|
+
workspace.stop(**payload)
|
|
129
128
|
elif action == 'resume':
|
|
130
|
-
workspace.resume(**
|
|
129
|
+
workspace.resume(**payload)
|
|
130
|
+
elif action == 'patch':
|
|
131
|
+
workspace.update(payload)
|
|
131
132
|
elif action == 'add_to_ingress':
|
|
132
133
|
if isinstance(workspace, KubernetesWorkspace):
|
|
133
|
-
workspace.add_to_ingress(**
|
|
134
|
+
workspace.add_to_ingress(**payload)
|
|
134
135
|
else:
|
|
135
136
|
raise Exception('This workspace does not support ingress.')
|
|
136
137
|
except Exception as ex:
|
|
@@ -294,7 +294,7 @@ class BlockActionObjectCache(BaseCache):
|
|
|
294
294
|
)
|
|
295
295
|
|
|
296
296
|
content = None
|
|
297
|
-
with open(block_file_absolute_path, 'r') as f:
|
|
297
|
+
with open(block_file_absolute_path, 'r', encoding='utf-8') as f:
|
|
298
298
|
content = f.read()
|
|
299
299
|
|
|
300
300
|
mapping[OBJECT_TYPE_BLOCK_FILE][key] = dict(
|
|
@@ -477,6 +477,57 @@ class WorkloadManager:
|
|
|
477
477
|
|
|
478
478
|
return k8s_service
|
|
479
479
|
|
|
480
|
+
def patch_workload(
|
|
481
|
+
self,
|
|
482
|
+
name: str,
|
|
483
|
+
workspace_config: KubernetesWorkspaceConfig,
|
|
484
|
+
update_workspace_settings: bool = False,
|
|
485
|
+
) -> None:
|
|
486
|
+
"""
|
|
487
|
+
Update workload for k8s. Currently the only fields that can be updated are
|
|
488
|
+
container_config and workspace settings.
|
|
489
|
+
|
|
490
|
+
Args:
|
|
491
|
+
name (str): name of the workload/workspace
|
|
492
|
+
workspace_config (KubernetesWorkspaceConfig): new workspace config
|
|
493
|
+
update_workspace_settings (bool): whether to update workspace settings with
|
|
494
|
+
the current pod's environment variables.
|
|
495
|
+
"""
|
|
496
|
+
container_config_yaml = workspace_config.container_config
|
|
497
|
+
container_config = dict()
|
|
498
|
+
if isinstance(container_config_yaml, str):
|
|
499
|
+
container_config = yaml.full_load(container_config_yaml)
|
|
500
|
+
|
|
501
|
+
if update_workspace_settings:
|
|
502
|
+
env_vars = self.__populate_env_vars(
|
|
503
|
+
name,
|
|
504
|
+
container_config=container_config,
|
|
505
|
+
)
|
|
506
|
+
container_config['env'] = env_vars
|
|
507
|
+
|
|
508
|
+
mage_container_config = {
|
|
509
|
+
'name': f'{name}-container',
|
|
510
|
+
**container_config,
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
stateful_set_template_spec = {
|
|
514
|
+
'containers': [mage_container_config],
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
updated_stateful_set = {
|
|
518
|
+
'spec': {
|
|
519
|
+
'template': {
|
|
520
|
+
'spec': stateful_set_template_spec,
|
|
521
|
+
},
|
|
522
|
+
},
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
self.apps_client.patch_namespaced_stateful_set(
|
|
526
|
+
name,
|
|
527
|
+
namespace=self.namespace,
|
|
528
|
+
body=updated_stateful_set,
|
|
529
|
+
)
|
|
530
|
+
|
|
480
531
|
def add_service_to_ingress_paths(
|
|
481
532
|
self,
|
|
482
533
|
ingress_name: str,
|
|
@@ -712,7 +763,7 @@ class WorkloadManager:
|
|
|
712
763
|
def __populate_env_vars(
|
|
713
764
|
self,
|
|
714
765
|
name,
|
|
715
|
-
project_type: str =
|
|
766
|
+
project_type: str = None,
|
|
716
767
|
project_uuid: str = None,
|
|
717
768
|
container_config: Dict = None,
|
|
718
769
|
initial_metadata: Dict = None,
|
|
@@ -148,6 +148,12 @@ class Workspace(abc.ABC):
|
|
|
148
148
|
"""
|
|
149
149
|
raise NotImplementedError('Initialize method not implemented')
|
|
150
150
|
|
|
151
|
+
def update(self, payload: Dict, **kwargs):
|
|
152
|
+
"""
|
|
153
|
+
Update the workspace configuration.
|
|
154
|
+
"""
|
|
155
|
+
raise NotImplementedError('Update method not implemented')
|
|
156
|
+
|
|
151
157
|
@abc.abstractmethod
|
|
152
158
|
@safe_db_query
|
|
153
159
|
def delete(self, **kwargs):
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import os
|
|
2
|
+
from typing import Dict
|
|
2
3
|
|
|
3
4
|
import yaml
|
|
4
5
|
|
|
@@ -7,7 +8,7 @@ from mage_ai.cluster_manager.constants import KUBE_NAMESPACE, ClusterType
|
|
|
7
8
|
from mage_ai.cluster_manager.kubernetes.workload_manager import WorkloadManager
|
|
8
9
|
from mage_ai.cluster_manager.workspace.base import Workspace
|
|
9
10
|
from mage_ai.data_preparation.repo_manager import ProjectType, get_project_type
|
|
10
|
-
from mage_ai.shared.hash import merge_dict
|
|
11
|
+
from mage_ai.shared.hash import extract, merge_dict
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
class KubernetesWorkspace(Workspace):
|
|
@@ -82,6 +83,26 @@ class KubernetesWorkspace(Workspace):
|
|
|
82
83
|
|
|
83
84
|
return cls(name)
|
|
84
85
|
|
|
86
|
+
def update(self, payload: Dict, **kwargs):
|
|
87
|
+
update_workspace_settings = payload.pop('update_workspace_settings', False)
|
|
88
|
+
extracted_payload = extract(payload, [
|
|
89
|
+
'container_config',
|
|
90
|
+
])
|
|
91
|
+
updated_config = merge_dict(
|
|
92
|
+
self.config.to_dict(),
|
|
93
|
+
extracted_payload,
|
|
94
|
+
)
|
|
95
|
+
workspace_config = KubernetesWorkspaceConfig.load(
|
|
96
|
+
config=updated_config
|
|
97
|
+
)
|
|
98
|
+
self.workload_manager.patch_workload(
|
|
99
|
+
self.name,
|
|
100
|
+
workspace_config,
|
|
101
|
+
update_workspace_settings=update_workspace_settings,
|
|
102
|
+
)
|
|
103
|
+
with open(self.config_path, 'w', encoding='utf-8') as fp:
|
|
104
|
+
yaml.dump(workspace_config.to_dict(), fp)
|
|
105
|
+
|
|
85
106
|
def delete(self, **kwargs):
|
|
86
107
|
try:
|
|
87
108
|
self.workload_manager.delete_workload(
|
|
@@ -17,7 +17,7 @@ from mage_ai.version_control.models import File
|
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
19
|
Icons:
|
|
20
|
-
|
|
20
|
+
Text Editor (file browser): ChurnV3
|
|
21
21
|
Portal Terminal (terminal): RankingV3
|
|
22
22
|
Version Control (file diffs): ForecastV3
|
|
23
23
|
|
|
@@ -65,7 +65,7 @@ async def build_application_arcane_library() -> Dict:
|
|
|
65
65
|
return build_generic(model_class=File, item_dict=dict(
|
|
66
66
|
item_type=ItemType.DETAIL,
|
|
67
67
|
object_type=ObjectType.FILE,
|
|
68
|
-
title='
|
|
68
|
+
title='Text editor',
|
|
69
69
|
description='browse and edit files across all projects',
|
|
70
70
|
subtitle='Browser / Editor',
|
|
71
71
|
applications=[
|
|
@@ -5,5 +5,5 @@ def application_title(uuid: ApplicationExpansionUUID) -> str:
|
|
|
5
5
|
if ApplicationExpansionUUID.VersionControlFileDiffs == uuid:
|
|
6
6
|
return 'version control file diffs'
|
|
7
7
|
if ApplicationExpansionUUID.ArcaneLibrary == uuid:
|
|
8
|
-
return 'the
|
|
8
|
+
return 'the Text Editor'
|
|
9
9
|
return f'the {uuid} application'
|
|
@@ -4,10 +4,13 @@ from jinja2 import Template
|
|
|
4
4
|
|
|
5
5
|
from mage_ai.data_preparation.executors.block_executor import BlockExecutor
|
|
6
6
|
from mage_ai.data_preparation.shared.utils import get_template_vars
|
|
7
|
+
from mage_ai.orchestration.db import safe_db_query
|
|
8
|
+
from mage_ai.orchestration.db.models.schedules import BlockRun
|
|
7
9
|
from mage_ai.services.k8s.config import K8sExecutorConfig
|
|
8
10
|
from mage_ai.services.k8s.constants import DEFAULT_NAMESPACE
|
|
9
11
|
from mage_ai.services.k8s.job_manager import JobManager as K8sJobManager
|
|
10
12
|
from mage_ai.shared.hash import merge_dict
|
|
13
|
+
from mage_ai.shared.utils import clean_name
|
|
11
14
|
|
|
12
15
|
|
|
13
16
|
class K8sBlockExecutor(BlockExecutor):
|
|
@@ -15,10 +18,13 @@ class K8sBlockExecutor(BlockExecutor):
|
|
|
15
18
|
|
|
16
19
|
def __init__(self, pipeline, block_uuid: str, execution_partition: str = None, **kwargs):
|
|
17
20
|
super().__init__(pipeline, block_uuid, execution_partition=execution_partition)
|
|
18
|
-
self.
|
|
21
|
+
self.executor_config_dict = self.pipeline.repo_config.k8s_executor_config or dict()
|
|
19
22
|
if self.block.executor_config is not None:
|
|
20
|
-
self.
|
|
21
|
-
|
|
23
|
+
self.executor_config_dict = merge_dict(
|
|
24
|
+
self.executor_config_dict,
|
|
25
|
+
self.block.executor_config,
|
|
26
|
+
)
|
|
27
|
+
self.executor_config = K8sExecutorConfig.load(config=self.executor_config_dict)
|
|
22
28
|
|
|
23
29
|
def _execute(
|
|
24
30
|
self,
|
|
@@ -26,10 +32,7 @@ class K8sBlockExecutor(BlockExecutor):
|
|
|
26
32
|
global_vars: Dict = None,
|
|
27
33
|
**kwargs,
|
|
28
34
|
) -> None:
|
|
29
|
-
|
|
30
|
-
job_name_prefix = 'data-prep'
|
|
31
|
-
else:
|
|
32
|
-
job_name_prefix = self.executor_config.job_name_prefix
|
|
35
|
+
job_name_prefix = self._get_job_name_prefix(block_run_id)
|
|
33
36
|
|
|
34
37
|
if self.executor_config.namespace:
|
|
35
38
|
namespace = Template(self.executor_config.namespace).render(
|
|
@@ -54,3 +57,23 @@ class K8sBlockExecutor(BlockExecutor):
|
|
|
54
57
|
cmd,
|
|
55
58
|
k8s_config=self.executor_config,
|
|
56
59
|
)
|
|
60
|
+
|
|
61
|
+
@safe_db_query
|
|
62
|
+
def _get_job_name_prefix(
|
|
63
|
+
self,
|
|
64
|
+
block_run_id,
|
|
65
|
+
):
|
|
66
|
+
if not self.executor_config.job_name_prefix:
|
|
67
|
+
job_name_prefix = 'data-prep'
|
|
68
|
+
else:
|
|
69
|
+
job_name_prefix = self.executor_config.job_name_prefix
|
|
70
|
+
if not block_run_id:
|
|
71
|
+
return job_name_prefix
|
|
72
|
+
|
|
73
|
+
if '{trigger_name}' in job_name_prefix:
|
|
74
|
+
block_run = BlockRun.query.get(block_run_id)
|
|
75
|
+
trigger = block_run.pipeline_run.pipeline_schedule
|
|
76
|
+
job_name_prefix = job_name_prefix.format(
|
|
77
|
+
trigger_name=clean_name(trigger.name).replace('_', '-'))
|
|
78
|
+
|
|
79
|
+
return job_name_prefix
|