mage-ai 0.9.69__py3-none-any.whl → 0.9.71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mage-ai might be problematic. Click here for more details.
- mage_ai/ai/utils/xgboost.py +222 -0
- mage_ai/api/errors.py +37 -25
- mage_ai/api/operations/base.py +13 -1
- mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
- mage_ai/api/policies/BackfillPolicy.py +1 -0
- mage_ai/api/policies/BlockOutputPolicy.py +40 -17
- mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
- mage_ai/api/policies/KernelPolicy.py +55 -32
- mage_ai/api/policies/KernelProcessPolicy.py +56 -0
- mage_ai/api/policies/OutputPolicy.py +73 -41
- mage_ai/api/policies/PipelinePolicy.py +206 -137
- mage_ai/api/policies/WorkspacePolicy.py +1 -0
- mage_ai/api/presenters/BackfillPresenter.py +1 -0
- mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
- mage_ai/api/presenters/BlockPresenter.py +1 -1
- mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
- mage_ai/api/presenters/KernelPresenter.py +5 -26
- mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
- mage_ai/api/presenters/PipelinePresenter.py +18 -5
- mage_ai/api/presenters/StatusPresenter.py +2 -0
- mage_ai/api/presenters/SyncPresenter.py +25 -0
- mage_ai/api/resources/AutocompleteItemResource.py +1 -1
- mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
- mage_ai/api/resources/BlockOutputResource.py +42 -9
- mage_ai/api/resources/BlockResource.py +4 -3
- mage_ai/api/resources/BlockRunResource.py +27 -22
- mage_ai/api/resources/ClusterResource.py +4 -1
- mage_ai/api/resources/CustomTemplateResource.py +34 -14
- mage_ai/api/resources/DataProviderResource.py +1 -1
- mage_ai/api/resources/ExecutionStateResource.py +3 -1
- mage_ai/api/resources/FileContentResource.py +8 -2
- mage_ai/api/resources/FileResource.py +10 -4
- mage_ai/api/resources/FileVersionResource.py +3 -1
- mage_ai/api/resources/GitBranchResource.py +101 -31
- mage_ai/api/resources/GitCustomBranchResource.py +29 -1
- mage_ai/api/resources/GlobalDataProductResource.py +44 -7
- mage_ai/api/resources/GlobalHookResource.py +4 -1
- mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
- mage_ai/api/resources/IntegrationSourceResource.py +8 -4
- mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
- mage_ai/api/resources/KernelProcessResource.py +44 -0
- mage_ai/api/resources/KernelResource.py +25 -3
- mage_ai/api/resources/OauthResource.py +1 -1
- mage_ai/api/resources/OutputResource.py +33 -11
- mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
- mage_ai/api/resources/PipelineInteractionResource.py +31 -15
- mage_ai/api/resources/PipelineResource.py +258 -125
- mage_ai/api/resources/PipelineRunResource.py +52 -7
- mage_ai/api/resources/PipelineScheduleResource.py +11 -2
- mage_ai/api/resources/PipelineTriggerResource.py +6 -1
- mage_ai/api/resources/ProjectResource.py +18 -7
- mage_ai/api/resources/PullRequestResource.py +6 -4
- mage_ai/api/resources/SecretResource.py +1 -1
- mage_ai/api/resources/SeedResource.py +8 -1
- mage_ai/api/resources/StatusResource.py +21 -6
- mage_ai/api/resources/SyncResource.py +6 -8
- mage_ai/api/resources/VariableResource.py +46 -26
- mage_ai/api/resources/VersionControlProjectResource.py +9 -2
- mage_ai/api/resources/WidgetResource.py +1 -1
- mage_ai/api/resources/WorkspaceResource.py +6 -5
- mage_ai/api/views.py +47 -40
- mage_ai/authentication/permissions/seed.py +16 -2
- mage_ai/authentication/providers/oidc.py +21 -1
- mage_ai/autocomplete/utils.py +13 -9
- mage_ai/cache/base.py +1 -1
- mage_ai/cache/block.py +18 -12
- mage_ai/cache/block_action_object/__init__.py +33 -5
- mage_ai/cache/file.py +22 -19
- mage_ai/cache/pipeline.py +18 -12
- mage_ai/cli/main.py +1 -0
- mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
- mage_ai/cluster_manager/config.py +2 -2
- mage_ai/cluster_manager/kubernetes/workload_manager.py +52 -1
- mage_ai/cluster_manager/manage.py +1 -1
- mage_ai/cluster_manager/workspace/base.py +7 -1
- mage_ai/cluster_manager/workspace/kubernetes.py +22 -1
- mage_ai/command_center/applications/factory.py +10 -7
- mage_ai/command_center/applications/utils.py +2 -2
- mage_ai/command_center/files/factory.py +17 -15
- mage_ai/command_center/presenters/text.py +1 -1
- mage_ai/command_center/utils.py +25 -13
- mage_ai/data/__init__.py +0 -0
- mage_ai/data/constants.py +45 -0
- mage_ai/data/models/__init__.py +0 -0
- mage_ai/data/models/base.py +119 -0
- mage_ai/data/models/constants.py +1 -0
- mage_ai/data/models/generator.py +115 -0
- mage_ai/data/models/manager.py +168 -0
- mage_ai/data/models/pyarrow/__init__.py +0 -0
- mage_ai/data/models/pyarrow/record_batch.py +55 -0
- mage_ai/data/models/pyarrow/shared.py +21 -0
- mage_ai/data/models/pyarrow/table.py +8 -0
- mage_ai/data/models/reader.py +103 -0
- mage_ai/data/models/utils.py +59 -0
- mage_ai/data/models/writer.py +91 -0
- mage_ai/data/tabular/__init__.py +0 -0
- mage_ai/data/tabular/constants.py +23 -0
- mage_ai/data/tabular/mocks.py +19 -0
- mage_ai/data/tabular/models.py +126 -0
- mage_ai/data/tabular/reader.py +602 -0
- mage_ai/data/tabular/utils.py +102 -0
- mage_ai/data/tabular/writer.py +266 -0
- mage_ai/data/variables/__init__.py +0 -0
- mage_ai/data/variables/wrapper.py +54 -0
- mage_ai/data_cleaner/analysis/charts.py +61 -39
- mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
- mage_ai/data_cleaner/estimators/encoders.py +5 -2
- mage_ai/data_integrations/utils/scheduler.py +16 -11
- mage_ai/data_preparation/decorators.py +1 -0
- mage_ai/data_preparation/executors/block_executor.py +237 -155
- mage_ai/data_preparation/executors/k8s_block_executor.py +30 -7
- mage_ai/data_preparation/executors/k8s_pipeline_executor.py +30 -7
- mage_ai/data_preparation/executors/streaming_pipeline_executor.py +2 -2
- mage_ai/data_preparation/git/__init__.py +77 -29
- mage_ai/data_preparation/git/api.py +69 -8
- mage_ai/data_preparation/git/utils.py +64 -34
- mage_ai/data_preparation/logging/logger_manager.py +4 -3
- mage_ai/data_preparation/models/block/__init__.py +1562 -879
- mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
- mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
- mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
- mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
- mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
- mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
- mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
- mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
- mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
- mage_ai/data_preparation/models/block/dynamic/utils.py +295 -167
- mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
- mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
- mage_ai/data_preparation/models/block/extension/utils.py +10 -1
- mage_ai/data_preparation/models/block/global_data_product/__init__.py +35 -3
- mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
- mage_ai/data_preparation/models/block/outputs.py +722 -0
- mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
- mage_ai/data_preparation/models/block/r/__init__.py +56 -38
- mage_ai/data_preparation/models/block/remote/__init__.py +0 -0
- mage_ai/data_preparation/models/block/remote/models.py +58 -0
- mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
- mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
- mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
- mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
- mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
- mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
- mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
- mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
- mage_ai/data_preparation/models/block/utils.py +164 -69
- mage_ai/data_preparation/models/constants.py +21 -14
- mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
- mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
- mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
- mage_ai/data_preparation/models/file.py +41 -28
- mage_ai/data_preparation/models/global_data_product/__init__.py +100 -58
- mage_ai/data_preparation/models/global_hooks/models.py +1 -0
- mage_ai/data_preparation/models/interfaces.py +29 -0
- mage_ai/data_preparation/models/pipeline.py +374 -185
- mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
- mage_ai/data_preparation/models/pipelines/seed.py +1 -1
- mage_ai/data_preparation/models/project/__init__.py +66 -18
- mage_ai/data_preparation/models/project/constants.py +2 -0
- mage_ai/data_preparation/models/triggers/__init__.py +124 -26
- mage_ai/data_preparation/models/utils.py +467 -17
- mage_ai/data_preparation/models/variable.py +1028 -137
- mage_ai/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/variables/cache.py +149 -0
- mage_ai/data_preparation/models/variables/constants.py +72 -0
- mage_ai/data_preparation/models/variables/summarizer.py +336 -0
- mage_ai/data_preparation/models/variables/utils.py +77 -0
- mage_ai/data_preparation/models/widget/__init__.py +63 -41
- mage_ai/data_preparation/models/widget/charts.py +40 -27
- mage_ai/data_preparation/models/widget/constants.py +2 -0
- mage_ai/data_preparation/models/widget/utils.py +3 -3
- mage_ai/data_preparation/preferences.py +3 -3
- mage_ai/data_preparation/repo_manager.py +55 -21
- mage_ai/data_preparation/storage/base_storage.py +2 -2
- mage_ai/data_preparation/storage/gcs_storage.py +7 -4
- mage_ai/data_preparation/storage/local_storage.py +18 -9
- mage_ai/data_preparation/storage/s3_storage.py +5 -2
- mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
- mage_ai/data_preparation/variable_manager.py +281 -76
- mage_ai/io/base.py +3 -2
- mage_ai/io/bigquery.py +1 -0
- mage_ai/io/redshift.py +7 -5
- mage_ai/kernels/__init__.py +0 -0
- mage_ai/kernels/models.py +188 -0
- mage_ai/kernels/utils.py +169 -0
- mage_ai/orchestration/concurrency.py +6 -2
- mage_ai/orchestration/db/__init__.py +1 -0
- mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
- mage_ai/orchestration/db/migrations/versions/42a14d6143f1_update_token_column_type.py +54 -0
- mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
- mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
- mage_ai/orchestration/db/models/oauth.py +12 -18
- mage_ai/orchestration/db/models/projects.py +10 -0
- mage_ai/orchestration/db/models/schedules.py +225 -187
- mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
- mage_ai/orchestration/db/models/utils.py +46 -5
- mage_ai/orchestration/metrics/pipeline_run.py +8 -9
- mage_ai/orchestration/notification/sender.py +38 -15
- mage_ai/orchestration/pipeline_scheduler_original.py +64 -33
- mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
- mage_ai/orchestration/run_status_checker.py +11 -4
- mage_ai/orchestration/triggers/api.py +41 -2
- mage_ai/orchestration/triggers/global_data_product.py +9 -4
- mage_ai/orchestration/triggers/utils.py +10 -1
- mage_ai/orchestration/utils/resources.py +3 -0
- mage_ai/presenters/charts/data_sources/base.py +4 -2
- mage_ai/presenters/charts/data_sources/block.py +15 -9
- mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
- mage_ai/presenters/charts/data_sources/constants.py +1 -0
- mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
- mage_ai/presenters/interactions/models.py +11 -7
- mage_ai/presenters/pages/loaders/pipelines.py +5 -3
- mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
- mage_ai/presenters/utils.py +2 -0
- mage_ai/server/api/blocks.py +2 -1
- mage_ai/server/api/downloads.py +9 -2
- mage_ai/server/api/runs.py +151 -0
- mage_ai/server/api/triggers.py +3 -1
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +8 -8
- mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
- mage_ai/server/frontend_dist/block-layout.html +3 -3
- mage_ai/server/frontend_dist/compute.html +6 -6
- mage_ai/server/frontend_dist/files.html +6 -6
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-data-products.html +6 -6
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/index.html +3 -3
- mage_ai/server/frontend_dist/manage/files.html +6 -6
- mage_ai/server/frontend_dist/manage/settings.html +6 -6
- mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist/manage/users.html +6 -6
- mage_ai/server/frontend_dist/manage.html +6 -6
- mage_ai/server/frontend_dist/oauth.html +5 -5
- mage_ai/server/frontend_dist/overview.html +6 -6
- mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist/pipelines.html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist/settings.html +3 -3
- mage_ai/server/frontend_dist/sign-in.html +15 -15
- mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist/templates.html +6 -6
- mage_ai/server/frontend_dist/terminal.html +6 -6
- mage_ai/server/frontend_dist/test.html +3 -3
- mage_ai/server/frontend_dist/triggers.html +6 -6
- mage_ai/server/frontend_dist/version-control.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/404.html +8 -8
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +15 -15
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
- mage_ai/server/kernel_output_parser.py +4 -1
- mage_ai/server/scheduler_manager.py +12 -1
- mage_ai/server/server.py +69 -42
- mage_ai/server/utils/custom_output.py +284 -0
- mage_ai/server/utils/execute_custom_code.py +245 -0
- mage_ai/server/utils/output_display.py +123 -289
- mage_ai/server/websocket_server.py +116 -69
- mage_ai/services/aws/ecs/ecs.py +1 -0
- mage_ai/services/k8s/config.py +27 -4
- mage_ai/services/k8s/job_manager.py +6 -1
- mage_ai/services/k8s/utils.py +97 -0
- mage_ai/services/ssh/aws/emr/utils.py +8 -8
- mage_ai/settings/keys/auth.py +1 -0
- mage_ai/settings/platform/__init__.py +159 -38
- mage_ai/settings/platform/constants.py +5 -0
- mage_ai/settings/platform/utils.py +53 -10
- mage_ai/settings/repo.py +26 -12
- mage_ai/settings/server.py +128 -37
- mage_ai/shared/array.py +24 -1
- mage_ai/shared/complex.py +45 -0
- mage_ai/shared/config.py +2 -1
- mage_ai/shared/custom_logger.py +11 -0
- mage_ai/shared/dates.py +10 -6
- mage_ai/shared/files.py +63 -8
- mage_ai/shared/hash.py +33 -9
- mage_ai/shared/io.py +9 -5
- mage_ai/shared/models.py +82 -24
- mage_ai/shared/outputs.py +87 -0
- mage_ai/shared/parsers.py +144 -13
- mage_ai/shared/path_fixer.py +11 -7
- mage_ai/shared/singletons/__init__.py +0 -0
- mage_ai/shared/singletons/base.py +47 -0
- mage_ai/shared/singletons/memory.py +38 -0
- mage_ai/shared/strings.py +34 -1
- mage_ai/shared/yaml.py +24 -0
- mage_ai/streaming/sinks/oracledb.py +57 -0
- mage_ai/streaming/sinks/sink_factory.py +4 -0
- mage_ai/system/__init__.py +0 -0
- mage_ai/system/constants.py +14 -0
- mage_ai/system/memory/__init__.py +0 -0
- mage_ai/system/memory/constants.py +1 -0
- mage_ai/system/memory/manager.py +174 -0
- mage_ai/system/memory/presenters.py +158 -0
- mage_ai/system/memory/process.py +216 -0
- mage_ai/system/memory/samples.py +13 -0
- mage_ai/system/memory/utils.py +656 -0
- mage_ai/system/memory/wrappers.py +177 -0
- mage_ai/system/models.py +58 -0
- mage_ai/system/storage/__init__.py +0 -0
- mage_ai/system/storage/utils.py +29 -0
- mage_ai/tests/api/endpoints/mixins.py +2 -2
- mage_ai/tests/api/endpoints/test_blocks.py +2 -1
- mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
- mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
- mage_ai/tests/api/endpoints/test_projects.py +2 -1
- mage_ai/tests/api/operations/base/mixins.py +1 -1
- mage_ai/tests/api/operations/base/test_base.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
- mage_ai/tests/api/operations/test_syncs.py +6 -4
- mage_ai/tests/api/resources/test_pipeline_resource.py +11 -4
- mage_ai/tests/authentication/oauth/test_utils.py +1 -1
- mage_ai/tests/authentication/providers/test_oidc.py +59 -0
- mage_ai/tests/base_test.py +2 -2
- mage_ai/tests/data/__init__.py +0 -0
- mage_ai/tests/data/models/__init__.py +0 -0
- mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
- mage_ai/tests/data_preparation/git/test_git.py +4 -1
- mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
- mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
- mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
- mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
- mage_ai/tests/data_preparation/models/block/test_global_data_product.py +5 -2
- mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
- mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
- mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
- mage_ai/tests/data_preparation/models/test_block.py +115 -120
- mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
- mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
- mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
- mage_ai/tests/data_preparation/models/test_project.py +4 -1
- mage_ai/tests/data_preparation/models/test_utils.py +80 -0
- mage_ai/tests/data_preparation/models/test_variable.py +242 -69
- mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
- mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
- mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
- mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
- mage_ai/tests/factory.py +64 -43
- mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
- mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
- mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
- mage_ai/tests/orchestration/triggers/test_global_data_product.py +141 -138
- mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
- mage_ai/tests/server/test_server.py +19 -0
- mage_ai/tests/services/k8s/test_job_manager.py +27 -6
- mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
- mage_ai/tests/test_shared.py +61 -0
- mage_ai/usage_statistics/logger.py +7 -2
- mage_ai/utils/code.py +33 -19
- mage_ai/version_control/branch/utils.py +2 -1
- mage_ai/version_control/models.py +3 -2
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/METADATA +6 -3
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/RECORD +555 -454
- mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
- mage_ai/server/frontend_dist/_next/static/_krrrgup_C-dPOpX36S8I/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/KLL5mirre9d7_ZeEpaw3s/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
- mage_ai/shared/memory.py +0 -90
- mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
- /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
- /mage_ai/server/frontend_dist/_next/static/{_krrrgup_C-dPOpX36S8I → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{KLL5mirre9d7_ZeEpaw3s → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
- /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from sqlalchemy import and_, desc, func
|
|
1
2
|
from sqlalchemy.orm import selectinload
|
|
2
3
|
|
|
3
4
|
from mage_ai.api.operations.constants import META_KEY_LIMIT, META_KEY_OFFSET
|
|
@@ -5,7 +6,10 @@ from mage_ai.api.resources.DatabaseResource import DatabaseResource
|
|
|
5
6
|
from mage_ai.api.utils import get_query_timestamps
|
|
6
7
|
from mage_ai.cache.tag import TagCache
|
|
7
8
|
from mage_ai.data_preparation.models.block.utils import get_all_descendants
|
|
8
|
-
from mage_ai.data_preparation.models.constants import
|
|
9
|
+
from mage_ai.data_preparation.models.constants import (
|
|
10
|
+
PIPELINE_RUN_STATUS_LAST_RUN_FAILED,
|
|
11
|
+
PipelineType,
|
|
12
|
+
)
|
|
9
13
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
10
14
|
from mage_ai.data_preparation.models.triggers import (
|
|
11
15
|
ScheduleInterval,
|
|
@@ -94,9 +98,38 @@ class PipelineRunResource(DatabaseResource):
|
|
|
94
98
|
|
|
95
99
|
repo_pipeline_schedule_ids = [s.id for s in PipelineSchedule.repo_query]
|
|
96
100
|
|
|
101
|
+
if status == PIPELINE_RUN_STATUS_LAST_RUN_FAILED:
|
|
102
|
+
"""
|
|
103
|
+
The initial query below is used to get the last pipeline run retry
|
|
104
|
+
(or individual pipeline run if there are no retries) for each
|
|
105
|
+
grouping of pipeline runs with the same execution_date,
|
|
106
|
+
pipeline_uuid, and pipeline_schedule_id.
|
|
107
|
+
"""
|
|
108
|
+
latest_pipeline_runs = PipelineRun.select(
|
|
109
|
+
PipelineRun.id,
|
|
110
|
+
func.row_number()
|
|
111
|
+
.over(
|
|
112
|
+
partition_by=(
|
|
113
|
+
PipelineRun.execution_date,
|
|
114
|
+
PipelineRun.pipeline_schedule_id,
|
|
115
|
+
PipelineRun.pipeline_uuid,
|
|
116
|
+
),
|
|
117
|
+
order_by=desc(PipelineRun.id))
|
|
118
|
+
.label('row_number')
|
|
119
|
+
).cte(name='latest_pipeline_runs')
|
|
120
|
+
query = (PipelineRun.select(
|
|
121
|
+
PipelineRun,
|
|
122
|
+
)
|
|
123
|
+
.join(latest_pipeline_runs, and_(
|
|
124
|
+
PipelineRun.id == latest_pipeline_runs.c.id,
|
|
125
|
+
latest_pipeline_runs.c.row_number == 1,
|
|
126
|
+
))
|
|
127
|
+
)
|
|
128
|
+
else:
|
|
129
|
+
query = PipelineRun.query
|
|
130
|
+
|
|
97
131
|
results = (
|
|
98
|
-
|
|
99
|
-
.query
|
|
132
|
+
query
|
|
100
133
|
.filter(PipelineRun.pipeline_schedule_id.in_(repo_pipeline_schedule_ids))
|
|
101
134
|
.options(selectinload(PipelineRun.block_runs))
|
|
102
135
|
.options(selectinload(PipelineRun.pipeline_schedule))
|
|
@@ -120,8 +153,12 @@ class PipelineRunResource(DatabaseResource):
|
|
|
120
153
|
results = results.filter(PipelineRun.pipeline_uuid.in_(pipeline_uuids))
|
|
121
154
|
if pipeline_uuids_with_tags:
|
|
122
155
|
results = results.filter(PipelineRun.pipeline_uuid.in_(pipeline_uuids_with_tags))
|
|
123
|
-
|
|
156
|
+
|
|
157
|
+
if status == PIPELINE_RUN_STATUS_LAST_RUN_FAILED:
|
|
158
|
+
results = results.filter(PipelineRun.status == PipelineRun.PipelineRunStatus.FAILED)
|
|
159
|
+
elif status is not None:
|
|
124
160
|
results = results.filter(PipelineRun.status == status)
|
|
161
|
+
|
|
125
162
|
if statuses:
|
|
126
163
|
results = results.filter(PipelineRun.status.in_(statuses))
|
|
127
164
|
if start_timestamp is not None:
|
|
@@ -152,6 +189,7 @@ class PipelineRunResource(DatabaseResource):
|
|
|
152
189
|
@classmethod
|
|
153
190
|
@safe_db_query
|
|
154
191
|
async def process_collection(self, query_arg, meta, user, **kwargs):
|
|
192
|
+
context_data = kwargs.get('context_data')
|
|
155
193
|
total_results = await self.collection(query_arg, meta, user, **kwargs)
|
|
156
194
|
total_count = total_results.count()
|
|
157
195
|
|
|
@@ -262,7 +300,11 @@ class PipelineRunResource(DatabaseResource):
|
|
|
262
300
|
}
|
|
263
301
|
|
|
264
302
|
if include_pipeline_uuids:
|
|
265
|
-
|
|
303
|
+
repo_path = get_repo_path(context_data=context_data, user=user)
|
|
304
|
+
pipeline_uuids = Pipeline.get_all_pipelines_all_projects(
|
|
305
|
+
context_data=context_data,
|
|
306
|
+
repo_path=repo_path,
|
|
307
|
+
)
|
|
266
308
|
result_set.metadata['pipeline_uuids'] = pipeline_uuids
|
|
267
309
|
|
|
268
310
|
return result_set
|
|
@@ -272,7 +314,8 @@ class PipelineRunResource(DatabaseResource):
|
|
|
272
314
|
def create(self, payload, user, **kwargs):
|
|
273
315
|
pipeline_schedule = kwargs.get('parent_model')
|
|
274
316
|
|
|
275
|
-
|
|
317
|
+
repo_path = get_repo_path(user=user)
|
|
318
|
+
pipeline = Pipeline.get(pipeline_schedule.pipeline_uuid, repo_path=repo_path)
|
|
276
319
|
configured_payload, _ = configure_pipeline_run_payload(
|
|
277
320
|
pipeline_schedule,
|
|
278
321
|
pipeline.type,
|
|
@@ -295,9 +338,10 @@ class PipelineRunResource(DatabaseResource):
|
|
|
295
338
|
|
|
296
339
|
@safe_db_query
|
|
297
340
|
def update(self, payload, **kwargs):
|
|
341
|
+
repo_path = get_repo_path(user=self.current_user)
|
|
298
342
|
if 'retry_blocks' == payload.get('pipeline_run_action'):
|
|
299
343
|
self.model.refresh()
|
|
300
|
-
pipeline = Pipeline.get(self.model.pipeline_uuid)
|
|
344
|
+
pipeline = Pipeline.get(self.model.pipeline_uuid, repo_path=repo_path)
|
|
301
345
|
block_runs_to_retry = []
|
|
302
346
|
from_block_uuid = payload.get('from_block_uuid')
|
|
303
347
|
if from_block_uuid is not None:
|
|
@@ -348,6 +392,7 @@ class PipelineRunResource(DatabaseResource):
|
|
|
348
392
|
pipeline = Pipeline.get(
|
|
349
393
|
self.model.pipeline_uuid,
|
|
350
394
|
check_if_exists=True,
|
|
395
|
+
repo_path=repo_path,
|
|
351
396
|
)
|
|
352
397
|
|
|
353
398
|
stop_pipeline_run(self.model, pipeline)
|
|
@@ -253,7 +253,10 @@ class PipelineScheduleResource(DatabaseResource):
|
|
|
253
253
|
payload['pipeline_uuid'] = pipeline.uuid
|
|
254
254
|
|
|
255
255
|
if 'repo_path' not in payload:
|
|
256
|
-
payload['repo_path'] = (
|
|
256
|
+
payload['repo_path'] = (
|
|
257
|
+
(pipeline.repo_path if pipeline else None)
|
|
258
|
+
or get_repo_path(user=user)
|
|
259
|
+
)
|
|
257
260
|
if 'token' not in payload:
|
|
258
261
|
payload['token'] = uuid.uuid4().hex
|
|
259
262
|
if payload.get('status') == ScheduleStatus.ACTIVE and \
|
|
@@ -403,10 +406,13 @@ class PipelineScheduleResource(DatabaseResource):
|
|
|
403
406
|
if updated_status == ScheduleStatus.ACTIVE and self.model.status == ScheduleStatus.INACTIVE:
|
|
404
407
|
payload['last_enabled_at'] = datetime.now(tz=pytz.UTC)
|
|
405
408
|
|
|
409
|
+
old_name = self.model.name
|
|
410
|
+
|
|
406
411
|
resource = super().update(payload)
|
|
407
412
|
updated_model = resource.model
|
|
408
413
|
|
|
409
|
-
|
|
414
|
+
repo_path = get_repo_path(user=self.current_user)
|
|
415
|
+
pipeline = Pipeline.get(updated_model.pipeline_uuid, repo_path=repo_path)
|
|
410
416
|
if pipeline:
|
|
411
417
|
trigger = Trigger(
|
|
412
418
|
last_enabled_at=updated_model.last_enabled_at,
|
|
@@ -418,17 +424,20 @@ class PipelineScheduleResource(DatabaseResource):
|
|
|
418
424
|
sla=updated_model.sla,
|
|
419
425
|
start_time=updated_model.start_time,
|
|
420
426
|
status=updated_model.status,
|
|
427
|
+
token=updated_model.token,
|
|
421
428
|
variables=updated_model.variables,
|
|
422
429
|
)
|
|
423
430
|
|
|
424
431
|
update_only_if_exists = (
|
|
425
432
|
not pipeline.should_save_trigger_in_code_automatically()
|
|
426
433
|
)
|
|
434
|
+
old_trigger_name = old_name if old_name != updated_model.name else None
|
|
427
435
|
|
|
428
436
|
add_or_update_trigger_for_pipeline_and_persist(
|
|
429
437
|
trigger,
|
|
430
438
|
pipeline.uuid,
|
|
431
439
|
update_only_if_exists=update_only_if_exists,
|
|
440
|
+
old_trigger_name=old_trigger_name,
|
|
432
441
|
)
|
|
433
442
|
|
|
434
443
|
return self
|
|
@@ -7,6 +7,7 @@ from mage_ai.data_preparation.models.triggers import (
|
|
|
7
7
|
)
|
|
8
8
|
from mage_ai.orchestration.db import safe_db_query
|
|
9
9
|
from mage_ai.orchestration.db.models.schedules import PipelineSchedule
|
|
10
|
+
from mage_ai.settings.repo import get_repo_path
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
class PipelineTriggerResource(GenericResource):
|
|
@@ -16,7 +17,10 @@ class PipelineTriggerResource(GenericResource):
|
|
|
16
17
|
parent_model = kwargs['parent_model']
|
|
17
18
|
|
|
18
19
|
return self.build_result_set(
|
|
19
|
-
get_triggers_by_pipeline(
|
|
20
|
+
get_triggers_by_pipeline(
|
|
21
|
+
parent_model.uuid,
|
|
22
|
+
repo_path=get_repo_path(context_data=kwargs.get('context_data'), user=user),
|
|
23
|
+
),
|
|
20
24
|
user,
|
|
21
25
|
**kwargs,
|
|
22
26
|
)
|
|
@@ -46,6 +50,7 @@ class PipelineTriggerResource(GenericResource):
|
|
|
46
50
|
sla=pipeline_schedule.sla,
|
|
47
51
|
start_time=pipeline_schedule.start_time,
|
|
48
52
|
status=pipeline_schedule.status,
|
|
53
|
+
token=pipeline_schedule.token,
|
|
49
54
|
variables=pipeline_schedule.variables,
|
|
50
55
|
)
|
|
51
56
|
else:
|
|
@@ -26,8 +26,19 @@ from mage_ai.shared.hash import combine_into, merge_dict
|
|
|
26
26
|
from mage_ai.usage_statistics.logger import UsageStatisticLogger
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
async def build_project(
|
|
30
|
-
|
|
29
|
+
async def build_project(
|
|
30
|
+
repo_config=None,
|
|
31
|
+
repo_path: str = None,
|
|
32
|
+
root_project: bool = False,
|
|
33
|
+
user=None,
|
|
34
|
+
**kwargs,
|
|
35
|
+
):
|
|
36
|
+
project = Project(
|
|
37
|
+
repo_config=repo_config,
|
|
38
|
+
context_data=kwargs.get('context_data'),
|
|
39
|
+
root_project=root_project,
|
|
40
|
+
user=user,
|
|
41
|
+
)
|
|
31
42
|
|
|
32
43
|
model = merge_dict(project.repo_config.to_dict(), dict(
|
|
33
44
|
emr_config=project.emr_config,
|
|
@@ -36,7 +47,7 @@ async def build_project(repo_config=None, root_project: bool = False, **kwargs):
|
|
|
36
47
|
features_override=project.features_override,
|
|
37
48
|
latest_version=await project.latest_version(),
|
|
38
49
|
name=project.name,
|
|
39
|
-
platform_settings=project.platform_settings,
|
|
50
|
+
platform_settings=project.platform_settings(),
|
|
40
51
|
project_uuid=project.project_uuid,
|
|
41
52
|
projects=project.projects(),
|
|
42
53
|
remote_variables_dir=project.remote_variables_dir,
|
|
@@ -137,13 +148,13 @@ class ProjectResource(GenericResource):
|
|
|
137
148
|
@classmethod
|
|
138
149
|
@safe_db_query
|
|
139
150
|
async def member(self, _, user, **kwargs):
|
|
140
|
-
model = await build_project(**kwargs)
|
|
151
|
+
model = await build_project(user=user, **kwargs)
|
|
141
152
|
return self(model, user, **kwargs)
|
|
142
153
|
|
|
143
154
|
@safe_db_query
|
|
144
155
|
async def update(self, payload, **kwargs):
|
|
145
156
|
if payload.get('activate_project'):
|
|
146
|
-
activate_project(payload.get('activate_project'))
|
|
157
|
+
activate_project(payload.get('activate_project'), user=self.current_user)
|
|
147
158
|
|
|
148
159
|
platform_settings = payload.get('platform_settings')
|
|
149
160
|
root_project = payload.get('root_project')
|
|
@@ -151,7 +162,7 @@ class ProjectResource(GenericResource):
|
|
|
151
162
|
|
|
152
163
|
if root_project and platform_settings:
|
|
153
164
|
update_settings(settings=platform_settings)
|
|
154
|
-
self.model = await build_project(repo_config)
|
|
165
|
+
self.model = await build_project(repo_config, user=self.current_user)
|
|
155
166
|
return self
|
|
156
167
|
|
|
157
168
|
data = {}
|
|
@@ -206,7 +217,7 @@ class ProjectResource(GenericResource):
|
|
|
206
217
|
if len(data.keys()) >= 1:
|
|
207
218
|
repo_config.save(**data)
|
|
208
219
|
|
|
209
|
-
self.model = await build_project(repo_config)
|
|
220
|
+
self.model = await build_project(repo_config, user=self.current_user)
|
|
210
221
|
|
|
211
222
|
if should_log_project:
|
|
212
223
|
await UsageStatisticLogger().project_impression()
|
|
@@ -3,9 +3,11 @@ from typing import Dict
|
|
|
3
3
|
from mage_ai.api.errors import ApiError
|
|
4
4
|
from mage_ai.api.resources.GenericResource import GenericResource
|
|
5
5
|
from mage_ai.authentication.oauth.constants import ProviderName
|
|
6
|
-
from mage_ai.data_preparation.git import api
|
|
7
6
|
from mage_ai.data_preparation.git.clients.base import Client as GitClient
|
|
8
|
-
from mage_ai.data_preparation.git.utils import
|
|
7
|
+
from mage_ai.data_preparation.git.utils import (
|
|
8
|
+
get_oauth_access_token_for_user,
|
|
9
|
+
get_provider_from_remote_url,
|
|
10
|
+
)
|
|
9
11
|
|
|
10
12
|
|
|
11
13
|
def pull_request_to_dict(pr) -> Dict:
|
|
@@ -40,7 +42,7 @@ class PullRequestResource(GenericResource):
|
|
|
40
42
|
if remote_url:
|
|
41
43
|
provider = get_provider_from_remote_url(remote_url)
|
|
42
44
|
|
|
43
|
-
access_token =
|
|
45
|
+
access_token = get_oauth_access_token_for_user(user, provider=provider)
|
|
44
46
|
|
|
45
47
|
if access_token:
|
|
46
48
|
arr = GitClient.get_client_for_provider(provider)(
|
|
@@ -84,7 +86,7 @@ class PullRequestResource(GenericResource):
|
|
|
84
86
|
remote_url = payload.get('remote_url')
|
|
85
87
|
provider = get_provider_from_remote_url(remote_url)
|
|
86
88
|
|
|
87
|
-
access_token =
|
|
89
|
+
access_token = get_oauth_access_token_for_user(user, provider=provider)
|
|
88
90
|
if not access_token:
|
|
89
91
|
error.update(
|
|
90
92
|
dict(
|
|
@@ -56,7 +56,7 @@ class SecretResource(DatabaseResource):
|
|
|
56
56
|
@classmethod
|
|
57
57
|
@safe_db_query
|
|
58
58
|
def member(self, pk, user, **kwargs):
|
|
59
|
-
repo_path = get_repo_path()
|
|
59
|
+
repo_path = get_repo_path(user=user)
|
|
60
60
|
model = Secret.query.filter(Secret.repo_name == repo_path, Secret.name == pk).first()
|
|
61
61
|
|
|
62
62
|
if not model:
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import threading
|
|
3
|
+
|
|
1
4
|
from mage_ai.api.resources.GenericResource import GenericResource
|
|
2
5
|
from mage_ai.authentication.permissions.seed import bootstrap_permissions
|
|
3
6
|
|
|
@@ -7,6 +10,10 @@ class SeedResource(GenericResource):
|
|
|
7
10
|
async def create(self, payload, user, **kwargs):
|
|
8
11
|
if payload.get('roles') and payload.get('permissions'):
|
|
9
12
|
policy_names = payload.get('policy_names')
|
|
10
|
-
|
|
13
|
+
t = threading.Thread(
|
|
14
|
+
target=asyncio.run,
|
|
15
|
+
args=(bootstrap_permissions(policy_names=policy_names),),
|
|
16
|
+
)
|
|
17
|
+
t.start()
|
|
11
18
|
|
|
12
19
|
return self({}, user, **kwargs)
|
|
@@ -31,9 +31,10 @@ class StatusResource(GenericResource):
|
|
|
31
31
|
KUBE_NAMESPACE,
|
|
32
32
|
)
|
|
33
33
|
|
|
34
|
+
context_data = kwargs.get('context_data')
|
|
34
35
|
instance_type = None
|
|
35
|
-
|
|
36
|
-
|
|
36
|
+
repo_config = get_repo_config(context_data=context_data, user=user)
|
|
37
|
+
project_type = get_project_type(repo_config=repo_config)
|
|
37
38
|
if repo_config.cluster_type:
|
|
38
39
|
instance_type = repo_config.cluster_type
|
|
39
40
|
elif os.getenv(ECS_CLUSTER_NAME):
|
|
@@ -53,10 +54,24 @@ class StatusResource(GenericResource):
|
|
|
53
54
|
|
|
54
55
|
status = {
|
|
55
56
|
'is_instance_manager': os.getenv(MANAGE_ENV_VAR) == '1',
|
|
56
|
-
'repo_path': get_repo_path(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
57
|
+
'repo_path': get_repo_path(
|
|
58
|
+
context_data=context_data,
|
|
59
|
+
root_project=False,
|
|
60
|
+
user=user),
|
|
61
|
+
'repo_path_relative': get_repo_path(
|
|
62
|
+
context_data=context_data,
|
|
63
|
+
root_project=False,
|
|
64
|
+
absolute_path=False,
|
|
65
|
+
user=user),
|
|
66
|
+
'repo_path_relative_root': get_repo_path(
|
|
67
|
+
context_data=context_data,
|
|
68
|
+
root_project=True,
|
|
69
|
+
absolute_path=False),
|
|
70
|
+
'repo_path_root': get_repo_path(
|
|
71
|
+
context_data=context_data,
|
|
72
|
+
root_project=True),
|
|
73
|
+
'server_process_id': os.getpid(),
|
|
74
|
+
'scheduler_process_id': scheduler_manager.get_scheduler_pid(),
|
|
60
75
|
'scheduler_status': scheduler_manager.get_status(),
|
|
61
76
|
'instance_type': instance_type,
|
|
62
77
|
'disable_pipeline_edit_access': is_disable_pipeline_edit_access(),
|
|
@@ -40,6 +40,7 @@ class SyncResource(GenericResource):
|
|
|
40
40
|
@classmethod
|
|
41
41
|
def collection(self, query, meta, user, **kwargs):
|
|
42
42
|
sync_config = self.get_project_sync_config(user)
|
|
43
|
+
|
|
43
44
|
return self.build_result_set(
|
|
44
45
|
[sync_config],
|
|
45
46
|
user,
|
|
@@ -66,9 +67,9 @@ class SyncResource(GenericResource):
|
|
|
66
67
|
user_payload = self.update_user_settings(user_settings, user=user, repo_name=repo_name)
|
|
67
68
|
UserGitConfig.load(config=user_payload)
|
|
68
69
|
|
|
69
|
-
repo_path = kwargs.get('repo_path') or get_repo_path()
|
|
70
|
+
repo_path = kwargs.get('repo_path') or get_repo_path(user=user)
|
|
70
71
|
user_preferences = user.preferences or {}
|
|
71
|
-
user_git_settings = user.
|
|
72
|
+
user_git_settings = user.get_git_settings(repo_path) or {}
|
|
72
73
|
user_preferences[repo_path] = {
|
|
73
74
|
**user_preferences.get(repo_path, {}),
|
|
74
75
|
'git_settings': {
|
|
@@ -146,16 +147,14 @@ class SyncResource(GenericResource):
|
|
|
146
147
|
|
|
147
148
|
if ssh_public_key:
|
|
148
149
|
secret_name = get_ssh_public_key_secret_name(user=user)
|
|
149
|
-
secret = Secret.query.filter(
|
|
150
|
-
Secret.name == secret_name).one_or_none()
|
|
150
|
+
secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
|
|
151
151
|
if secret:
|
|
152
152
|
secret.delete()
|
|
153
153
|
create_secret(secret_name, ssh_public_key, repo_name=repo_name)
|
|
154
154
|
user_payload['ssh_public_key_secret_name'] = secret_name
|
|
155
155
|
if ssh_private_key:
|
|
156
156
|
secret_name = get_ssh_private_key_secret_name(user=user)
|
|
157
|
-
secret = Secret.query.filter(
|
|
158
|
-
Secret.name == secret_name).one_or_none()
|
|
157
|
+
secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
|
|
159
158
|
if secret:
|
|
160
159
|
secret.delete()
|
|
161
160
|
create_secret(secret_name, ssh_private_key, repo_name=repo_name)
|
|
@@ -164,8 +163,7 @@ class SyncResource(GenericResource):
|
|
|
164
163
|
access_token = user_payload.pop('access_token', None)
|
|
165
164
|
if access_token:
|
|
166
165
|
secret_name = get_access_token_secret_name(user=user)
|
|
167
|
-
secret = Secret.query.filter(
|
|
168
|
-
Secret.name == secret_name).one_or_none()
|
|
166
|
+
secret = Secret.query.filter(Secret.name == secret_name).one_or_none()
|
|
169
167
|
if secret:
|
|
170
168
|
secret.delete()
|
|
171
169
|
create_secret(secret_name, access_token, repo_name=repo_name)
|
|
@@ -2,7 +2,7 @@ from typing import Dict
|
|
|
2
2
|
|
|
3
3
|
from mage_ai.api.errors import ApiError
|
|
4
4
|
from mage_ai.api.resources.GenericResource import GenericResource
|
|
5
|
-
from mage_ai.data_preparation.models.
|
|
5
|
+
from mage_ai.data_preparation.models.variables.constants import VariableType
|
|
6
6
|
from mage_ai.data_preparation.variable_manager import (
|
|
7
7
|
VariableManager,
|
|
8
8
|
delete_global_variable,
|
|
@@ -10,7 +10,7 @@ from mage_ai.data_preparation.variable_manager import (
|
|
|
10
10
|
set_global_variable,
|
|
11
11
|
)
|
|
12
12
|
from mage_ai.orchestration.db import safe_db_query
|
|
13
|
-
from mage_ai.settings.repo import get_variables_dir
|
|
13
|
+
from mage_ai.settings.repo import get_repo_path, get_variables_dir
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
def get_variable_value(
|
|
@@ -41,6 +41,9 @@ class VariableResource(GenericResource):
|
|
|
41
41
|
@classmethod
|
|
42
42
|
@safe_db_query
|
|
43
43
|
def collection(self, query, meta, user, **kwargs):
|
|
44
|
+
context_data = kwargs.get('context_data')
|
|
45
|
+
repo_path = get_repo_path(context_data=context_data, user=user)
|
|
46
|
+
variables_dir = get_variables_dir(repo_path=repo_path)
|
|
44
47
|
pipeline_uuid = kwargs['parent_model'].uuid
|
|
45
48
|
|
|
46
49
|
global_only = query.get('global_only', [False])
|
|
@@ -49,12 +52,13 @@ class VariableResource(GenericResource):
|
|
|
49
52
|
|
|
50
53
|
# Get global variables from project's path
|
|
51
54
|
global_variables = [
|
|
52
|
-
dict(
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
dict(uuid=uuid, type=str(type(value)), value=value)
|
|
56
|
+
for uuid, value in get_global_variables(
|
|
57
|
+
pipeline_uuid,
|
|
58
|
+
context_data=context_data,
|
|
59
|
+
repo_path=repo_path,
|
|
60
|
+
variables_dir=variables_dir,
|
|
61
|
+
).items()
|
|
58
62
|
]
|
|
59
63
|
global_variables_arr = [
|
|
60
64
|
dict(
|
|
@@ -65,7 +69,10 @@ class VariableResource(GenericResource):
|
|
|
65
69
|
]
|
|
66
70
|
variables = global_variables_arr
|
|
67
71
|
if not global_only:
|
|
68
|
-
variable_manager = VariableManager(
|
|
72
|
+
variable_manager = VariableManager(
|
|
73
|
+
repo_path=repo_path,
|
|
74
|
+
variables_dir=variables_dir,
|
|
75
|
+
)
|
|
69
76
|
variables_dict = variable_manager.get_variables_by_pipeline(pipeline_uuid)
|
|
70
77
|
variables = [
|
|
71
78
|
dict(
|
|
@@ -77,12 +84,14 @@ class VariableResource(GenericResource):
|
|
|
77
84
|
pipeline_uuid,
|
|
78
85
|
uuid,
|
|
79
86
|
var,
|
|
80
|
-
)
|
|
87
|
+
)
|
|
88
|
+
for var in arr
|
|
81
89
|
# Not return printed outputs
|
|
82
90
|
if var == 'df' or var.startswith('output')
|
|
83
91
|
],
|
|
84
92
|
)
|
|
85
|
-
for uuid, arr in variables_dict.items()
|
|
93
|
+
for uuid, arr in variables_dict.items()
|
|
94
|
+
if uuid != 'global'
|
|
86
95
|
] + global_variables_arr
|
|
87
96
|
|
|
88
97
|
return self.build_result_set(
|
|
@@ -100,7 +109,9 @@ class VariableResource(GenericResource):
|
|
|
100
109
|
|
|
101
110
|
variable_uuid = payload.get('name')
|
|
102
111
|
if not variable_uuid.isidentifier():
|
|
103
|
-
error.update(
|
|
112
|
+
error.update(
|
|
113
|
+
message=f'Invalid variable name syntax for variable name {variable_uuid}.'
|
|
114
|
+
)
|
|
104
115
|
raise ApiError(error)
|
|
105
116
|
|
|
106
117
|
variable_value = payload.get('value')
|
|
@@ -116,13 +127,17 @@ class VariableResource(GenericResource):
|
|
|
116
127
|
|
|
117
128
|
global_variables = get_global_variables(pipeline_uuid)
|
|
118
129
|
|
|
119
|
-
return self(
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
130
|
+
return self(
|
|
131
|
+
dict(
|
|
132
|
+
block=dict(uuid='global'),
|
|
133
|
+
name=variable_uuid,
|
|
134
|
+
pipeline=dict(uuid=pipeline_uuid),
|
|
135
|
+
value=variable_value,
|
|
136
|
+
variables=list(global_variables.keys()),
|
|
137
|
+
),
|
|
138
|
+
user,
|
|
139
|
+
**kwargs,
|
|
140
|
+
)
|
|
126
141
|
|
|
127
142
|
@classmethod
|
|
128
143
|
@safe_db_query
|
|
@@ -138,6 +153,9 @@ class VariableResource(GenericResource):
|
|
|
138
153
|
variable_uuid = payload.get('name')
|
|
139
154
|
if not variable_uuid.isidentifier():
|
|
140
155
|
error.update(message=f'Invalid variable name syntax for variable name {self.name}.')
|
|
156
|
+
error.update(
|
|
157
|
+
message=f'Invalid variable name syntax for variable name {self.name}.'
|
|
158
|
+
)
|
|
141
159
|
raise ApiError(error)
|
|
142
160
|
|
|
143
161
|
variable_value = payload.get('value')
|
|
@@ -153,13 +171,15 @@ class VariableResource(GenericResource):
|
|
|
153
171
|
|
|
154
172
|
global_variables = get_global_variables(pipeline_uuid)
|
|
155
173
|
|
|
156
|
-
self.model.update(
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
174
|
+
self.model.update(
|
|
175
|
+
dict(
|
|
176
|
+
block=dict(uuid='global'),
|
|
177
|
+
name=variable_uuid,
|
|
178
|
+
pipeline=dict(uuid=pipeline_uuid),
|
|
179
|
+
value=variable_value,
|
|
180
|
+
variables=list(global_variables.keys()),
|
|
181
|
+
)
|
|
182
|
+
)
|
|
163
183
|
|
|
164
184
|
return self
|
|
165
185
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import inspect
|
|
1
2
|
import urllib.parse
|
|
2
3
|
from typing import Dict
|
|
3
4
|
|
|
@@ -41,11 +42,14 @@ class VersionControlProjectResource(VersionControlErrors, AsyncBaseResource):
|
|
|
41
42
|
model = self.get_model(pk)
|
|
42
43
|
|
|
43
44
|
sync_resource = SyncResource.member(None, user, repo_path=model.repo_path)
|
|
44
|
-
|
|
45
|
+
result = SyncResource.presenter_class()(
|
|
45
46
|
sync_resource,
|
|
46
47
|
user,
|
|
47
48
|
**kwargs,
|
|
48
49
|
).present()
|
|
50
|
+
if result and inspect.isawaitable(result):
|
|
51
|
+
result = await result
|
|
52
|
+
model.sync_config = result
|
|
49
53
|
|
|
50
54
|
res = self(model, user, **kwargs)
|
|
51
55
|
res.validate_output()
|
|
@@ -71,11 +75,14 @@ class VersionControlProjectResource(VersionControlErrors, AsyncBaseResource):
|
|
|
71
75
|
self.current_user,
|
|
72
76
|
repo_path=self.model.repo_path,
|
|
73
77
|
)
|
|
74
|
-
|
|
78
|
+
result = SyncResource.presenter_class()(
|
|
75
79
|
sync_resource,
|
|
76
80
|
self.current_user,
|
|
77
81
|
**kwargs,
|
|
78
82
|
).present()
|
|
83
|
+
if result and inspect.isawaitable(result):
|
|
84
|
+
result = await result
|
|
85
|
+
self.model.sync_config = result
|
|
79
86
|
|
|
80
87
|
self.model.update_attributes()
|
|
81
88
|
self.validate_output()
|
|
@@ -39,7 +39,7 @@ class WidgetResource(GenericResource):
|
|
|
39
39
|
resource = Widget.create(
|
|
40
40
|
payload.get('name') or payload.get('uuid'),
|
|
41
41
|
payload.get('type'),
|
|
42
|
-
get_repo_path(),
|
|
42
|
+
get_repo_path(user=user),
|
|
43
43
|
config=payload.get('config'),
|
|
44
44
|
language=payload.get('language'),
|
|
45
45
|
pipeline=pipeline,
|
|
@@ -123,14 +123,15 @@ class WorkspaceResource(GenericResource):
|
|
|
123
123
|
error = ApiError.RESOURCE_ERROR.copy()
|
|
124
124
|
try:
|
|
125
125
|
action = payload.pop('action')
|
|
126
|
-
args = ignore_keys(payload, ['name', 'cluster_type'])
|
|
127
126
|
if action == 'stop':
|
|
128
|
-
workspace.stop(**
|
|
127
|
+
workspace.stop(**payload)
|
|
129
128
|
elif action == 'resume':
|
|
130
|
-
workspace.resume(**
|
|
129
|
+
workspace.resume(**payload)
|
|
130
|
+
elif action == 'patch':
|
|
131
|
+
workspace.update(payload)
|
|
131
132
|
elif action == 'add_to_ingress':
|
|
132
133
|
if isinstance(workspace, KubernetesWorkspace):
|
|
133
|
-
workspace.add_to_ingress(**
|
|
134
|
+
workspace.add_to_ingress(**payload)
|
|
134
135
|
else:
|
|
135
136
|
raise Exception('This workspace does not support ingress.')
|
|
136
137
|
except Exception as ex:
|
|
@@ -162,7 +163,7 @@ class WorkspaceResource(GenericResource):
|
|
|
162
163
|
raise ApiError(error)
|
|
163
164
|
|
|
164
165
|
if project_type == ProjectType.MAIN and subproject:
|
|
165
|
-
repo_path = get_repo_path()
|
|
166
|
+
repo_path = get_repo_path(user=self.current_user)
|
|
166
167
|
projects_folder = os.path.join(repo_path, 'projects')
|
|
167
168
|
projects = [
|
|
168
169
|
f.name.split('.')[0]
|