mage-ai 0.9.69__py3-none-any.whl → 0.9.71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mage-ai might be problematic. Click here for more details.
- mage_ai/ai/utils/xgboost.py +222 -0
- mage_ai/api/errors.py +37 -25
- mage_ai/api/operations/base.py +13 -1
- mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
- mage_ai/api/policies/BackfillPolicy.py +1 -0
- mage_ai/api/policies/BlockOutputPolicy.py +40 -17
- mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
- mage_ai/api/policies/KernelPolicy.py +55 -32
- mage_ai/api/policies/KernelProcessPolicy.py +56 -0
- mage_ai/api/policies/OutputPolicy.py +73 -41
- mage_ai/api/policies/PipelinePolicy.py +206 -137
- mage_ai/api/policies/WorkspacePolicy.py +1 -0
- mage_ai/api/presenters/BackfillPresenter.py +1 -0
- mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
- mage_ai/api/presenters/BlockPresenter.py +1 -1
- mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
- mage_ai/api/presenters/KernelPresenter.py +5 -26
- mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
- mage_ai/api/presenters/PipelinePresenter.py +18 -5
- mage_ai/api/presenters/StatusPresenter.py +2 -0
- mage_ai/api/presenters/SyncPresenter.py +25 -0
- mage_ai/api/resources/AutocompleteItemResource.py +1 -1
- mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
- mage_ai/api/resources/BlockOutputResource.py +42 -9
- mage_ai/api/resources/BlockResource.py +4 -3
- mage_ai/api/resources/BlockRunResource.py +27 -22
- mage_ai/api/resources/ClusterResource.py +4 -1
- mage_ai/api/resources/CustomTemplateResource.py +34 -14
- mage_ai/api/resources/DataProviderResource.py +1 -1
- mage_ai/api/resources/ExecutionStateResource.py +3 -1
- mage_ai/api/resources/FileContentResource.py +8 -2
- mage_ai/api/resources/FileResource.py +10 -4
- mage_ai/api/resources/FileVersionResource.py +3 -1
- mage_ai/api/resources/GitBranchResource.py +101 -31
- mage_ai/api/resources/GitCustomBranchResource.py +29 -1
- mage_ai/api/resources/GlobalDataProductResource.py +44 -7
- mage_ai/api/resources/GlobalHookResource.py +4 -1
- mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
- mage_ai/api/resources/IntegrationSourceResource.py +8 -4
- mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
- mage_ai/api/resources/KernelProcessResource.py +44 -0
- mage_ai/api/resources/KernelResource.py +25 -3
- mage_ai/api/resources/OauthResource.py +1 -1
- mage_ai/api/resources/OutputResource.py +33 -11
- mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
- mage_ai/api/resources/PipelineInteractionResource.py +31 -15
- mage_ai/api/resources/PipelineResource.py +258 -125
- mage_ai/api/resources/PipelineRunResource.py +52 -7
- mage_ai/api/resources/PipelineScheduleResource.py +11 -2
- mage_ai/api/resources/PipelineTriggerResource.py +6 -1
- mage_ai/api/resources/ProjectResource.py +18 -7
- mage_ai/api/resources/PullRequestResource.py +6 -4
- mage_ai/api/resources/SecretResource.py +1 -1
- mage_ai/api/resources/SeedResource.py +8 -1
- mage_ai/api/resources/StatusResource.py +21 -6
- mage_ai/api/resources/SyncResource.py +6 -8
- mage_ai/api/resources/VariableResource.py +46 -26
- mage_ai/api/resources/VersionControlProjectResource.py +9 -2
- mage_ai/api/resources/WidgetResource.py +1 -1
- mage_ai/api/resources/WorkspaceResource.py +6 -5
- mage_ai/api/views.py +47 -40
- mage_ai/authentication/permissions/seed.py +16 -2
- mage_ai/authentication/providers/oidc.py +21 -1
- mage_ai/autocomplete/utils.py +13 -9
- mage_ai/cache/base.py +1 -1
- mage_ai/cache/block.py +18 -12
- mage_ai/cache/block_action_object/__init__.py +33 -5
- mage_ai/cache/file.py +22 -19
- mage_ai/cache/pipeline.py +18 -12
- mage_ai/cli/main.py +1 -0
- mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
- mage_ai/cluster_manager/config.py +2 -2
- mage_ai/cluster_manager/kubernetes/workload_manager.py +52 -1
- mage_ai/cluster_manager/manage.py +1 -1
- mage_ai/cluster_manager/workspace/base.py +7 -1
- mage_ai/cluster_manager/workspace/kubernetes.py +22 -1
- mage_ai/command_center/applications/factory.py +10 -7
- mage_ai/command_center/applications/utils.py +2 -2
- mage_ai/command_center/files/factory.py +17 -15
- mage_ai/command_center/presenters/text.py +1 -1
- mage_ai/command_center/utils.py +25 -13
- mage_ai/data/__init__.py +0 -0
- mage_ai/data/constants.py +45 -0
- mage_ai/data/models/__init__.py +0 -0
- mage_ai/data/models/base.py +119 -0
- mage_ai/data/models/constants.py +1 -0
- mage_ai/data/models/generator.py +115 -0
- mage_ai/data/models/manager.py +168 -0
- mage_ai/data/models/pyarrow/__init__.py +0 -0
- mage_ai/data/models/pyarrow/record_batch.py +55 -0
- mage_ai/data/models/pyarrow/shared.py +21 -0
- mage_ai/data/models/pyarrow/table.py +8 -0
- mage_ai/data/models/reader.py +103 -0
- mage_ai/data/models/utils.py +59 -0
- mage_ai/data/models/writer.py +91 -0
- mage_ai/data/tabular/__init__.py +0 -0
- mage_ai/data/tabular/constants.py +23 -0
- mage_ai/data/tabular/mocks.py +19 -0
- mage_ai/data/tabular/models.py +126 -0
- mage_ai/data/tabular/reader.py +602 -0
- mage_ai/data/tabular/utils.py +102 -0
- mage_ai/data/tabular/writer.py +266 -0
- mage_ai/data/variables/__init__.py +0 -0
- mage_ai/data/variables/wrapper.py +54 -0
- mage_ai/data_cleaner/analysis/charts.py +61 -39
- mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
- mage_ai/data_cleaner/estimators/encoders.py +5 -2
- mage_ai/data_integrations/utils/scheduler.py +16 -11
- mage_ai/data_preparation/decorators.py +1 -0
- mage_ai/data_preparation/executors/block_executor.py +237 -155
- mage_ai/data_preparation/executors/k8s_block_executor.py +30 -7
- mage_ai/data_preparation/executors/k8s_pipeline_executor.py +30 -7
- mage_ai/data_preparation/executors/streaming_pipeline_executor.py +2 -2
- mage_ai/data_preparation/git/__init__.py +77 -29
- mage_ai/data_preparation/git/api.py +69 -8
- mage_ai/data_preparation/git/utils.py +64 -34
- mage_ai/data_preparation/logging/logger_manager.py +4 -3
- mage_ai/data_preparation/models/block/__init__.py +1562 -879
- mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
- mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
- mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
- mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
- mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
- mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
- mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
- mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
- mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
- mage_ai/data_preparation/models/block/dynamic/utils.py +295 -167
- mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
- mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
- mage_ai/data_preparation/models/block/extension/utils.py +10 -1
- mage_ai/data_preparation/models/block/global_data_product/__init__.py +35 -3
- mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
- mage_ai/data_preparation/models/block/outputs.py +722 -0
- mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
- mage_ai/data_preparation/models/block/r/__init__.py +56 -38
- mage_ai/data_preparation/models/block/remote/__init__.py +0 -0
- mage_ai/data_preparation/models/block/remote/models.py +58 -0
- mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
- mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
- mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
- mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
- mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
- mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
- mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
- mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
- mage_ai/data_preparation/models/block/utils.py +164 -69
- mage_ai/data_preparation/models/constants.py +21 -14
- mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
- mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
- mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
- mage_ai/data_preparation/models/file.py +41 -28
- mage_ai/data_preparation/models/global_data_product/__init__.py +100 -58
- mage_ai/data_preparation/models/global_hooks/models.py +1 -0
- mage_ai/data_preparation/models/interfaces.py +29 -0
- mage_ai/data_preparation/models/pipeline.py +374 -185
- mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
- mage_ai/data_preparation/models/pipelines/seed.py +1 -1
- mage_ai/data_preparation/models/project/__init__.py +66 -18
- mage_ai/data_preparation/models/project/constants.py +2 -0
- mage_ai/data_preparation/models/triggers/__init__.py +124 -26
- mage_ai/data_preparation/models/utils.py +467 -17
- mage_ai/data_preparation/models/variable.py +1028 -137
- mage_ai/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/variables/cache.py +149 -0
- mage_ai/data_preparation/models/variables/constants.py +72 -0
- mage_ai/data_preparation/models/variables/summarizer.py +336 -0
- mage_ai/data_preparation/models/variables/utils.py +77 -0
- mage_ai/data_preparation/models/widget/__init__.py +63 -41
- mage_ai/data_preparation/models/widget/charts.py +40 -27
- mage_ai/data_preparation/models/widget/constants.py +2 -0
- mage_ai/data_preparation/models/widget/utils.py +3 -3
- mage_ai/data_preparation/preferences.py +3 -3
- mage_ai/data_preparation/repo_manager.py +55 -21
- mage_ai/data_preparation/storage/base_storage.py +2 -2
- mage_ai/data_preparation/storage/gcs_storage.py +7 -4
- mage_ai/data_preparation/storage/local_storage.py +18 -9
- mage_ai/data_preparation/storage/s3_storage.py +5 -2
- mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
- mage_ai/data_preparation/variable_manager.py +281 -76
- mage_ai/io/base.py +3 -2
- mage_ai/io/bigquery.py +1 -0
- mage_ai/io/redshift.py +7 -5
- mage_ai/kernels/__init__.py +0 -0
- mage_ai/kernels/models.py +188 -0
- mage_ai/kernels/utils.py +169 -0
- mage_ai/orchestration/concurrency.py +6 -2
- mage_ai/orchestration/db/__init__.py +1 -0
- mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
- mage_ai/orchestration/db/migrations/versions/42a14d6143f1_update_token_column_type.py +54 -0
- mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
- mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
- mage_ai/orchestration/db/models/oauth.py +12 -18
- mage_ai/orchestration/db/models/projects.py +10 -0
- mage_ai/orchestration/db/models/schedules.py +225 -187
- mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
- mage_ai/orchestration/db/models/utils.py +46 -5
- mage_ai/orchestration/metrics/pipeline_run.py +8 -9
- mage_ai/orchestration/notification/sender.py +38 -15
- mage_ai/orchestration/pipeline_scheduler_original.py +64 -33
- mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
- mage_ai/orchestration/run_status_checker.py +11 -4
- mage_ai/orchestration/triggers/api.py +41 -2
- mage_ai/orchestration/triggers/global_data_product.py +9 -4
- mage_ai/orchestration/triggers/utils.py +10 -1
- mage_ai/orchestration/utils/resources.py +3 -0
- mage_ai/presenters/charts/data_sources/base.py +4 -2
- mage_ai/presenters/charts/data_sources/block.py +15 -9
- mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
- mage_ai/presenters/charts/data_sources/constants.py +1 -0
- mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
- mage_ai/presenters/interactions/models.py +11 -7
- mage_ai/presenters/pages/loaders/pipelines.py +5 -3
- mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
- mage_ai/presenters/utils.py +2 -0
- mage_ai/server/api/blocks.py +2 -1
- mage_ai/server/api/downloads.py +9 -2
- mage_ai/server/api/runs.py +151 -0
- mage_ai/server/api/triggers.py +3 -1
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +8 -8
- mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
- mage_ai/server/frontend_dist/block-layout.html +3 -3
- mage_ai/server/frontend_dist/compute.html +6 -6
- mage_ai/server/frontend_dist/files.html +6 -6
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-data-products.html +6 -6
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/index.html +3 -3
- mage_ai/server/frontend_dist/manage/files.html +6 -6
- mage_ai/server/frontend_dist/manage/settings.html +6 -6
- mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist/manage/users.html +6 -6
- mage_ai/server/frontend_dist/manage.html +6 -6
- mage_ai/server/frontend_dist/oauth.html +5 -5
- mage_ai/server/frontend_dist/overview.html +6 -6
- mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist/pipelines.html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist/settings.html +3 -3
- mage_ai/server/frontend_dist/sign-in.html +15 -15
- mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist/templates.html +6 -6
- mage_ai/server/frontend_dist/terminal.html +6 -6
- mage_ai/server/frontend_dist/test.html +3 -3
- mage_ai/server/frontend_dist/triggers.html +6 -6
- mage_ai/server/frontend_dist/version-control.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/404.html +8 -8
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +15 -15
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
- mage_ai/server/kernel_output_parser.py +4 -1
- mage_ai/server/scheduler_manager.py +12 -1
- mage_ai/server/server.py +69 -42
- mage_ai/server/utils/custom_output.py +284 -0
- mage_ai/server/utils/execute_custom_code.py +245 -0
- mage_ai/server/utils/output_display.py +123 -289
- mage_ai/server/websocket_server.py +116 -69
- mage_ai/services/aws/ecs/ecs.py +1 -0
- mage_ai/services/k8s/config.py +27 -4
- mage_ai/services/k8s/job_manager.py +6 -1
- mage_ai/services/k8s/utils.py +97 -0
- mage_ai/services/ssh/aws/emr/utils.py +8 -8
- mage_ai/settings/keys/auth.py +1 -0
- mage_ai/settings/platform/__init__.py +159 -38
- mage_ai/settings/platform/constants.py +5 -0
- mage_ai/settings/platform/utils.py +53 -10
- mage_ai/settings/repo.py +26 -12
- mage_ai/settings/server.py +128 -37
- mage_ai/shared/array.py +24 -1
- mage_ai/shared/complex.py +45 -0
- mage_ai/shared/config.py +2 -1
- mage_ai/shared/custom_logger.py +11 -0
- mage_ai/shared/dates.py +10 -6
- mage_ai/shared/files.py +63 -8
- mage_ai/shared/hash.py +33 -9
- mage_ai/shared/io.py +9 -5
- mage_ai/shared/models.py +82 -24
- mage_ai/shared/outputs.py +87 -0
- mage_ai/shared/parsers.py +144 -13
- mage_ai/shared/path_fixer.py +11 -7
- mage_ai/shared/singletons/__init__.py +0 -0
- mage_ai/shared/singletons/base.py +47 -0
- mage_ai/shared/singletons/memory.py +38 -0
- mage_ai/shared/strings.py +34 -1
- mage_ai/shared/yaml.py +24 -0
- mage_ai/streaming/sinks/oracledb.py +57 -0
- mage_ai/streaming/sinks/sink_factory.py +4 -0
- mage_ai/system/__init__.py +0 -0
- mage_ai/system/constants.py +14 -0
- mage_ai/system/memory/__init__.py +0 -0
- mage_ai/system/memory/constants.py +1 -0
- mage_ai/system/memory/manager.py +174 -0
- mage_ai/system/memory/presenters.py +158 -0
- mage_ai/system/memory/process.py +216 -0
- mage_ai/system/memory/samples.py +13 -0
- mage_ai/system/memory/utils.py +656 -0
- mage_ai/system/memory/wrappers.py +177 -0
- mage_ai/system/models.py +58 -0
- mage_ai/system/storage/__init__.py +0 -0
- mage_ai/system/storage/utils.py +29 -0
- mage_ai/tests/api/endpoints/mixins.py +2 -2
- mage_ai/tests/api/endpoints/test_blocks.py +2 -1
- mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
- mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
- mage_ai/tests/api/endpoints/test_projects.py +2 -1
- mage_ai/tests/api/operations/base/mixins.py +1 -1
- mage_ai/tests/api/operations/base/test_base.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
- mage_ai/tests/api/operations/test_syncs.py +6 -4
- mage_ai/tests/api/resources/test_pipeline_resource.py +11 -4
- mage_ai/tests/authentication/oauth/test_utils.py +1 -1
- mage_ai/tests/authentication/providers/test_oidc.py +59 -0
- mage_ai/tests/base_test.py +2 -2
- mage_ai/tests/data/__init__.py +0 -0
- mage_ai/tests/data/models/__init__.py +0 -0
- mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
- mage_ai/tests/data_preparation/git/test_git.py +4 -1
- mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
- mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
- mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
- mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
- mage_ai/tests/data_preparation/models/block/test_global_data_product.py +5 -2
- mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
- mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
- mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
- mage_ai/tests/data_preparation/models/test_block.py +115 -120
- mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
- mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
- mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
- mage_ai/tests/data_preparation/models/test_project.py +4 -1
- mage_ai/tests/data_preparation/models/test_utils.py +80 -0
- mage_ai/tests/data_preparation/models/test_variable.py +242 -69
- mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
- mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
- mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
- mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
- mage_ai/tests/factory.py +64 -43
- mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
- mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
- mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
- mage_ai/tests/orchestration/triggers/test_global_data_product.py +141 -138
- mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
- mage_ai/tests/server/test_server.py +19 -0
- mage_ai/tests/services/k8s/test_job_manager.py +27 -6
- mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
- mage_ai/tests/test_shared.py +61 -0
- mage_ai/usage_statistics/logger.py +7 -2
- mage_ai/utils/code.py +33 -19
- mage_ai/version_control/branch/utils.py +2 -1
- mage_ai/version_control/models.py +3 -2
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/METADATA +6 -3
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/RECORD +555 -454
- mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
- mage_ai/server/frontend_dist/_next/static/_krrrgup_C-dPOpX36S8I/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/KLL5mirre9d7_ZeEpaw3s/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
- mage_ai/shared/memory.py +0 -90
- mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
- /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
- /mage_ai/server/frontend_dist/_next/static/{_krrrgup_C-dPOpX36S8I → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{KLL5mirre9d7_ZeEpaw3s → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
- /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
|
@@ -62,29 +62,38 @@ from mage_ai.usage_statistics.logger import UsageStatisticLogger
|
|
|
62
62
|
|
|
63
63
|
|
|
64
64
|
@safe_db_query
|
|
65
|
-
def query_pipeline_schedules(
|
|
65
|
+
def query_pipeline_schedules(
|
|
66
|
+
pipeline_uuids: List[str],
|
|
67
|
+
context_data: Dict = None,
|
|
68
|
+
repo_path: str = None,
|
|
69
|
+
):
|
|
66
70
|
a = aliased(PipelineSchedule, name='a')
|
|
67
71
|
result = (
|
|
68
|
-
PipelineSchedule.
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
filter(
|
|
72
|
+
PipelineSchedule.select(
|
|
73
|
+
*[
|
|
74
|
+
a.created_at,
|
|
75
|
+
a.id,
|
|
76
|
+
a.name,
|
|
77
|
+
a.pipeline_uuid,
|
|
78
|
+
a.schedule_interval,
|
|
79
|
+
a.schedule_type,
|
|
80
|
+
a.start_time,
|
|
81
|
+
a.status,
|
|
82
|
+
a.updated_at,
|
|
83
|
+
]
|
|
84
|
+
).filter(
|
|
81
85
|
a.pipeline_uuid.in_(pipeline_uuids),
|
|
82
86
|
or_(
|
|
83
|
-
a.repo_path.in_(
|
|
84
|
-
|
|
85
|
-
|
|
87
|
+
a.repo_path.in_(
|
|
88
|
+
Project(
|
|
89
|
+
context_data=context_data,
|
|
90
|
+
repo_path=repo_path,
|
|
91
|
+
).repo_path_for_database_query(
|
|
92
|
+
'pipeline_schedules',
|
|
93
|
+
)
|
|
94
|
+
),
|
|
86
95
|
a.repo_path.is_(None),
|
|
87
|
-
)
|
|
96
|
+
),
|
|
88
97
|
)
|
|
89
98
|
).all()
|
|
90
99
|
return group_by(lambda x: x.pipeline_uuid, result)
|
|
@@ -94,6 +103,7 @@ class PipelineResource(BaseResource):
|
|
|
94
103
|
@classmethod
|
|
95
104
|
@safe_db_query
|
|
96
105
|
async def collection(self, query, meta, user, **kwargs):
|
|
106
|
+
context_data = kwargs.get('context_data')
|
|
97
107
|
limit = (meta or {}).get(META_KEY_LIMIT, None)
|
|
98
108
|
if limit is not None:
|
|
99
109
|
limit = int(limit)
|
|
@@ -119,6 +129,16 @@ class PipelineResource(BaseResource):
|
|
|
119
129
|
if include_schedules:
|
|
120
130
|
include_schedules = include_schedules[0]
|
|
121
131
|
|
|
132
|
+
repo_path = query.get('repo_path', [None])
|
|
133
|
+
if repo_path:
|
|
134
|
+
repo_path = repo_path[0]
|
|
135
|
+
if not repo_path:
|
|
136
|
+
repo_path = get_repo_path(
|
|
137
|
+
context_data=context_data,
|
|
138
|
+
root_project=False,
|
|
139
|
+
user=user,
|
|
140
|
+
)
|
|
141
|
+
|
|
122
142
|
search_query = query.get('search', [None])
|
|
123
143
|
if search_query:
|
|
124
144
|
search_query = search_query[0]
|
|
@@ -148,9 +168,12 @@ class PipelineResource(BaseResource):
|
|
|
148
168
|
|
|
149
169
|
history_by_pipeline_uuid = {}
|
|
150
170
|
if from_history_days is not None and is_number(from_history_days):
|
|
151
|
-
timestamp_start = (
|
|
152
|
-
|
|
153
|
-
|
|
171
|
+
timestamp_start = (
|
|
172
|
+
datetime.utcnow()
|
|
173
|
+
- timedelta(
|
|
174
|
+
hours=24 * int(from_history_days),
|
|
175
|
+
)
|
|
176
|
+
).timestamp()
|
|
154
177
|
history = load_pipelines_detail(timestamp_start=timestamp_start)
|
|
155
178
|
history = sorted(
|
|
156
179
|
history,
|
|
@@ -174,21 +197,29 @@ class PipelineResource(BaseResource):
|
|
|
174
197
|
|
|
175
198
|
if NO_TAGS_QUERY in tags:
|
|
176
199
|
pipeline_uuids_with_tags = set(cache.get_all_pipeline_uuids_with_tags())
|
|
177
|
-
all_pipeline_uuids = set(
|
|
178
|
-
|
|
200
|
+
all_pipeline_uuids = set(
|
|
201
|
+
Pipeline.get_all_pipelines(repo_path=repo_path)
|
|
202
|
+
)
|
|
203
|
+
pipeline_uuids_without_tags = list(
|
|
204
|
+
all_pipeline_uuids - pipeline_uuids_with_tags
|
|
205
|
+
)
|
|
179
206
|
pipeline_uuids = pipeline_uuids + pipeline_uuids_without_tags
|
|
180
207
|
else:
|
|
181
|
-
pipeline_uuids = Pipeline.get_all_pipelines(
|
|
208
|
+
pipeline_uuids = Pipeline.get_all_pipelines(repo_path=repo_path)
|
|
182
209
|
|
|
183
210
|
if search_query:
|
|
184
|
-
pipeline_uuids = list(
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
211
|
+
pipeline_uuids = list(
|
|
212
|
+
filter(
|
|
213
|
+
lambda x: search_query.lower() in x.lower()
|
|
214
|
+
or search_query.lower() in x.lower().split(' '),
|
|
215
|
+
pipeline_uuids,
|
|
216
|
+
)
|
|
217
|
+
)
|
|
189
218
|
|
|
190
219
|
total_count = len(pipeline_uuids)
|
|
191
|
-
await UsageStatisticLogger(
|
|
220
|
+
await UsageStatisticLogger(
|
|
221
|
+
context_data=context_data, repo_path=repo_path,
|
|
222
|
+
).pipelines_impression(lambda: total_count)
|
|
192
223
|
|
|
193
224
|
if not sorts:
|
|
194
225
|
pipeline_uuids = sorted(pipeline_uuids, reverse=reverse_sort)
|
|
@@ -199,17 +230,20 @@ class PipelineResource(BaseResource):
|
|
|
199
230
|
if offset:
|
|
200
231
|
pipeline_uuids = pipeline_uuids[offset:]
|
|
201
232
|
if limit:
|
|
202
|
-
pipeline_uuids = pipeline_uuids[:(limit + 1)]
|
|
233
|
+
pipeline_uuids = pipeline_uuids[: (limit + 1)]
|
|
203
234
|
offset_limit_applied = True
|
|
204
235
|
|
|
205
236
|
cache = await PipelineCache.initialize_cache()
|
|
206
237
|
|
|
207
238
|
async def get_pipeline(uuid: str) -> Pipeline:
|
|
208
239
|
try:
|
|
209
|
-
return await Pipeline.get_async(uuid)
|
|
240
|
+
return await Pipeline.get_async(uuid, repo_path=repo_path)
|
|
210
241
|
except Exception as err:
|
|
211
242
|
err_message = f'Error loading pipeline {uuid}: {err}.'
|
|
212
|
-
if
|
|
243
|
+
if (
|
|
244
|
+
err.__class__.__name__ == 'OSError'
|
|
245
|
+
and 'Too many open files' in err.strerror
|
|
246
|
+
):
|
|
213
247
|
raise Exception(err_message)
|
|
214
248
|
else:
|
|
215
249
|
print(err_message)
|
|
@@ -217,10 +251,13 @@ class PipelineResource(BaseResource):
|
|
|
217
251
|
|
|
218
252
|
def get_pipeline_with_config(uuid, config: Dict) -> Pipeline:
|
|
219
253
|
try:
|
|
220
|
-
return Pipeline(uuid, config=config)
|
|
254
|
+
return Pipeline(uuid, config=config, repo_path=repo_path)
|
|
221
255
|
except Exception as err:
|
|
222
256
|
err_message = f'Error loading pipeline sync {uuid}: {err}.'
|
|
223
|
-
if
|
|
257
|
+
if (
|
|
258
|
+
err.__class__.__name__ == 'OSError'
|
|
259
|
+
and 'Too many open files' in err.strerror
|
|
260
|
+
):
|
|
224
261
|
raise Exception(err_message)
|
|
225
262
|
else:
|
|
226
263
|
print(err_message)
|
|
@@ -233,13 +270,16 @@ class PipelineResource(BaseResource):
|
|
|
233
270
|
for pipeline_dict in cache.get_models(types=pipeline_types):
|
|
234
271
|
pipeline_uuid_from_cache = pipeline_dict['pipeline']['uuid']
|
|
235
272
|
if pipeline_uuid_from_cache in pipeline_uuids:
|
|
236
|
-
pipelines.append(
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
273
|
+
pipelines.append(
|
|
274
|
+
Pipeline(
|
|
275
|
+
pipeline_uuid_from_cache,
|
|
276
|
+
config=pipeline_dict['pipeline'],
|
|
277
|
+
repo_path=repo_path,
|
|
278
|
+
)
|
|
279
|
+
)
|
|
240
280
|
else:
|
|
241
281
|
for uuid in pipeline_uuids:
|
|
242
|
-
pipeline_dict = cache.get_model(dict(uuid=uuid))
|
|
282
|
+
pipeline_dict = cache.get_model(dict(uuid=uuid, repo_path=repo_path))
|
|
243
283
|
if pipeline_dict and pipeline_dict.get('pipeline'):
|
|
244
284
|
pipeline = get_pipeline_with_config(uuid, pipeline_dict['pipeline'])
|
|
245
285
|
if pipeline:
|
|
@@ -247,10 +287,13 @@ class PipelineResource(BaseResource):
|
|
|
247
287
|
else:
|
|
248
288
|
# Add pipeline with type "invalid" so pipeline with invalid config
|
|
249
289
|
# can still be displayed in UI and visible to user
|
|
250
|
-
pipelines.append(
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
290
|
+
pipelines.append(
|
|
291
|
+
Pipeline(
|
|
292
|
+
uuid,
|
|
293
|
+
config=dict(type='invalid'),
|
|
294
|
+
repo_path=repo_path,
|
|
295
|
+
)
|
|
296
|
+
)
|
|
254
297
|
else:
|
|
255
298
|
pipeline_uuids_miss.append(uuid)
|
|
256
299
|
|
|
@@ -263,7 +306,11 @@ class PipelineResource(BaseResource):
|
|
|
263
306
|
|
|
264
307
|
mapping = {}
|
|
265
308
|
if include_schedules:
|
|
266
|
-
mapping = query_pipeline_schedules(
|
|
309
|
+
mapping = query_pipeline_schedules(
|
|
310
|
+
pipeline_uuids,
|
|
311
|
+
context_data=context_data,
|
|
312
|
+
repo_path=repo_path,
|
|
313
|
+
)
|
|
267
314
|
|
|
268
315
|
filtered_pipelines = []
|
|
269
316
|
for pipeline in pipelines:
|
|
@@ -273,15 +320,23 @@ class PipelineResource(BaseResource):
|
|
|
273
320
|
pipeline.schedules = schedules
|
|
274
321
|
|
|
275
322
|
if pipeline_statuses and (
|
|
276
|
-
(
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
len(pipeline.schedules)
|
|
323
|
+
(
|
|
324
|
+
PipelineStatus.ACTIVE in pipeline_statuses
|
|
325
|
+
and any(
|
|
326
|
+
s.status == ScheduleStatus.ACTIVE for s in pipeline.schedules
|
|
327
|
+
)
|
|
328
|
+
)
|
|
329
|
+
or (
|
|
330
|
+
PipelineStatus.INACTIVE in pipeline_statuses
|
|
331
|
+
and len(pipeline.schedules) > 0
|
|
332
|
+
and all(
|
|
333
|
+
s.status == ScheduleStatus.INACTIVE for s in pipeline.schedules
|
|
334
|
+
)
|
|
335
|
+
)
|
|
336
|
+
or (
|
|
337
|
+
PipelineStatus.NO_SCHEDULES in pipeline_statuses
|
|
338
|
+
and len(pipeline.schedules) == 0
|
|
339
|
+
)
|
|
285
340
|
):
|
|
286
341
|
filtered_pipelines.append(pipeline)
|
|
287
342
|
|
|
@@ -294,6 +349,7 @@ class PipelineResource(BaseResource):
|
|
|
294
349
|
pipeline.history = history_by_pipeline_uuid.get(pipeline.uuid)
|
|
295
350
|
|
|
296
351
|
if sorts:
|
|
352
|
+
|
|
297
353
|
def _sort_key(p, sorts=sorts, reverse_sort=reverse_sort):
|
|
298
354
|
bools = []
|
|
299
355
|
vals = []
|
|
@@ -301,24 +357,35 @@ class PipelineResource(BaseResource):
|
|
|
301
357
|
if 'blocks' == k.lower():
|
|
302
358
|
val = len(p.blocks_by_uuid)
|
|
303
359
|
vals.append(val)
|
|
304
|
-
bools.append(
|
|
360
|
+
bools.append(
|
|
361
|
+
val is None if not reverse_sort else val is not None
|
|
362
|
+
)
|
|
305
363
|
elif 'triggers' == k.lower():
|
|
306
364
|
val = len(p.schedules)
|
|
307
365
|
vals.append(val)
|
|
308
|
-
bools.append(
|
|
366
|
+
bools.append(
|
|
367
|
+
val is None if not reverse_sort else val is not None
|
|
368
|
+
)
|
|
309
369
|
elif 'status' == k.lower():
|
|
310
370
|
if len(p.schedules) == 0:
|
|
311
371
|
val = 'no_schedules'
|
|
312
|
-
elif
|
|
372
|
+
elif (
|
|
373
|
+
find(lambda s: s.status == 'active', p.schedules)
|
|
374
|
+
is not None
|
|
375
|
+
):
|
|
313
376
|
val = 'active'
|
|
314
377
|
else:
|
|
315
378
|
val = 'inactive'
|
|
316
379
|
vals.append(val)
|
|
317
|
-
bools.append(
|
|
380
|
+
bools.append(
|
|
381
|
+
val is None if not reverse_sort else val is not None
|
|
382
|
+
)
|
|
318
383
|
elif hasattr(p, k):
|
|
319
384
|
val = getattr(p, k)
|
|
320
385
|
vals.append(val)
|
|
321
|
-
bools.append(
|
|
386
|
+
bools.append(
|
|
387
|
+
val is None if not reverse_sort else val is not None
|
|
388
|
+
)
|
|
322
389
|
else:
|
|
323
390
|
bools.append(False)
|
|
324
391
|
|
|
@@ -353,12 +420,15 @@ class PipelineResource(BaseResource):
|
|
|
353
420
|
'results': len(arr),
|
|
354
421
|
'next': has_next,
|
|
355
422
|
}
|
|
356
|
-
|
|
423
|
+
for p in arr:
|
|
424
|
+
p.context_data = context_data
|
|
357
425
|
return result_set
|
|
358
426
|
|
|
359
427
|
@classmethod
|
|
360
428
|
@safe_db_query
|
|
361
429
|
async def create(self, payload, user, **kwargs):
|
|
430
|
+
context_data = kwargs.get('context_data')
|
|
431
|
+
|
|
362
432
|
clone_pipeline_uuid = payload.get('clone_pipeline_uuid')
|
|
363
433
|
template_uuid = payload.get('custom_template_uuid')
|
|
364
434
|
name = payload.get('name')
|
|
@@ -368,18 +438,22 @@ class PipelineResource(BaseResource):
|
|
|
368
438
|
llm_payload = payload.get('llm')
|
|
369
439
|
pipeline = None
|
|
370
440
|
|
|
441
|
+
repo_path = get_repo_path(context_data=context_data, user=user)
|
|
371
442
|
if template_uuid:
|
|
372
|
-
custom_template = CustomPipelineTemplate.load(
|
|
443
|
+
custom_template = CustomPipelineTemplate.load(
|
|
444
|
+
repo_path,
|
|
445
|
+
template_uuid=template_uuid,
|
|
446
|
+
)
|
|
373
447
|
pipeline = custom_template.create_pipeline(name)
|
|
374
448
|
elif clone_pipeline_uuid is not None:
|
|
375
|
-
source = Pipeline.get(clone_pipeline_uuid)
|
|
449
|
+
source = Pipeline.get(clone_pipeline_uuid, repo_path=repo_path)
|
|
376
450
|
pipeline = await Pipeline.duplicate(source, name)
|
|
377
451
|
else:
|
|
378
452
|
pipeline = Pipeline.create(
|
|
379
453
|
name,
|
|
380
454
|
description=description,
|
|
381
455
|
pipeline_type=pipeline_type,
|
|
382
|
-
repo_path=
|
|
456
|
+
repo_path=repo_path,
|
|
383
457
|
tags=tags,
|
|
384
458
|
)
|
|
385
459
|
|
|
@@ -403,25 +477,37 @@ class PipelineResource(BaseResource):
|
|
|
403
477
|
block_payload['configuration'] = configuration
|
|
404
478
|
|
|
405
479
|
block_uuid = f'{pipeline.uuid}_block_{block_number}'
|
|
406
|
-
block_resource = await BlockResource.create(
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
480
|
+
block_resource = await BlockResource.create(
|
|
481
|
+
merge_dict(
|
|
482
|
+
dict(
|
|
483
|
+
name=block_uuid,
|
|
484
|
+
type=block_payload.get('block_type'),
|
|
485
|
+
),
|
|
486
|
+
ignore_keys(
|
|
487
|
+
block_payload,
|
|
488
|
+
[
|
|
489
|
+
'block_type',
|
|
490
|
+
'upstream_blocks',
|
|
491
|
+
],
|
|
492
|
+
),
|
|
493
|
+
),
|
|
494
|
+
user,
|
|
495
|
+
**merge_dict(
|
|
496
|
+
kwargs,
|
|
497
|
+
dict(
|
|
498
|
+
parent_model=pipeline,
|
|
499
|
+
),
|
|
410
500
|
),
|
|
411
|
-
|
|
412
|
-
'block_type',
|
|
413
|
-
'upstream_blocks',
|
|
414
|
-
]),
|
|
415
|
-
), user, **merge_dict(kwargs, dict(
|
|
416
|
-
parent_model=pipeline,
|
|
417
|
-
)))
|
|
501
|
+
)
|
|
418
502
|
|
|
419
503
|
upstream_block_uuids = block_payload.get('upstream_blocks')
|
|
420
504
|
|
|
421
505
|
pipeline.add_block(
|
|
422
506
|
block_resource.model,
|
|
423
507
|
None,
|
|
424
|
-
priority=len(upstream_block_uuids)
|
|
508
|
+
priority=len(upstream_block_uuids)
|
|
509
|
+
if upstream_block_uuids
|
|
510
|
+
else 0,
|
|
425
511
|
widget=False,
|
|
426
512
|
)
|
|
427
513
|
|
|
@@ -435,11 +521,17 @@ class PipelineResource(BaseResource):
|
|
|
435
521
|
|
|
436
522
|
if upstream_block_uuids and len(upstream_block_uuids) >= 1:
|
|
437
523
|
block = config['block']
|
|
438
|
-
arr = [
|
|
524
|
+
arr = [
|
|
525
|
+
f'{pipeline.uuid}_block_{bn}'
|
|
526
|
+
for bn in upstream_block_uuids
|
|
527
|
+
]
|
|
439
528
|
block.update(dict(upstream_blocks=arr))
|
|
440
529
|
|
|
441
530
|
if pipeline:
|
|
442
|
-
await UsageStatisticLogger(
|
|
531
|
+
await UsageStatisticLogger(
|
|
532
|
+
context_data=context_data,
|
|
533
|
+
repo_path=repo_path,
|
|
534
|
+
).pipeline_create(
|
|
443
535
|
pipeline,
|
|
444
536
|
clone_pipeline_uuid=clone_pipeline_uuid,
|
|
445
537
|
llm_payload=llm_payload,
|
|
@@ -466,19 +558,26 @@ class PipelineResource(BaseResource):
|
|
|
466
558
|
|
|
467
559
|
self.on_create_callback = _on_create_callback
|
|
468
560
|
|
|
561
|
+
pipeline.context_data = context_data
|
|
469
562
|
return self(pipeline, user, **kwargs)
|
|
470
563
|
|
|
471
564
|
@classmethod
|
|
472
565
|
@safe_db_query
|
|
473
|
-
async def __fetch_model(self, pipeline_uuid: str, **
|
|
566
|
+
async def __fetch_model(self, pipeline_uuid: str, repo_path: str, **kwargs):
|
|
474
567
|
all_projects = project_platform_activated()
|
|
475
568
|
|
|
476
569
|
if all_projects:
|
|
477
570
|
return await get_pipeline_from_platform_async(
|
|
478
571
|
pipeline_uuid,
|
|
572
|
+
repo_path=repo_path,
|
|
573
|
+
context_data=kwargs.get('context_data'),
|
|
479
574
|
)
|
|
480
575
|
|
|
481
|
-
return await Pipeline.get_async(
|
|
576
|
+
return await Pipeline.get_async(
|
|
577
|
+
pipeline_uuid,
|
|
578
|
+
all_projects=all_projects,
|
|
579
|
+
repo_path=repo_path,
|
|
580
|
+
)
|
|
482
581
|
|
|
483
582
|
@classmethod
|
|
484
583
|
@safe_db_query
|
|
@@ -488,23 +587,33 @@ class PipelineResource(BaseResource):
|
|
|
488
587
|
**kwargs,
|
|
489
588
|
):
|
|
490
589
|
pipeline_uuid = urllib.parse.unquote(pk)
|
|
491
|
-
|
|
590
|
+
user = kwargs.get('user')
|
|
591
|
+
repo_path = get_repo_path(context_data=kwargs.get('context_data'), user=user)
|
|
592
|
+
return await self.__fetch_model(pipeline_uuid, repo_path, **kwargs)
|
|
492
593
|
|
|
493
594
|
@classmethod
|
|
494
595
|
@safe_db_query
|
|
495
596
|
async def member(self, pk, user, **kwargs):
|
|
496
|
-
|
|
597
|
+
context_data = kwargs.get('context_data')
|
|
598
|
+
|
|
599
|
+
repo_path = get_repo_path(context_data=context_data, user=user)
|
|
600
|
+
pipeline = await self.__fetch_model(pk, repo_path, **kwargs)
|
|
497
601
|
|
|
498
602
|
api_operation_action = kwargs.get('api_operation_action', None)
|
|
499
603
|
if api_operation_action != DELETE:
|
|
500
604
|
kernel_name = PIPELINE_TO_KERNEL_NAME[pipeline.type]
|
|
501
605
|
switch_active_kernel(
|
|
502
606
|
kernel_name,
|
|
503
|
-
emr_config=pipeline.executor_config
|
|
607
|
+
emr_config=pipeline.executor_config
|
|
608
|
+
if kernel_name == KernelName.PYSPARK
|
|
609
|
+
else None,
|
|
504
610
|
)
|
|
505
611
|
|
|
506
612
|
if api_operation_action == DETAIL:
|
|
507
|
-
if Project(
|
|
613
|
+
if Project(
|
|
614
|
+
context_data=context_data,
|
|
615
|
+
repo_config=pipeline.repo_config,
|
|
616
|
+
).is_feature_enabled(
|
|
508
617
|
FeatureUUID.OPERATION_HISTORY,
|
|
509
618
|
):
|
|
510
619
|
record_detail_pipeline(
|
|
@@ -531,6 +640,7 @@ class PipelineResource(BaseResource):
|
|
|
531
640
|
if mapping.get(pipeline.uuid):
|
|
532
641
|
pipeline.schedules = mapping[pipeline.uuid] or []
|
|
533
642
|
|
|
643
|
+
pipeline.context_data = context_data
|
|
534
644
|
return self(pipeline, user, **kwargs)
|
|
535
645
|
|
|
536
646
|
@safe_db_query
|
|
@@ -557,26 +667,30 @@ class PipelineResource(BaseResource):
|
|
|
557
667
|
|
|
558
668
|
block_cache = await BlockCache.initialize_cache()
|
|
559
669
|
for block in pipeline.blocks_by_uuid.values():
|
|
560
|
-
block_cache.remove_pipeline(
|
|
670
|
+
block_cache.remove_pipeline(
|
|
671
|
+
block, pipeline.uuid, pipeline.repo_path
|
|
672
|
+
)
|
|
561
673
|
|
|
562
674
|
self.on_delete_callback = _on_delete_callback
|
|
563
675
|
return self.model.delete()
|
|
564
676
|
|
|
565
677
|
@safe_db_query
|
|
566
678
|
async def update(self, payload, **kwargs):
|
|
679
|
+
context_data = kwargs.get('context_data')
|
|
567
680
|
if 'add_upstream_for_block_uuid' in payload:
|
|
568
681
|
block_uuid = payload['add_upstream_for_block_uuid']
|
|
569
682
|
block = self.model.get_block(block_uuid, widget=False)
|
|
570
683
|
if BlockType.DBT == block.type and block.language == BlockLanguage.SQL:
|
|
571
684
|
upstream_dbt_blocks_by_uuid = {
|
|
572
|
-
block.uuid: block
|
|
573
|
-
for block in block.upstream_dbt_blocks()
|
|
685
|
+
block.uuid: block for block in block.upstream_dbt_blocks()
|
|
574
686
|
}
|
|
575
687
|
self.model.blocks_by_uuid.update(upstream_dbt_blocks_by_uuid)
|
|
576
688
|
self.model.validate('A cycle was formed while adding a block')
|
|
577
689
|
self.model.save()
|
|
578
690
|
return self
|
|
579
691
|
|
|
692
|
+
repo_path = get_repo_path(context_data=context_data, user=self.current_user)
|
|
693
|
+
|
|
580
694
|
query = kwargs.get('query', {})
|
|
581
695
|
update_content = query.get('update_content', [False])
|
|
582
696
|
if update_content:
|
|
@@ -590,23 +704,34 @@ class PipelineResource(BaseResource):
|
|
|
590
704
|
if 'pipeline_uuid' not in llm_payload:
|
|
591
705
|
llm_payload['pipeline_uuid'] = self.model.uuid
|
|
592
706
|
|
|
593
|
-
llm_resource = await LlmResource.create(
|
|
707
|
+
llm_resource = await LlmResource.create(
|
|
708
|
+
llm_payload, self.current_user, **kwargs
|
|
709
|
+
)
|
|
594
710
|
llm_response = llm_resource.model.get('response')
|
|
595
711
|
|
|
596
712
|
pipeline_doc = None
|
|
597
713
|
block_docs = []
|
|
598
714
|
blocks = self.model.blocks_by_uuid.values()
|
|
599
715
|
|
|
600
|
-
async def _add_markdown_block(
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
716
|
+
async def _add_markdown_block(
|
|
717
|
+
block_doc: str, block_uuid: str, priority: int
|
|
718
|
+
):
|
|
719
|
+
return await BlockResource.create(
|
|
720
|
+
dict(
|
|
721
|
+
content=block_doc.strip() if block_doc else block_doc,
|
|
722
|
+
language=BlockLanguage.MARKDOWN,
|
|
723
|
+
name=f'Documentation for {block_uuid}',
|
|
724
|
+
priority=priority,
|
|
725
|
+
type=BlockType.MARKDOWN,
|
|
726
|
+
),
|
|
727
|
+
self.current_user,
|
|
728
|
+
**merge_dict(
|
|
729
|
+
kwargs,
|
|
730
|
+
dict(
|
|
731
|
+
parent_model=self.model,
|
|
732
|
+
),
|
|
733
|
+
),
|
|
734
|
+
)
|
|
610
735
|
|
|
611
736
|
if LLMUseCase.GENERATE_DOC_FOR_BLOCK == llm_use_case:
|
|
612
737
|
block_doc = llm_response.get('block_doc')
|
|
@@ -650,11 +775,15 @@ class PipelineResource(BaseResource):
|
|
|
650
775
|
switch_active_kernel(
|
|
651
776
|
kernel_name,
|
|
652
777
|
emr_config=self.model.executor_config
|
|
653
|
-
if kernel_name == KernelName.PYSPARK
|
|
778
|
+
if kernel_name == KernelName.PYSPARK
|
|
779
|
+
else None,
|
|
654
780
|
)
|
|
655
781
|
except Exception as e:
|
|
656
782
|
pipeline_type_updated = payload.get('type')
|
|
657
|
-
if
|
|
783
|
+
if (
|
|
784
|
+
pipeline_type_updated is not None
|
|
785
|
+
and pipeline_type_updated != pipeline_type
|
|
786
|
+
):
|
|
658
787
|
await self.model.update(dict(type=pipeline_type))
|
|
659
788
|
|
|
660
789
|
raise e
|
|
@@ -664,9 +793,9 @@ class PipelineResource(BaseResource):
|
|
|
664
793
|
trigger_configs_by_name = get_trigger_configs_by_name(pipeline_uuid)
|
|
665
794
|
triggers_in_code_to_update = []
|
|
666
795
|
schedules = (
|
|
667
|
-
PipelineSchedule.
|
|
668
|
-
|
|
669
|
-
|
|
796
|
+
PipelineSchedule.query.filter(
|
|
797
|
+
PipelineSchedule.pipeline_uuid == pipeline_uuid
|
|
798
|
+
)
|
|
670
799
|
).all()
|
|
671
800
|
for schedule in schedules:
|
|
672
801
|
trigger_config = trigger_configs_by_name.get(schedule.name)
|
|
@@ -689,29 +818,30 @@ class PipelineResource(BaseResource):
|
|
|
689
818
|
):
|
|
690
819
|
if pipeline_runs is not None:
|
|
691
820
|
pipeline_run_ids = [run.get('id') for run in pipeline_runs]
|
|
692
|
-
pipeline_runs_to_cancel = (
|
|
693
|
-
PipelineRun.
|
|
694
|
-
query.
|
|
695
|
-
filter(PipelineRun.id.in_(pipeline_run_ids))
|
|
821
|
+
pipeline_runs_to_cancel = PipelineRun.query.filter(
|
|
822
|
+
PipelineRun.id.in_(pipeline_run_ids)
|
|
696
823
|
)
|
|
697
824
|
elif pipeline_schedule_id is not None:
|
|
698
|
-
pipeline_runs_to_cancel = PipelineRun.in_progress_runs(
|
|
825
|
+
pipeline_runs_to_cancel = PipelineRun.in_progress_runs(
|
|
826
|
+
[pipeline_schedule_id]
|
|
827
|
+
)
|
|
699
828
|
else:
|
|
700
|
-
pipeline_runs_to_cancel = (
|
|
701
|
-
PipelineRun.
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
829
|
+
pipeline_runs_to_cancel = PipelineRun.query.filter(
|
|
830
|
+
PipelineRun.pipeline_uuid == pipeline_uuid
|
|
831
|
+
).filter(
|
|
832
|
+
PipelineRun.status.in_(
|
|
833
|
+
[
|
|
834
|
+
PipelineRun.PipelineRunStatus.INITIAL,
|
|
835
|
+
PipelineRun.PipelineRunStatus.RUNNING,
|
|
836
|
+
]
|
|
837
|
+
)
|
|
708
838
|
)
|
|
709
839
|
for pipeline_run in pipeline_runs_to_cancel:
|
|
710
840
|
PipelineScheduler(pipeline_run).stop()
|
|
711
841
|
|
|
712
842
|
def retry_pipeline_runs(pipeline_runs):
|
|
713
843
|
for run in pipeline_runs:
|
|
714
|
-
retry_pipeline_run(run)
|
|
844
|
+
retry_pipeline_run(run, repo_path)
|
|
715
845
|
|
|
716
846
|
@safe_db_query
|
|
717
847
|
def query_incomplete_block_runs(pipeline_uuid: str):
|
|
@@ -725,15 +855,18 @@ class PipelineResource(BaseResource):
|
|
|
725
855
|
a.pipeline_uuid,
|
|
726
856
|
]
|
|
727
857
|
result = (
|
|
728
|
-
PipelineRun.
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
filter(a.
|
|
732
|
-
filter(
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
858
|
+
PipelineRun.select(*columns)
|
|
859
|
+
.join(b, a.id == b.pipeline_run_id)
|
|
860
|
+
.filter(a.pipeline_uuid == pipeline_uuid)
|
|
861
|
+
.filter(a.status == PipelineRun.PipelineRunStatus.FAILED)
|
|
862
|
+
.filter(
|
|
863
|
+
b.status.not_in(
|
|
864
|
+
[
|
|
865
|
+
BlockRun.BlockRunStatus.COMPLETED,
|
|
866
|
+
BlockRun.BlockRunStatus.CONDITION_FAILED,
|
|
867
|
+
]
|
|
868
|
+
)
|
|
869
|
+
)
|
|
737
870
|
).all()
|
|
738
871
|
|
|
739
872
|
return result
|