mage-ai 0.9.70__py3-none-any.whl → 0.9.71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mage-ai might be problematic. Click here for more details.
- mage_ai/ai/utils/xgboost.py +222 -0
- mage_ai/api/errors.py +37 -25
- mage_ai/api/operations/base.py +13 -1
- mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
- mage_ai/api/policies/BlockOutputPolicy.py +40 -17
- mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
- mage_ai/api/policies/KernelPolicy.py +55 -32
- mage_ai/api/policies/KernelProcessPolicy.py +56 -0
- mage_ai/api/policies/OutputPolicy.py +73 -41
- mage_ai/api/policies/PipelinePolicy.py +206 -138
- mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
- mage_ai/api/presenters/BlockPresenter.py +1 -1
- mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
- mage_ai/api/presenters/KernelPresenter.py +5 -26
- mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
- mage_ai/api/presenters/PipelinePresenter.py +18 -5
- mage_ai/api/presenters/StatusPresenter.py +2 -0
- mage_ai/api/presenters/SyncPresenter.py +25 -0
- mage_ai/api/resources/AutocompleteItemResource.py +1 -1
- mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
- mage_ai/api/resources/BlockOutputResource.py +42 -9
- mage_ai/api/resources/BlockResource.py +4 -3
- mage_ai/api/resources/BlockRunResource.py +27 -22
- mage_ai/api/resources/ClusterResource.py +4 -1
- mage_ai/api/resources/CustomTemplateResource.py +34 -14
- mage_ai/api/resources/DataProviderResource.py +1 -1
- mage_ai/api/resources/ExecutionStateResource.py +3 -1
- mage_ai/api/resources/FileContentResource.py +8 -2
- mage_ai/api/resources/FileResource.py +10 -4
- mage_ai/api/resources/FileVersionResource.py +3 -1
- mage_ai/api/resources/GitBranchResource.py +46 -9
- mage_ai/api/resources/GlobalDataProductResource.py +44 -7
- mage_ai/api/resources/GlobalHookResource.py +4 -1
- mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
- mage_ai/api/resources/IntegrationSourceResource.py +8 -4
- mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
- mage_ai/api/resources/KernelProcessResource.py +44 -0
- mage_ai/api/resources/KernelResource.py +25 -3
- mage_ai/api/resources/OutputResource.py +33 -11
- mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
- mage_ai/api/resources/PipelineInteractionResource.py +31 -15
- mage_ai/api/resources/PipelineResource.py +250 -123
- mage_ai/api/resources/PipelineRunResource.py +11 -3
- mage_ai/api/resources/PipelineScheduleResource.py +7 -2
- mage_ai/api/resources/PipelineTriggerResource.py +6 -1
- mage_ai/api/resources/ProjectResource.py +18 -7
- mage_ai/api/resources/SecretResource.py +1 -1
- mage_ai/api/resources/SeedResource.py +8 -1
- mage_ai/api/resources/StatusResource.py +21 -6
- mage_ai/api/resources/SyncResource.py +6 -8
- mage_ai/api/resources/VariableResource.py +46 -26
- mage_ai/api/resources/VersionControlProjectResource.py +9 -2
- mage_ai/api/resources/WidgetResource.py +1 -1
- mage_ai/api/resources/WorkspaceResource.py +1 -1
- mage_ai/api/views.py +47 -40
- mage_ai/authentication/permissions/seed.py +16 -2
- mage_ai/authentication/providers/oidc.py +21 -1
- mage_ai/autocomplete/utils.py +13 -9
- mage_ai/cache/base.py +1 -1
- mage_ai/cache/block.py +18 -12
- mage_ai/cache/block_action_object/__init__.py +32 -4
- mage_ai/cache/file.py +22 -19
- mage_ai/cache/pipeline.py +18 -12
- mage_ai/cli/main.py +1 -0
- mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
- mage_ai/cluster_manager/config.py +2 -2
- mage_ai/cluster_manager/manage.py +1 -1
- mage_ai/cluster_manager/workspace/base.py +1 -1
- mage_ai/command_center/applications/factory.py +10 -7
- mage_ai/command_center/files/factory.py +17 -15
- mage_ai/command_center/utils.py +25 -13
- mage_ai/data/__init__.py +0 -0
- mage_ai/data/constants.py +45 -0
- mage_ai/data/models/__init__.py +0 -0
- mage_ai/data/models/base.py +119 -0
- mage_ai/data/models/constants.py +1 -0
- mage_ai/data/models/generator.py +115 -0
- mage_ai/data/models/manager.py +168 -0
- mage_ai/data/models/pyarrow/__init__.py +0 -0
- mage_ai/data/models/pyarrow/record_batch.py +55 -0
- mage_ai/data/models/pyarrow/shared.py +21 -0
- mage_ai/data/models/pyarrow/table.py +8 -0
- mage_ai/data/models/reader.py +103 -0
- mage_ai/data/models/utils.py +59 -0
- mage_ai/data/models/writer.py +91 -0
- mage_ai/data/tabular/__init__.py +0 -0
- mage_ai/data/tabular/constants.py +23 -0
- mage_ai/data/tabular/mocks.py +19 -0
- mage_ai/data/tabular/models.py +126 -0
- mage_ai/data/tabular/reader.py +602 -0
- mage_ai/data/tabular/utils.py +102 -0
- mage_ai/data/tabular/writer.py +266 -0
- mage_ai/data/variables/__init__.py +0 -0
- mage_ai/data/variables/wrapper.py +54 -0
- mage_ai/data_cleaner/analysis/charts.py +61 -39
- mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
- mage_ai/data_cleaner/estimators/encoders.py +5 -2
- mage_ai/data_integrations/utils/scheduler.py +16 -11
- mage_ai/data_preparation/decorators.py +1 -0
- mage_ai/data_preparation/executors/block_executor.py +237 -155
- mage_ai/data_preparation/executors/streaming_pipeline_executor.py +1 -1
- mage_ai/data_preparation/git/__init__.py +27 -7
- mage_ai/data_preparation/git/api.py +7 -1
- mage_ai/data_preparation/git/utils.py +22 -16
- mage_ai/data_preparation/logging/logger_manager.py +4 -3
- mage_ai/data_preparation/models/block/__init__.py +1542 -878
- mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
- mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
- mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
- mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
- mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
- mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
- mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
- mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
- mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
- mage_ai/data_preparation/models/block/dynamic/utils.py +291 -168
- mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
- mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
- mage_ai/data_preparation/models/block/extension/utils.py +10 -1
- mage_ai/data_preparation/models/block/global_data_product/__init__.py +10 -1
- mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
- mage_ai/data_preparation/models/block/outputs.py +722 -0
- mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
- mage_ai/data_preparation/models/block/r/__init__.py +56 -38
- mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
- mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
- mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
- mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
- mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
- mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
- mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
- mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
- mage_ai/data_preparation/models/block/utils.py +127 -70
- mage_ai/data_preparation/models/constants.py +19 -14
- mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
- mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
- mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
- mage_ai/data_preparation/models/file.py +41 -28
- mage_ai/data_preparation/models/global_data_product/__init__.py +88 -58
- mage_ai/data_preparation/models/global_hooks/models.py +1 -0
- mage_ai/data_preparation/models/interfaces.py +29 -0
- mage_ai/data_preparation/models/pipeline.py +365 -180
- mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
- mage_ai/data_preparation/models/pipelines/seed.py +1 -1
- mage_ai/data_preparation/models/project/__init__.py +66 -18
- mage_ai/data_preparation/models/project/constants.py +2 -0
- mage_ai/data_preparation/models/triggers/__init__.py +120 -24
- mage_ai/data_preparation/models/utils.py +467 -17
- mage_ai/data_preparation/models/variable.py +1028 -137
- mage_ai/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/variables/cache.py +149 -0
- mage_ai/data_preparation/models/variables/constants.py +72 -0
- mage_ai/data_preparation/models/variables/summarizer.py +336 -0
- mage_ai/data_preparation/models/variables/utils.py +77 -0
- mage_ai/data_preparation/models/widget/__init__.py +63 -41
- mage_ai/data_preparation/models/widget/charts.py +40 -27
- mage_ai/data_preparation/models/widget/constants.py +2 -0
- mage_ai/data_preparation/models/widget/utils.py +3 -3
- mage_ai/data_preparation/preferences.py +3 -3
- mage_ai/data_preparation/repo_manager.py +55 -21
- mage_ai/data_preparation/storage/base_storage.py +2 -2
- mage_ai/data_preparation/storage/gcs_storage.py +7 -4
- mage_ai/data_preparation/storage/local_storage.py +6 -3
- mage_ai/data_preparation/storage/s3_storage.py +5 -2
- mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
- mage_ai/data_preparation/variable_manager.py +281 -76
- mage_ai/io/base.py +3 -2
- mage_ai/io/bigquery.py +1 -0
- mage_ai/io/redshift.py +7 -5
- mage_ai/kernels/__init__.py +0 -0
- mage_ai/kernels/models.py +188 -0
- mage_ai/kernels/utils.py +169 -0
- mage_ai/orchestration/concurrency.py +6 -2
- mage_ai/orchestration/db/__init__.py +1 -0
- mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
- mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
- mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
- mage_ai/orchestration/db/models/oauth.py +2 -9
- mage_ai/orchestration/db/models/projects.py +10 -0
- mage_ai/orchestration/db/models/schedules.py +204 -187
- mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
- mage_ai/orchestration/db/models/utils.py +46 -5
- mage_ai/orchestration/metrics/pipeline_run.py +8 -9
- mage_ai/orchestration/notification/sender.py +1 -0
- mage_ai/orchestration/pipeline_scheduler_original.py +32 -8
- mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
- mage_ai/orchestration/run_status_checker.py +11 -4
- mage_ai/orchestration/triggers/api.py +12 -1
- mage_ai/presenters/charts/data_sources/base.py +4 -2
- mage_ai/presenters/charts/data_sources/block.py +15 -9
- mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
- mage_ai/presenters/charts/data_sources/constants.py +1 -0
- mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
- mage_ai/presenters/interactions/models.py +11 -7
- mage_ai/presenters/pages/loaders/pipelines.py +5 -3
- mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
- mage_ai/presenters/utils.py +2 -0
- mage_ai/server/api/blocks.py +2 -1
- mage_ai/server/api/downloads.py +5 -1
- mage_ai/server/api/triggers.py +3 -1
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +5 -5
- mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{1557-b3502f3f1aa92ac7.js → 1557-25a7d985d5564fd3.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/3763-61b542dafdbf5754.js → frontend_dist/_next/static/chunks/3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/overview-597b74828bf105db.js → frontend_dist/_next/static/chunks/pages/overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-7181b086c93784d2.js → [...slug]-7e737f6fc7e83e9b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js → frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
- mage_ai/server/frontend_dist/block-layout.html +3 -3
- mage_ai/server/frontend_dist/compute.html +6 -6
- mage_ai/server/frontend_dist/files.html +6 -6
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-data-products.html +6 -6
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/index.html +3 -3
- mage_ai/server/frontend_dist/manage/files.html +6 -6
- mage_ai/server/frontend_dist/manage/settings.html +6 -6
- mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist/manage/users.html +6 -6
- mage_ai/server/frontend_dist/manage.html +6 -6
- mage_ai/server/frontend_dist/oauth.html +5 -5
- mage_ai/server/frontend_dist/overview.html +6 -6
- mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist/pipelines.html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist/settings.html +3 -3
- mage_ai/server/frontend_dist/sign-in.html +12 -12
- mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist/templates.html +6 -6
- mage_ai/server/frontend_dist/terminal.html +6 -6
- mage_ai/server/frontend_dist/test.html +3 -3
- mage_ai/server/frontend_dist/triggers.html +6 -6
- mage_ai/server/frontend_dist/version-control.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/404.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{1557-b3502f3f1aa92ac7.js → 1557-25a7d985d5564fd3.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/3763-61b542dafdbf5754.js → frontend_dist_base_path_template/_next/static/chunks/3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/overview-597b74828bf105db.js → frontend_dist_base_path_template/_next/static/chunks/pages/overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-7181b086c93784d2.js → [...slug]-7e737f6fc7e83e9b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +12 -12
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
- mage_ai/server/kernel_output_parser.py +4 -1
- mage_ai/server/scheduler_manager.py +9 -0
- mage_ai/server/server.py +35 -31
- mage_ai/server/utils/custom_output.py +284 -0
- mage_ai/server/utils/execute_custom_code.py +245 -0
- mage_ai/server/utils/output_display.py +123 -289
- mage_ai/server/websocket_server.py +116 -69
- mage_ai/services/k8s/config.py +23 -0
- mage_ai/services/k8s/job_manager.py +6 -1
- mage_ai/services/ssh/aws/emr/utils.py +8 -8
- mage_ai/settings/keys/auth.py +1 -0
- mage_ai/settings/platform/__init__.py +159 -38
- mage_ai/settings/platform/constants.py +5 -0
- mage_ai/settings/platform/utils.py +53 -10
- mage_ai/settings/repo.py +26 -12
- mage_ai/settings/server.py +128 -37
- mage_ai/shared/array.py +24 -1
- mage_ai/shared/complex.py +45 -0
- mage_ai/shared/config.py +2 -1
- mage_ai/shared/custom_logger.py +11 -0
- mage_ai/shared/dates.py +10 -6
- mage_ai/shared/files.py +63 -8
- mage_ai/shared/hash.py +33 -9
- mage_ai/shared/io.py +9 -5
- mage_ai/shared/models.py +82 -24
- mage_ai/shared/outputs.py +87 -0
- mage_ai/shared/parsers.py +141 -15
- mage_ai/shared/path_fixer.py +11 -7
- mage_ai/shared/singletons/__init__.py +0 -0
- mage_ai/shared/singletons/base.py +47 -0
- mage_ai/shared/singletons/memory.py +38 -0
- mage_ai/shared/strings.py +34 -1
- mage_ai/shared/yaml.py +24 -0
- mage_ai/streaming/sinks/oracledb.py +57 -0
- mage_ai/streaming/sinks/sink_factory.py +4 -0
- mage_ai/system/__init__.py +0 -0
- mage_ai/system/constants.py +14 -0
- mage_ai/system/memory/__init__.py +0 -0
- mage_ai/system/memory/constants.py +1 -0
- mage_ai/system/memory/manager.py +174 -0
- mage_ai/system/memory/presenters.py +158 -0
- mage_ai/system/memory/process.py +216 -0
- mage_ai/system/memory/samples.py +13 -0
- mage_ai/system/memory/utils.py +656 -0
- mage_ai/system/memory/wrappers.py +177 -0
- mage_ai/system/models.py +58 -0
- mage_ai/system/storage/__init__.py +0 -0
- mage_ai/system/storage/utils.py +29 -0
- mage_ai/tests/api/endpoints/mixins.py +2 -2
- mage_ai/tests/api/endpoints/test_blocks.py +2 -1
- mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
- mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
- mage_ai/tests/api/endpoints/test_projects.py +2 -1
- mage_ai/tests/api/operations/base/test_base.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
- mage_ai/tests/api/operations/test_syncs.py +6 -4
- mage_ai/tests/api/resources/test_pipeline_resource.py +9 -2
- mage_ai/tests/authentication/providers/test_oidc.py +59 -0
- mage_ai/tests/base_test.py +2 -2
- mage_ai/tests/data/__init__.py +0 -0
- mage_ai/tests/data/models/__init__.py +0 -0
- mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
- mage_ai/tests/data_preparation/git/test_git.py +4 -1
- mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
- mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
- mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
- mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
- mage_ai/tests/data_preparation/models/block/test_global_data_product.py +3 -2
- mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
- mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
- mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
- mage_ai/tests/data_preparation/models/test_block.py +115 -120
- mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
- mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
- mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
- mage_ai/tests/data_preparation/models/test_project.py +4 -1
- mage_ai/tests/data_preparation/models/test_utils.py +80 -0
- mage_ai/tests/data_preparation/models/test_variable.py +242 -69
- mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
- mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
- mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
- mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
- mage_ai/tests/factory.py +64 -43
- mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
- mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
- mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
- mage_ai/tests/orchestration/triggers/test_global_data_product.py +3 -2
- mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
- mage_ai/tests/services/k8s/test_job_manager.py +18 -0
- mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
- mage_ai/tests/test_shared.py +61 -0
- mage_ai/usage_statistics/logger.py +7 -2
- mage_ai/utils/code.py +33 -19
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/METADATA +5 -2
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/RECORD +513 -417
- mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
- mage_ai/server/frontend_dist/_next/static/RhDiJSkcjCsh4xxX4BFBk/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7966-b9b85ba10667e654.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/TdpLLFome13qvM0gXvpHs/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-b9b85ba10667e654.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +0 -1
- mage_ai/shared/memory.py +0 -90
- mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
- /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
- /mage_ai/server/frontend_dist/_next/static/{RhDiJSkcjCsh4xxX4BFBk → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{TdpLLFome13qvM0gXvpHs → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
- /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
|
@@ -39,13 +39,22 @@ class PipelineScheduleProjectPlatformMixin:
|
|
|
39
39
|
def repo_query_project_platform(cls):
|
|
40
40
|
repo_paths = []
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
context_data = dict()
|
|
43
|
+
queries = Project(
|
|
44
|
+
context_data=context_data,
|
|
45
|
+
).repo_path_for_database_query('pipeline_schedules')
|
|
43
46
|
if queries:
|
|
44
47
|
repo_paths.extend(queries)
|
|
45
48
|
|
|
46
|
-
repo_paths.extend([
|
|
47
|
-
|
|
48
|
-
|
|
49
|
+
repo_paths.extend([
|
|
50
|
+
d.get(
|
|
51
|
+
'full_path',
|
|
52
|
+
)
|
|
53
|
+
for d in build_repo_path_for_all_projects(
|
|
54
|
+
context_data=context_data,
|
|
55
|
+
mage_projects_only=True,
|
|
56
|
+
).values()
|
|
57
|
+
])
|
|
49
58
|
|
|
50
59
|
return cls.query.filter(
|
|
51
60
|
or_(
|
|
@@ -149,14 +158,11 @@ class PipelineScheduleProjectPlatformMixin:
|
|
|
149
158
|
|
|
150
159
|
pipeline_use = pipeline or self.pipeline
|
|
151
160
|
if not pipeline_use:
|
|
152
|
-
|
|
153
|
-
Pipeline
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
+ f'for pipeline schedule ID {self.id}.',
|
|
158
|
-
)
|
|
159
|
-
return False
|
|
161
|
+
print(
|
|
162
|
+
f'[WARNING] Pipeline {self.pipeline_uuid} cannot be found '
|
|
163
|
+
+ f'for pipeline schedule ID {self.id}.',
|
|
164
|
+
)
|
|
165
|
+
return False
|
|
160
166
|
|
|
161
167
|
if self.schedule_interval == ScheduleInterval.ONCE:
|
|
162
168
|
pipeline_run_count = self.pipeline_runs_count
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
from mage_ai.orchestration.db import db_connection, safe_db_query
|
|
2
|
-
from mage_ai.orchestration.db.models.schedules import (
|
|
3
|
-
Backfill,
|
|
4
|
-
PipelineRun,
|
|
5
|
-
PipelineSchedule,
|
|
6
|
-
)
|
|
7
2
|
|
|
8
3
|
|
|
9
4
|
@safe_db_query
|
|
10
5
|
def transfer_related_models_for_pipeline(old_uuid: str, new_uuid: str):
|
|
6
|
+
from mage_ai.orchestration.db.models.schedules import (
|
|
7
|
+
Backfill,
|
|
8
|
+
PipelineRun,
|
|
9
|
+
PipelineSchedule,
|
|
10
|
+
)
|
|
11
|
+
|
|
11
12
|
# Migrate pipeline schedules
|
|
12
13
|
PipelineSchedule.query.filter(PipelineSchedule.pipeline_uuid == old_uuid).update({
|
|
13
14
|
PipelineSchedule.pipeline_uuid: new_uuid
|
|
@@ -21,3 +22,43 @@ def transfer_related_models_for_pipeline(old_uuid: str, new_uuid: str):
|
|
|
21
22
|
Backfill.pipeline_uuid: new_uuid
|
|
22
23
|
}, synchronize_session=False)
|
|
23
24
|
db_connection.session.commit()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@safe_db_query
|
|
28
|
+
def get_active_project_for_user(user, root_project_uuid: str) -> str:
|
|
29
|
+
from mage_ai.orchestration.db.models.projects import UserProject
|
|
30
|
+
return UserProject.query.filter(
|
|
31
|
+
UserProject.user_id == user.id,
|
|
32
|
+
UserProject.root_project_uuid == root_project_uuid,
|
|
33
|
+
UserProject.active.is_(True),
|
|
34
|
+
).one_or_none()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@safe_db_query
|
|
38
|
+
def activate_project_for_user(user, root_project_uuid: str, project_name: str):
|
|
39
|
+
from mage_ai.orchestration.db.models.projects import UserProject
|
|
40
|
+
projects = UserProject.query.filter(
|
|
41
|
+
UserProject.user_id == user.id,
|
|
42
|
+
UserProject.root_project_uuid == root_project_uuid,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
projects.update({
|
|
46
|
+
UserProject.active: False
|
|
47
|
+
}, synchronize_session=False)
|
|
48
|
+
|
|
49
|
+
project_updated = False
|
|
50
|
+
for project in projects:
|
|
51
|
+
if project.project_name == project_name:
|
|
52
|
+
project.update(active=True, commit=False)
|
|
53
|
+
project_updated = True
|
|
54
|
+
|
|
55
|
+
if not project_updated:
|
|
56
|
+
UserProject.create(
|
|
57
|
+
user_id=user.id,
|
|
58
|
+
root_project_uuid=root_project_uuid,
|
|
59
|
+
project_name=project_name,
|
|
60
|
+
active=True,
|
|
61
|
+
commit=False,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
db_connection.session.commit()
|
|
@@ -4,9 +4,6 @@ from copy import deepcopy
|
|
|
4
4
|
from typing import Dict, List, Tuple
|
|
5
5
|
|
|
6
6
|
from mage_ai.data_preparation.models.constants import PipelineType
|
|
7
|
-
from mage_ai.data_preparation.models.pipelines.integration_pipeline import (
|
|
8
|
-
IntegrationPipeline,
|
|
9
|
-
)
|
|
10
7
|
from mage_ai.orchestration.db.models.schedules import BlockRun, PipelineRun
|
|
11
8
|
from mage_ai.shared.hash import merge_dict
|
|
12
9
|
|
|
@@ -40,6 +37,7 @@ KEY_TO_METRICS = {
|
|
|
40
37
|
|
|
41
38
|
def calculate_pipeline_run_metrics(
|
|
42
39
|
pipeline_run: PipelineRun,
|
|
40
|
+
pipeline,
|
|
43
41
|
logger=None,
|
|
44
42
|
logging_tags: Dict = None,
|
|
45
43
|
) -> Dict:
|
|
@@ -54,7 +52,7 @@ def calculate_pipeline_run_metrics(
|
|
|
54
52
|
**logging_tags,
|
|
55
53
|
)
|
|
56
54
|
try:
|
|
57
|
-
__calculate_metrics(pipeline_run)
|
|
55
|
+
__calculate_metrics(pipeline_run, pipeline)
|
|
58
56
|
if logger:
|
|
59
57
|
logger.info(
|
|
60
58
|
f'Calculate metrics for pipeline run {pipeline_run.id} completed.',
|
|
@@ -72,6 +70,7 @@ def calculate_pipeline_run_metrics(
|
|
|
72
70
|
def calculate_source_metrics(
|
|
73
71
|
pipeline_run: PipelineRun,
|
|
74
72
|
block_run: BlockRun,
|
|
73
|
+
pipeline,
|
|
75
74
|
stream: str,
|
|
76
75
|
logger=None,
|
|
77
76
|
logging_tags: Dict = None,
|
|
@@ -79,6 +78,7 @@ def calculate_source_metrics(
|
|
|
79
78
|
return __calculate_block_metrics(
|
|
80
79
|
pipeline_run,
|
|
81
80
|
block_run,
|
|
81
|
+
pipeline,
|
|
82
82
|
stream,
|
|
83
83
|
KEY_SOURCE,
|
|
84
84
|
logger=logger,
|
|
@@ -89,6 +89,7 @@ def calculate_source_metrics(
|
|
|
89
89
|
def calculate_destination_metrics(
|
|
90
90
|
pipeline_run: PipelineRun,
|
|
91
91
|
block_run: BlockRun,
|
|
92
|
+
pipeline,
|
|
92
93
|
stream: str,
|
|
93
94
|
logger=None,
|
|
94
95
|
logging_tags: Dict = None,
|
|
@@ -96,6 +97,7 @@ def calculate_destination_metrics(
|
|
|
96
97
|
return __calculate_block_metrics(
|
|
97
98
|
pipeline_run,
|
|
98
99
|
block_run,
|
|
100
|
+
pipeline,
|
|
99
101
|
stream,
|
|
100
102
|
KEY_DESTINATION,
|
|
101
103
|
logger=logger,
|
|
@@ -106,6 +108,7 @@ def calculate_destination_metrics(
|
|
|
106
108
|
def __calculate_block_metrics(
|
|
107
109
|
pipeline_run: PipelineRun,
|
|
108
110
|
block_run: BlockRun,
|
|
111
|
+
pipeline,
|
|
109
112
|
stream: str,
|
|
110
113
|
key: str,
|
|
111
114
|
logger=None,
|
|
@@ -135,8 +138,6 @@ def __calculate_block_metrics(
|
|
|
135
138
|
**logging_tags,
|
|
136
139
|
)
|
|
137
140
|
try:
|
|
138
|
-
pipeline = IntegrationPipeline.get(pipeline_run.pipeline_uuid)
|
|
139
|
-
|
|
140
141
|
logs_arr = block_run.logs['content'].split('\n')
|
|
141
142
|
logs_by_uuid = {key: [logs_arr]}
|
|
142
143
|
metrics = get_metrics(
|
|
@@ -192,7 +193,7 @@ def __calculate_block_metrics(
|
|
|
192
193
|
)
|
|
193
194
|
|
|
194
195
|
|
|
195
|
-
def __calculate_metrics(pipeline_run: PipelineRun) -> Dict:
|
|
196
|
+
def __calculate_metrics(pipeline_run: PipelineRun, pipeline) -> Dict:
|
|
196
197
|
"""
|
|
197
198
|
Calculate metrics for an integration pipeline run. Only the "pipeline" field
|
|
198
199
|
in the metrics will be updated by calling this method.
|
|
@@ -205,8 +206,6 @@ def __calculate_metrics(pipeline_run: PipelineRun) -> Dict:
|
|
|
205
206
|
Returns:
|
|
206
207
|
Dict: The calculated metrics.
|
|
207
208
|
"""
|
|
208
|
-
pipeline = IntegrationPipeline.get(pipeline_run.pipeline_uuid)
|
|
209
|
-
|
|
210
209
|
if PipelineType.INTEGRATION != pipeline.type:
|
|
211
210
|
return
|
|
212
211
|
|
|
@@ -179,6 +179,7 @@ class NotificationSender:
|
|
|
179
179
|
pipeline_run_url=self.__pipeline_run_url(pipeline, pipeline_run),
|
|
180
180
|
pipeline_schedule_id=pipeline_run.pipeline_schedule.id,
|
|
181
181
|
pipeline_schedule_name=pipeline_run.pipeline_schedule.name,
|
|
182
|
+
pipeline_schedule_description=pipeline_run.pipeline_schedule.description,
|
|
182
183
|
pipeline_uuid=pipeline.uuid,
|
|
183
184
|
stacktrace=stacktrace,
|
|
184
185
|
)
|
|
@@ -23,7 +23,7 @@ from mage_ai.data_preparation.models.triggers import (
|
|
|
23
23
|
ScheduleInterval,
|
|
24
24
|
ScheduleStatus,
|
|
25
25
|
ScheduleType,
|
|
26
|
-
|
|
26
|
+
get_triggers_by_pipeline_with_cache,
|
|
27
27
|
)
|
|
28
28
|
from mage_ai.data_preparation.repo_manager import get_repo_config
|
|
29
29
|
from mage_ai.data_preparation.sync.git_sync import get_sync_config
|
|
@@ -155,10 +155,12 @@ class PipelineScheduler:
|
|
|
155
155
|
if is_integration:
|
|
156
156
|
clear_source_output_files(
|
|
157
157
|
self.pipeline_run,
|
|
158
|
+
self.pipeline,
|
|
158
159
|
self.logger,
|
|
159
160
|
)
|
|
160
161
|
initialize_state_and_runs(
|
|
161
162
|
self.pipeline_run,
|
|
163
|
+
self.pipeline,
|
|
162
164
|
self.logger,
|
|
163
165
|
self.pipeline_run.get_variables(),
|
|
164
166
|
)
|
|
@@ -228,6 +230,7 @@ class PipelineScheduler:
|
|
|
228
230
|
tags = self.build_tags()
|
|
229
231
|
calculate_pipeline_run_metrics(
|
|
230
232
|
self.pipeline_run,
|
|
233
|
+
self.pipeline,
|
|
231
234
|
logger=self.logger,
|
|
232
235
|
logging_tags=tags,
|
|
233
236
|
)
|
|
@@ -463,6 +466,7 @@ class PipelineScheduler:
|
|
|
463
466
|
|
|
464
467
|
calculate_pipeline_run_metrics(
|
|
465
468
|
self.pipeline_run,
|
|
469
|
+
self.pipeline,
|
|
466
470
|
logger=self.logger,
|
|
467
471
|
logging_tags=tags,
|
|
468
472
|
)
|
|
@@ -485,6 +489,7 @@ class PipelineScheduler:
|
|
|
485
489
|
if PipelineType.INTEGRATION == self.pipeline.type:
|
|
486
490
|
calculate_pipeline_run_metrics(
|
|
487
491
|
self.pipeline_run,
|
|
492
|
+
self.pipeline,
|
|
488
493
|
logger=self.logger,
|
|
489
494
|
logging_tags=tags,
|
|
490
495
|
)
|
|
@@ -1069,6 +1074,7 @@ def run_integration_stream(
|
|
|
1069
1074
|
calculate_source_metrics(
|
|
1070
1075
|
pipeline_run,
|
|
1071
1076
|
block_run,
|
|
1077
|
+
pipeline,
|
|
1072
1078
|
stream=tap_stream_id,
|
|
1073
1079
|
logger=pipeline_scheduler.logger,
|
|
1074
1080
|
logging_tags=merge_dict(tags_updated, dict(tags=tags2)),
|
|
@@ -1077,6 +1083,7 @@ def run_integration_stream(
|
|
|
1077
1083
|
calculate_destination_metrics(
|
|
1078
1084
|
pipeline_run,
|
|
1079
1085
|
block_run,
|
|
1086
|
+
pipeline,
|
|
1080
1087
|
stream=tap_stream_id,
|
|
1081
1088
|
logger=pipeline_scheduler.logger,
|
|
1082
1089
|
logging_tags=merge_dict(tags_updated, dict(tags=tags2)),
|
|
@@ -1163,7 +1170,7 @@ def run_block(
|
|
|
1163
1170
|
else:
|
|
1164
1171
|
repo_path = get_repo_path()
|
|
1165
1172
|
retry_config = merge_dict(
|
|
1166
|
-
get_repo_config(repo_path).retry_config or dict(),
|
|
1173
|
+
get_repo_config(repo_path=repo_path).retry_config or dict(),
|
|
1167
1174
|
block.retry_config or dict(),
|
|
1168
1175
|
)
|
|
1169
1176
|
|
|
@@ -1253,9 +1260,10 @@ def configure_pipeline_run_payload(
|
|
|
1253
1260
|
@safe_db_query
|
|
1254
1261
|
def retry_pipeline_run(
|
|
1255
1262
|
pipeline_run: Dict,
|
|
1263
|
+
repo_path: str,
|
|
1256
1264
|
) -> 'PipelineRun':
|
|
1257
1265
|
pipeline_uuid = pipeline_run['pipeline_uuid']
|
|
1258
|
-
pipeline = Pipeline.get(pipeline_uuid, check_if_exists=True)
|
|
1266
|
+
pipeline = Pipeline.get(pipeline_uuid, check_if_exists=True, repo_path=repo_path)
|
|
1259
1267
|
if pipeline is None or not pipeline.is_valid_pipeline(pipeline.dir_path):
|
|
1260
1268
|
raise Exception(f'Pipeline {pipeline_uuid} is not a valid pipeline.')
|
|
1261
1269
|
|
|
@@ -1399,7 +1407,7 @@ def check_sla():
|
|
|
1399
1407
|
else pipeline_run.created_at
|
|
1400
1408
|
if compare(start_date + timedelta(seconds=sla), current_time) == -1:
|
|
1401
1409
|
# passed SLA for pipeline_run
|
|
1402
|
-
pipeline =
|
|
1410
|
+
pipeline = pipeline_schedule.pipeline
|
|
1403
1411
|
notification_sender = NotificationSender(
|
|
1404
1412
|
NotificationConfig.load(
|
|
1405
1413
|
config=merge_dict(
|
|
@@ -1436,7 +1444,7 @@ def schedule_all():
|
|
|
1436
1444
|
schedules, so that needs to be addressed at some point.
|
|
1437
1445
|
"""
|
|
1438
1446
|
db_connection.session.expire_all()
|
|
1439
|
-
|
|
1447
|
+
repo_config = get_repo_config()
|
|
1440
1448
|
repo_pipelines = set(Pipeline.get_all_pipelines_all_projects(
|
|
1441
1449
|
get_repo_path(),
|
|
1442
1450
|
disable_pipelines_folder_creation=True,
|
|
@@ -1501,6 +1509,7 @@ def schedule_all():
|
|
|
1501
1509
|
|
|
1502
1510
|
for pipeline_uuid, pipeline, active_schedules in gen_pipeline_with_schedules(
|
|
1503
1511
|
active_pipeline_schedules,
|
|
1512
|
+
repo_config=repo_config,
|
|
1504
1513
|
):
|
|
1505
1514
|
concurrency_config = ConcurrencyConfig.load(config=pipeline.concurrency_config)
|
|
1506
1515
|
pipeline_runs_to_start = []
|
|
@@ -1531,6 +1540,7 @@ def schedule_all():
|
|
|
1531
1540
|
should_schedule = pipeline_schedule.should_schedule(
|
|
1532
1541
|
previous_runtimes=previous_runtimes,
|
|
1533
1542
|
pipeline=pipeline,
|
|
1543
|
+
repo_config=repo_config,
|
|
1534
1544
|
)
|
|
1535
1545
|
initial_pipeline_runs = [
|
|
1536
1546
|
r for r in pipeline_schedule.pipeline_runs
|
|
@@ -1668,6 +1678,7 @@ def schedule_all():
|
|
|
1668
1678
|
|
|
1669
1679
|
def gen_pipeline_with_schedules_single_project(
|
|
1670
1680
|
active_pipeline_schedules: List[PipelineSchedule],
|
|
1681
|
+
repo_config=None,
|
|
1671
1682
|
) -> Generator[Tuple[str, Pipeline, List[PipelineSchedule]], None, None]:
|
|
1672
1683
|
"""
|
|
1673
1684
|
Generate pipelines with associated schedules for a single project.
|
|
@@ -1681,6 +1692,8 @@ def gen_pipeline_with_schedules_single_project(
|
|
|
1681
1692
|
- The pipeline object.
|
|
1682
1693
|
- A list of active schedules associated with the pipeline.
|
|
1683
1694
|
"""
|
|
1695
|
+
# Avoid calling get_repo_config repeatedly in Pipeline init method.
|
|
1696
|
+
repo_config = repo_config or get_repo_config()
|
|
1684
1697
|
pipeline_schedules_by_pipeline = collections.defaultdict(list)
|
|
1685
1698
|
for schedule in active_pipeline_schedules:
|
|
1686
1699
|
pipeline_schedules_by_pipeline[schedule.pipeline_uuid].append(schedule)
|
|
@@ -1689,15 +1702,21 @@ def gen_pipeline_with_schedules_single_project(
|
|
|
1689
1702
|
# each pipeline.
|
|
1690
1703
|
for pipeline_uuid, active_schedules in pipeline_schedules_by_pipeline.items():
|
|
1691
1704
|
try:
|
|
1692
|
-
pipeline = Pipeline.get(
|
|
1705
|
+
pipeline = Pipeline.get(
|
|
1706
|
+
pipeline_uuid,
|
|
1707
|
+
repo_path=get_repo_path(),
|
|
1708
|
+
repo_config=repo_config,
|
|
1709
|
+
)
|
|
1693
1710
|
except Exception as e:
|
|
1694
1711
|
print(f'Error fetching pipeline {pipeline_uuid}: {e}')
|
|
1712
|
+
traceback.print_exc()
|
|
1695
1713
|
continue
|
|
1696
1714
|
yield pipeline_uuid, pipeline, active_schedules
|
|
1697
1715
|
|
|
1698
1716
|
|
|
1699
1717
|
def gen_pipeline_with_schedules_project_platform(
|
|
1700
1718
|
active_pipeline_schedules: List[PipelineSchedule],
|
|
1719
|
+
repo_config=None,
|
|
1701
1720
|
) -> Generator[Tuple[str, Pipeline, List[PipelineSchedule]], None, None]:
|
|
1702
1721
|
"""
|
|
1703
1722
|
Generate pipelines with associated schedules for project platform.
|
|
@@ -1761,6 +1780,7 @@ def gen_pipeline_with_schedules_project_platform(
|
|
|
1761
1780
|
)
|
|
1762
1781
|
except Exception as e:
|
|
1763
1782
|
print(f'Error fetching pipeline {pipeline_uuid}: {e}')
|
|
1783
|
+
traceback.print_exc()
|
|
1764
1784
|
continue
|
|
1765
1785
|
yield pipeline_uuid, pipeline, active_schedules
|
|
1766
1786
|
|
|
@@ -1812,8 +1832,12 @@ def sync_schedules(pipeline_uuids: List[str]):
|
|
|
1812
1832
|
|
|
1813
1833
|
# Sync schedule configs from triggers.yaml to DB
|
|
1814
1834
|
for pipeline_uuid in pipeline_uuids:
|
|
1815
|
-
pipeline_triggers =
|
|
1816
|
-
logger.debug(f'Sync pipeline trigger configs for {pipeline_uuid}: {pipeline_triggers}.'
|
|
1835
|
+
pipeline_triggers, from_cache = get_triggers_by_pipeline_with_cache(pipeline_uuid)
|
|
1836
|
+
logger.debug(f'Sync pipeline trigger configs for {pipeline_uuid}: {pipeline_triggers}.'
|
|
1837
|
+
f'From cache: {from_cache}')
|
|
1838
|
+
if from_cache:
|
|
1839
|
+
# No updates need to be made to database
|
|
1840
|
+
continue
|
|
1817
1841
|
for pipeline_trigger in pipeline_triggers:
|
|
1818
1842
|
if pipeline_trigger.envs and get_env() not in pipeline_trigger.envs:
|
|
1819
1843
|
continue
|
|
@@ -1120,7 +1120,7 @@ def run_block(
|
|
|
1120
1120
|
repo_path = get_repo_path()
|
|
1121
1121
|
|
|
1122
1122
|
retry_config = merge_dict(
|
|
1123
|
-
get_repo_config(repo_path).retry_config or dict(),
|
|
1123
|
+
get_repo_config(repo_path=repo_path).retry_config or dict(),
|
|
1124
1124
|
block.retry_config or dict(),
|
|
1125
1125
|
)
|
|
1126
1126
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from datetime import datetime, timedelta, timezone
|
|
2
2
|
|
|
3
3
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
4
|
+
from mage_ai.data_preparation.models.utils import warn_for_repo_path
|
|
4
5
|
from mage_ai.orchestration.db.models.schedules import (
|
|
5
6
|
BlockRun,
|
|
6
7
|
PipelineRun,
|
|
@@ -23,10 +24,11 @@ BLOCK_FAILURE_STATUSES = [
|
|
|
23
24
|
def check_status(
|
|
24
25
|
pipeline_uuid: str,
|
|
25
26
|
execution_date: datetime,
|
|
27
|
+
repo_path: str = None,
|
|
26
28
|
block_uuid: str = None,
|
|
27
29
|
hours: int = 24,
|
|
28
30
|
) -> bool:
|
|
29
|
-
__validate_pipeline_and_block(pipeline_uuid, block_uuid)
|
|
31
|
+
__validate_pipeline_and_block(pipeline_uuid, block_uuid, repo_path=repo_path)
|
|
30
32
|
|
|
31
33
|
execution_date = execution_date.replace(tzinfo=timezone.utc)
|
|
32
34
|
pipeline_run = (
|
|
@@ -62,11 +64,16 @@ def check_status(
|
|
|
62
64
|
return pipeline_run.status == PipelineRun.PipelineRunStatus.COMPLETED
|
|
63
65
|
|
|
64
66
|
|
|
65
|
-
def __validate_pipeline_and_block(pipeline_uuid, block_uuid):
|
|
66
|
-
|
|
67
|
+
def __validate_pipeline_and_block(pipeline_uuid: str, block_uuid: str, repo_path: str = None):
|
|
68
|
+
warn_for_repo_path(repo_path=repo_path)
|
|
69
|
+
|
|
70
|
+
if repo_path is None:
|
|
71
|
+
repo_path = get_repo_path()
|
|
72
|
+
|
|
73
|
+
if pipeline_uuid not in Pipeline.get_all_pipelines(repo_path=repo_path):
|
|
67
74
|
raise Exception('Pipeline not found, stopping sensor...')
|
|
68
75
|
|
|
69
|
-
pipeline = Pipeline(pipeline_uuid,
|
|
76
|
+
pipeline = Pipeline.get(pipeline_uuid, repo_path=repo_path)
|
|
70
77
|
|
|
71
78
|
if block_uuid is not None:
|
|
72
79
|
block = pipeline.get_block(block_uuid)
|
|
@@ -5,6 +5,7 @@ from mage_ai.api.resources.PipelineScheduleResource import PipelineScheduleResou
|
|
|
5
5
|
from mage_ai.data_preparation.models.block.remote.models import RemoteBlock
|
|
6
6
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
7
7
|
from mage_ai.data_preparation.models.triggers import ScheduleStatus, ScheduleType
|
|
8
|
+
from mage_ai.data_preparation.models.utils import warn_for_repo_path
|
|
8
9
|
from mage_ai.orchestration.db.models.schedules import PipelineRun, PipelineSchedule
|
|
9
10
|
from mage_ai.orchestration.triggers.constants import (
|
|
10
11
|
DEFAULT_POLL_INTERVAL,
|
|
@@ -15,10 +16,12 @@ from mage_ai.orchestration.triggers.utils import (
|
|
|
15
16
|
create_and_start_pipeline_run,
|
|
16
17
|
)
|
|
17
18
|
from mage_ai.settings.platform import project_platform_activated
|
|
19
|
+
from mage_ai.settings.repo import get_repo_path
|
|
18
20
|
|
|
19
21
|
|
|
20
22
|
def trigger_pipeline(
|
|
21
23
|
pipeline_uuid: str,
|
|
24
|
+
repo_path: str = None,
|
|
22
25
|
variables: Dict = None,
|
|
23
26
|
check_status: bool = False,
|
|
24
27
|
error_on_failure: bool = False,
|
|
@@ -33,6 +36,8 @@ def trigger_pipeline(
|
|
|
33
36
|
if variables is None:
|
|
34
37
|
variables = {}
|
|
35
38
|
|
|
39
|
+
warn_for_repo_path(repo_path)
|
|
40
|
+
|
|
36
41
|
if remote_blocks:
|
|
37
42
|
arr = []
|
|
38
43
|
for remote_block in remote_blocks:
|
|
@@ -41,7 +46,13 @@ def trigger_pipeline(
|
|
|
41
46
|
arr.append(remote_block.to_dict())
|
|
42
47
|
variables['remote_blocks'] = arr
|
|
43
48
|
|
|
44
|
-
|
|
49
|
+
repo_path_use = repo_path or get_repo_path()
|
|
50
|
+
pipeline = Pipeline.get(
|
|
51
|
+
pipeline_uuid,
|
|
52
|
+
all_projects=project_platform_activated(),
|
|
53
|
+
repo_path=repo_path_use,
|
|
54
|
+
use_repo_path=repo_path is not None,
|
|
55
|
+
)
|
|
45
56
|
|
|
46
57
|
pipeline_schedule = __fetch_or_create_pipeline_schedule(pipeline, schedule_name=schedule_name)
|
|
47
58
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Any
|
|
2
3
|
|
|
3
4
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
5
|
+
from mage_ai.settings.repo import get_repo_path
|
|
4
6
|
|
|
5
7
|
|
|
6
8
|
class ChartDataSourceBase(ABC):
|
|
@@ -18,10 +20,10 @@ class ChartDataSourceBase(ABC):
|
|
|
18
20
|
@property
|
|
19
21
|
def pipeline(self):
|
|
20
22
|
if self.pipeline_uuid and not self._pipeline:
|
|
21
|
-
self._pipeline = Pipeline.get(self.pipeline_uuid)
|
|
23
|
+
self._pipeline = Pipeline.get(self.pipeline_uuid, repo_path=get_repo_path())
|
|
22
24
|
|
|
23
25
|
return self._pipeline
|
|
24
26
|
|
|
25
27
|
@abstractmethod
|
|
26
|
-
def load_data(self, **kwargs):
|
|
28
|
+
def load_data(self, **kwargs) -> Any:
|
|
27
29
|
pass
|
|
@@ -39,9 +39,13 @@ class ChartDataSourceBlock(ChartDataSourceBase):
|
|
|
39
39
|
)
|
|
40
40
|
|
|
41
41
|
if partitions >= 0:
|
|
42
|
-
pipeline_runs = pipeline_runs.order_by(
|
|
42
|
+
pipeline_runs = pipeline_runs.order_by(
|
|
43
|
+
PipelineRun.execution_date.desc()
|
|
44
|
+
)
|
|
43
45
|
else:
|
|
44
|
-
pipeline_runs = pipeline_runs.order_by(
|
|
46
|
+
pipeline_runs = pipeline_runs.order_by(
|
|
47
|
+
PipelineRun.execution_date.asc()
|
|
48
|
+
)
|
|
45
49
|
|
|
46
50
|
pipeline_runs = pipeline_runs.limit(abs(partitions))
|
|
47
51
|
|
|
@@ -54,18 +58,20 @@ class ChartDataSourceBlock(ChartDataSourceBase):
|
|
|
54
58
|
)
|
|
55
59
|
|
|
56
60
|
for v in output_variable_objects:
|
|
57
|
-
arr.append(
|
|
58
|
-
self.pipeline.
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
arr.append(
|
|
62
|
+
self.pipeline.variable_manager.get_variable(
|
|
63
|
+
self.pipeline.uuid,
|
|
64
|
+
block.uuid,
|
|
65
|
+
v.uuid,
|
|
66
|
+
)
|
|
67
|
+
)
|
|
68
|
+
elif block:
|
|
63
69
|
block_output = block.execute_with_callback(
|
|
64
70
|
global_vars=merge_dict(
|
|
65
71
|
get_global_variables(self.pipeline_uuid),
|
|
66
72
|
variables or {},
|
|
67
73
|
),
|
|
68
74
|
)
|
|
69
|
-
arr += block_output[
|
|
75
|
+
arr += block_output["output"] or []
|
|
70
76
|
|
|
71
77
|
return arr
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Dict
|
|
1
|
+
from typing import Dict, Optional
|
|
2
2
|
|
|
3
3
|
from mage_ai.data_preparation.models.constants import BlockLanguage, BlockType
|
|
4
4
|
from mage_ai.data_preparation.models.widget import Widget
|
|
@@ -10,9 +10,10 @@ from mage_ai.shared.hash import merge_dict
|
|
|
10
10
|
class ChartDataSourceChartCode(ChartDataSourceBase):
|
|
11
11
|
def load_data(
|
|
12
12
|
self,
|
|
13
|
-
block: Widget = None,
|
|
14
|
-
configuration: Dict = None,
|
|
15
|
-
|
|
13
|
+
block: Optional[Widget] = None,
|
|
14
|
+
configuration: Optional[Dict] = None,
|
|
15
|
+
custom_code: Optional[str] = None,
|
|
16
|
+
variables: Optional[Dict] = None,
|
|
16
17
|
**kwargs,
|
|
17
18
|
):
|
|
18
19
|
block_use = block or Widget.get_block(
|
|
@@ -22,7 +23,9 @@ class ChartDataSourceChartCode(ChartDataSourceBase):
|
|
|
22
23
|
configuration=configuration,
|
|
23
24
|
language=BlockLanguage.PYTHON,
|
|
24
25
|
)
|
|
26
|
+
|
|
25
27
|
block_output = block_use.execute_with_callback(
|
|
28
|
+
custom_code=custom_code,
|
|
26
29
|
disable_json_serialization=True,
|
|
27
30
|
global_vars=merge_dict(
|
|
28
31
|
get_global_variables(self.pipeline_uuid) if self.pipeline_uuid else {},
|
|
@@ -30,4 +33,4 @@ class ChartDataSourceChartCode(ChartDataSourceBase):
|
|
|
30
33
|
),
|
|
31
34
|
)
|
|
32
35
|
|
|
33
|
-
return block_output[
|
|
36
|
+
return block_output["output"] or []
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import pandas as pd
|
|
4
|
+
|
|
5
|
+
from mage_ai.data_preparation.models.constants import PIPELINES_FOLDER
|
|
6
|
+
from mage_ai.presenters.charts.data_sources.base import ChartDataSourceBase
|
|
7
|
+
from mage_ai.system.memory.presenters import to_dataframe
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ChartDataSourceSystemMetrics(ChartDataSourceBase):
|
|
11
|
+
def load_data(
|
|
12
|
+
self,
|
|
13
|
+
**kwargs,
|
|
14
|
+
) -> pd.DataFrame:
|
|
15
|
+
df = to_dataframe(
|
|
16
|
+
os.path.join(
|
|
17
|
+
PIPELINES_FOLDER,
|
|
18
|
+
self.pipeline_uuid or '',
|
|
19
|
+
self.block_uuid or '',
|
|
20
|
+
),
|
|
21
|
+
).to_pandas()
|
|
22
|
+
return df
|
|
@@ -144,14 +144,14 @@ class Interaction:
|
|
|
144
144
|
return self._content
|
|
145
145
|
|
|
146
146
|
async def content_parsed(self, interpolate_variables: bool = False) -> Dict:
|
|
147
|
-
if self._content_parsed is not None or
|
|
148
|
-
BlockLanguage.YAML != self.language:
|
|
149
|
-
|
|
147
|
+
if self._content_parsed is not None or BlockLanguage.YAML != self.language:
|
|
150
148
|
return self._content_parsed
|
|
151
149
|
|
|
152
150
|
text = await self.content_async()
|
|
153
151
|
self._content_parsed = yaml.safe_load(
|
|
154
|
-
interpolate_content(text, self.__pipeline_variables)
|
|
152
|
+
interpolate_content(text, self.__pipeline_variables)
|
|
153
|
+
if interpolate_variables
|
|
154
|
+
else text,
|
|
155
155
|
)
|
|
156
156
|
|
|
157
157
|
return self._content_parsed
|
|
@@ -173,7 +173,7 @@ class Interaction:
|
|
|
173
173
|
async def layout(self) -> Dict:
|
|
174
174
|
settings = await self.content_parsed() or {}
|
|
175
175
|
rows = []
|
|
176
|
-
for row in
|
|
176
|
+
for row in settings.get('layout') or []:
|
|
177
177
|
items = []
|
|
178
178
|
for item in row:
|
|
179
179
|
items.append(InteractionLayoutItem(**item))
|
|
@@ -199,7 +199,7 @@ class Interaction:
|
|
|
199
199
|
for _uuid, item in (settings.get('inputs') or {}).items():
|
|
200
200
|
InteractionInput(**item)
|
|
201
201
|
|
|
202
|
-
for row in
|
|
202
|
+
for row in settings.get('layout') or []:
|
|
203
203
|
for item in row:
|
|
204
204
|
InteractionLayoutItem(**item)
|
|
205
205
|
|
|
@@ -287,4 +287,8 @@ class Interaction:
|
|
|
287
287
|
|
|
288
288
|
@property
|
|
289
289
|
def __repo_path(self) -> str:
|
|
290
|
-
return
|
|
290
|
+
return (
|
|
291
|
+
self.pipeline.repo_path
|
|
292
|
+
if self.pipeline is not None
|
|
293
|
+
else get_repo_path(root_project=True)
|
|
294
|
+
)
|
|
@@ -3,11 +3,12 @@ from typing import Any, Dict, List, Union
|
|
|
3
3
|
|
|
4
4
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
5
5
|
from mage_ai.presenters.pages.loaders.base import BaseLoader
|
|
6
|
+
from mage_ai.settings.repo import get_repo_path
|
|
6
7
|
|
|
7
8
|
|
|
8
|
-
async def get_pipeline(uuid: str) -> Pipeline:
|
|
9
|
+
async def get_pipeline(uuid: str, repo_path: str) -> Pipeline:
|
|
9
10
|
try:
|
|
10
|
-
return await Pipeline.get_async(uuid)
|
|
11
|
+
return await Pipeline.get_async(uuid, repo_path=repo_path)
|
|
11
12
|
except Exception as err:
|
|
12
13
|
err_message = f'Error loading pipeline {uuid}: {err}.'
|
|
13
14
|
if err.__class__.__name__ == 'OSError' and 'Too many open files' in err.strerror:
|
|
@@ -20,4 +21,5 @@ async def get_pipeline(uuid: str) -> Pipeline:
|
|
|
20
21
|
class Loader(BaseLoader):
|
|
21
22
|
@classmethod
|
|
22
23
|
async def load(self, ids: List[Union[int, str]], query: Dict = None, **kwargs) -> List[Any]:
|
|
23
|
-
|
|
24
|
+
repo_path = get_repo_path()
|
|
25
|
+
return await asyncio.gather(*[get_pipeline(uuid, repo_path) for uuid in ids])
|