mage-ai 0.9.69__py3-none-any.whl → 0.9.71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mage-ai might be problematic. Click here for more details.
- mage_ai/ai/utils/xgboost.py +222 -0
- mage_ai/api/errors.py +37 -25
- mage_ai/api/operations/base.py +13 -1
- mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
- mage_ai/api/policies/BackfillPolicy.py +1 -0
- mage_ai/api/policies/BlockOutputPolicy.py +40 -17
- mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
- mage_ai/api/policies/KernelPolicy.py +55 -32
- mage_ai/api/policies/KernelProcessPolicy.py +56 -0
- mage_ai/api/policies/OutputPolicy.py +73 -41
- mage_ai/api/policies/PipelinePolicy.py +206 -137
- mage_ai/api/policies/WorkspacePolicy.py +1 -0
- mage_ai/api/presenters/BackfillPresenter.py +1 -0
- mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
- mage_ai/api/presenters/BlockPresenter.py +1 -1
- mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
- mage_ai/api/presenters/KernelPresenter.py +5 -26
- mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
- mage_ai/api/presenters/PipelinePresenter.py +18 -5
- mage_ai/api/presenters/StatusPresenter.py +2 -0
- mage_ai/api/presenters/SyncPresenter.py +25 -0
- mage_ai/api/resources/AutocompleteItemResource.py +1 -1
- mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
- mage_ai/api/resources/BlockOutputResource.py +42 -9
- mage_ai/api/resources/BlockResource.py +4 -3
- mage_ai/api/resources/BlockRunResource.py +27 -22
- mage_ai/api/resources/ClusterResource.py +4 -1
- mage_ai/api/resources/CustomTemplateResource.py +34 -14
- mage_ai/api/resources/DataProviderResource.py +1 -1
- mage_ai/api/resources/ExecutionStateResource.py +3 -1
- mage_ai/api/resources/FileContentResource.py +8 -2
- mage_ai/api/resources/FileResource.py +10 -4
- mage_ai/api/resources/FileVersionResource.py +3 -1
- mage_ai/api/resources/GitBranchResource.py +101 -31
- mage_ai/api/resources/GitCustomBranchResource.py +29 -1
- mage_ai/api/resources/GlobalDataProductResource.py +44 -7
- mage_ai/api/resources/GlobalHookResource.py +4 -1
- mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
- mage_ai/api/resources/IntegrationSourceResource.py +8 -4
- mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
- mage_ai/api/resources/KernelProcessResource.py +44 -0
- mage_ai/api/resources/KernelResource.py +25 -3
- mage_ai/api/resources/OauthResource.py +1 -1
- mage_ai/api/resources/OutputResource.py +33 -11
- mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
- mage_ai/api/resources/PipelineInteractionResource.py +31 -15
- mage_ai/api/resources/PipelineResource.py +258 -125
- mage_ai/api/resources/PipelineRunResource.py +52 -7
- mage_ai/api/resources/PipelineScheduleResource.py +11 -2
- mage_ai/api/resources/PipelineTriggerResource.py +6 -1
- mage_ai/api/resources/ProjectResource.py +18 -7
- mage_ai/api/resources/PullRequestResource.py +6 -4
- mage_ai/api/resources/SecretResource.py +1 -1
- mage_ai/api/resources/SeedResource.py +8 -1
- mage_ai/api/resources/StatusResource.py +21 -6
- mage_ai/api/resources/SyncResource.py +6 -8
- mage_ai/api/resources/VariableResource.py +46 -26
- mage_ai/api/resources/VersionControlProjectResource.py +9 -2
- mage_ai/api/resources/WidgetResource.py +1 -1
- mage_ai/api/resources/WorkspaceResource.py +6 -5
- mage_ai/api/views.py +47 -40
- mage_ai/authentication/permissions/seed.py +16 -2
- mage_ai/authentication/providers/oidc.py +21 -1
- mage_ai/autocomplete/utils.py +13 -9
- mage_ai/cache/base.py +1 -1
- mage_ai/cache/block.py +18 -12
- mage_ai/cache/block_action_object/__init__.py +33 -5
- mage_ai/cache/file.py +22 -19
- mage_ai/cache/pipeline.py +18 -12
- mage_ai/cli/main.py +1 -0
- mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
- mage_ai/cluster_manager/config.py +2 -2
- mage_ai/cluster_manager/kubernetes/workload_manager.py +52 -1
- mage_ai/cluster_manager/manage.py +1 -1
- mage_ai/cluster_manager/workspace/base.py +7 -1
- mage_ai/cluster_manager/workspace/kubernetes.py +22 -1
- mage_ai/command_center/applications/factory.py +10 -7
- mage_ai/command_center/applications/utils.py +2 -2
- mage_ai/command_center/files/factory.py +17 -15
- mage_ai/command_center/presenters/text.py +1 -1
- mage_ai/command_center/utils.py +25 -13
- mage_ai/data/__init__.py +0 -0
- mage_ai/data/constants.py +45 -0
- mage_ai/data/models/__init__.py +0 -0
- mage_ai/data/models/base.py +119 -0
- mage_ai/data/models/constants.py +1 -0
- mage_ai/data/models/generator.py +115 -0
- mage_ai/data/models/manager.py +168 -0
- mage_ai/data/models/pyarrow/__init__.py +0 -0
- mage_ai/data/models/pyarrow/record_batch.py +55 -0
- mage_ai/data/models/pyarrow/shared.py +21 -0
- mage_ai/data/models/pyarrow/table.py +8 -0
- mage_ai/data/models/reader.py +103 -0
- mage_ai/data/models/utils.py +59 -0
- mage_ai/data/models/writer.py +91 -0
- mage_ai/data/tabular/__init__.py +0 -0
- mage_ai/data/tabular/constants.py +23 -0
- mage_ai/data/tabular/mocks.py +19 -0
- mage_ai/data/tabular/models.py +126 -0
- mage_ai/data/tabular/reader.py +602 -0
- mage_ai/data/tabular/utils.py +102 -0
- mage_ai/data/tabular/writer.py +266 -0
- mage_ai/data/variables/__init__.py +0 -0
- mage_ai/data/variables/wrapper.py +54 -0
- mage_ai/data_cleaner/analysis/charts.py +61 -39
- mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
- mage_ai/data_cleaner/estimators/encoders.py +5 -2
- mage_ai/data_integrations/utils/scheduler.py +16 -11
- mage_ai/data_preparation/decorators.py +1 -0
- mage_ai/data_preparation/executors/block_executor.py +237 -155
- mage_ai/data_preparation/executors/k8s_block_executor.py +30 -7
- mage_ai/data_preparation/executors/k8s_pipeline_executor.py +30 -7
- mage_ai/data_preparation/executors/streaming_pipeline_executor.py +2 -2
- mage_ai/data_preparation/git/__init__.py +77 -29
- mage_ai/data_preparation/git/api.py +69 -8
- mage_ai/data_preparation/git/utils.py +64 -34
- mage_ai/data_preparation/logging/logger_manager.py +4 -3
- mage_ai/data_preparation/models/block/__init__.py +1562 -879
- mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
- mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
- mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
- mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
- mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
- mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
- mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
- mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
- mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
- mage_ai/data_preparation/models/block/dynamic/utils.py +295 -167
- mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
- mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
- mage_ai/data_preparation/models/block/extension/utils.py +10 -1
- mage_ai/data_preparation/models/block/global_data_product/__init__.py +35 -3
- mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
- mage_ai/data_preparation/models/block/outputs.py +722 -0
- mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
- mage_ai/data_preparation/models/block/r/__init__.py +56 -38
- mage_ai/data_preparation/models/block/remote/__init__.py +0 -0
- mage_ai/data_preparation/models/block/remote/models.py +58 -0
- mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
- mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
- mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
- mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
- mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
- mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
- mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
- mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
- mage_ai/data_preparation/models/block/utils.py +164 -69
- mage_ai/data_preparation/models/constants.py +21 -14
- mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
- mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
- mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
- mage_ai/data_preparation/models/file.py +41 -28
- mage_ai/data_preparation/models/global_data_product/__init__.py +100 -58
- mage_ai/data_preparation/models/global_hooks/models.py +1 -0
- mage_ai/data_preparation/models/interfaces.py +29 -0
- mage_ai/data_preparation/models/pipeline.py +374 -185
- mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
- mage_ai/data_preparation/models/pipelines/seed.py +1 -1
- mage_ai/data_preparation/models/project/__init__.py +66 -18
- mage_ai/data_preparation/models/project/constants.py +2 -0
- mage_ai/data_preparation/models/triggers/__init__.py +124 -26
- mage_ai/data_preparation/models/utils.py +467 -17
- mage_ai/data_preparation/models/variable.py +1028 -137
- mage_ai/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/variables/cache.py +149 -0
- mage_ai/data_preparation/models/variables/constants.py +72 -0
- mage_ai/data_preparation/models/variables/summarizer.py +336 -0
- mage_ai/data_preparation/models/variables/utils.py +77 -0
- mage_ai/data_preparation/models/widget/__init__.py +63 -41
- mage_ai/data_preparation/models/widget/charts.py +40 -27
- mage_ai/data_preparation/models/widget/constants.py +2 -0
- mage_ai/data_preparation/models/widget/utils.py +3 -3
- mage_ai/data_preparation/preferences.py +3 -3
- mage_ai/data_preparation/repo_manager.py +55 -21
- mage_ai/data_preparation/storage/base_storage.py +2 -2
- mage_ai/data_preparation/storage/gcs_storage.py +7 -4
- mage_ai/data_preparation/storage/local_storage.py +18 -9
- mage_ai/data_preparation/storage/s3_storage.py +5 -2
- mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
- mage_ai/data_preparation/variable_manager.py +281 -76
- mage_ai/io/base.py +3 -2
- mage_ai/io/bigquery.py +1 -0
- mage_ai/io/redshift.py +7 -5
- mage_ai/kernels/__init__.py +0 -0
- mage_ai/kernels/models.py +188 -0
- mage_ai/kernels/utils.py +169 -0
- mage_ai/orchestration/concurrency.py +6 -2
- mage_ai/orchestration/db/__init__.py +1 -0
- mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
- mage_ai/orchestration/db/migrations/versions/42a14d6143f1_update_token_column_type.py +54 -0
- mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
- mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
- mage_ai/orchestration/db/models/oauth.py +12 -18
- mage_ai/orchestration/db/models/projects.py +10 -0
- mage_ai/orchestration/db/models/schedules.py +225 -187
- mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
- mage_ai/orchestration/db/models/utils.py +46 -5
- mage_ai/orchestration/metrics/pipeline_run.py +8 -9
- mage_ai/orchestration/notification/sender.py +38 -15
- mage_ai/orchestration/pipeline_scheduler_original.py +64 -33
- mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
- mage_ai/orchestration/run_status_checker.py +11 -4
- mage_ai/orchestration/triggers/api.py +41 -2
- mage_ai/orchestration/triggers/global_data_product.py +9 -4
- mage_ai/orchestration/triggers/utils.py +10 -1
- mage_ai/orchestration/utils/resources.py +3 -0
- mage_ai/presenters/charts/data_sources/base.py +4 -2
- mage_ai/presenters/charts/data_sources/block.py +15 -9
- mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
- mage_ai/presenters/charts/data_sources/constants.py +1 -0
- mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
- mage_ai/presenters/interactions/models.py +11 -7
- mage_ai/presenters/pages/loaders/pipelines.py +5 -3
- mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
- mage_ai/presenters/utils.py +2 -0
- mage_ai/server/api/blocks.py +2 -1
- mage_ai/server/api/downloads.py +9 -2
- mage_ai/server/api/runs.py +151 -0
- mage_ai/server/api/triggers.py +3 -1
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +8 -8
- mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
- mage_ai/server/frontend_dist/block-layout.html +3 -3
- mage_ai/server/frontend_dist/compute.html +6 -6
- mage_ai/server/frontend_dist/files.html +6 -6
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-data-products.html +6 -6
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/index.html +3 -3
- mage_ai/server/frontend_dist/manage/files.html +6 -6
- mage_ai/server/frontend_dist/manage/settings.html +6 -6
- mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist/manage/users.html +6 -6
- mage_ai/server/frontend_dist/manage.html +6 -6
- mage_ai/server/frontend_dist/oauth.html +5 -5
- mage_ai/server/frontend_dist/overview.html +6 -6
- mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist/pipelines.html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist/settings.html +3 -3
- mage_ai/server/frontend_dist/sign-in.html +15 -15
- mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist/templates.html +6 -6
- mage_ai/server/frontend_dist/terminal.html +6 -6
- mage_ai/server/frontend_dist/test.html +3 -3
- mage_ai/server/frontend_dist/triggers.html +6 -6
- mage_ai/server/frontend_dist/version-control.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/404.html +8 -8
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-25a7d985d5564fd3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3548-fa0792ddb88f4646.js → 3548-9d26185b3fb663b1.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3763-61b542dafdbf5754.js → 3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-8b8e100079ab69e1.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-4bfc84ff07d7656f.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{overview-597b74828bf105db.js → overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipeline-runs-3edc6270c5b0e962.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipeline-runs-6d183f91a2ff6668.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-7e737f6fc7e83e9b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-38e1fbcfbfc1014e.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-eb11c5390c982b49.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{triggers-1bdfda8edc9cf4a8.js → triggers-4612d15a65c35912.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{triggers-9cba3211434a8966.js → triggers-a599c6ac89be8c8d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +15 -15
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
- mage_ai/server/kernel_output_parser.py +4 -1
- mage_ai/server/scheduler_manager.py +12 -1
- mage_ai/server/server.py +69 -42
- mage_ai/server/utils/custom_output.py +284 -0
- mage_ai/server/utils/execute_custom_code.py +245 -0
- mage_ai/server/utils/output_display.py +123 -289
- mage_ai/server/websocket_server.py +116 -69
- mage_ai/services/aws/ecs/ecs.py +1 -0
- mage_ai/services/k8s/config.py +27 -4
- mage_ai/services/k8s/job_manager.py +6 -1
- mage_ai/services/k8s/utils.py +97 -0
- mage_ai/services/ssh/aws/emr/utils.py +8 -8
- mage_ai/settings/keys/auth.py +1 -0
- mage_ai/settings/platform/__init__.py +159 -38
- mage_ai/settings/platform/constants.py +5 -0
- mage_ai/settings/platform/utils.py +53 -10
- mage_ai/settings/repo.py +26 -12
- mage_ai/settings/server.py +128 -37
- mage_ai/shared/array.py +24 -1
- mage_ai/shared/complex.py +45 -0
- mage_ai/shared/config.py +2 -1
- mage_ai/shared/custom_logger.py +11 -0
- mage_ai/shared/dates.py +10 -6
- mage_ai/shared/files.py +63 -8
- mage_ai/shared/hash.py +33 -9
- mage_ai/shared/io.py +9 -5
- mage_ai/shared/models.py +82 -24
- mage_ai/shared/outputs.py +87 -0
- mage_ai/shared/parsers.py +144 -13
- mage_ai/shared/path_fixer.py +11 -7
- mage_ai/shared/singletons/__init__.py +0 -0
- mage_ai/shared/singletons/base.py +47 -0
- mage_ai/shared/singletons/memory.py +38 -0
- mage_ai/shared/strings.py +34 -1
- mage_ai/shared/yaml.py +24 -0
- mage_ai/streaming/sinks/oracledb.py +57 -0
- mage_ai/streaming/sinks/sink_factory.py +4 -0
- mage_ai/system/__init__.py +0 -0
- mage_ai/system/constants.py +14 -0
- mage_ai/system/memory/__init__.py +0 -0
- mage_ai/system/memory/constants.py +1 -0
- mage_ai/system/memory/manager.py +174 -0
- mage_ai/system/memory/presenters.py +158 -0
- mage_ai/system/memory/process.py +216 -0
- mage_ai/system/memory/samples.py +13 -0
- mage_ai/system/memory/utils.py +656 -0
- mage_ai/system/memory/wrappers.py +177 -0
- mage_ai/system/models.py +58 -0
- mage_ai/system/storage/__init__.py +0 -0
- mage_ai/system/storage/utils.py +29 -0
- mage_ai/tests/api/endpoints/mixins.py +2 -2
- mage_ai/tests/api/endpoints/test_blocks.py +2 -1
- mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
- mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
- mage_ai/tests/api/endpoints/test_projects.py +2 -1
- mage_ai/tests/api/operations/base/mixins.py +1 -1
- mage_ai/tests/api/operations/base/test_base.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
- mage_ai/tests/api/operations/test_syncs.py +6 -4
- mage_ai/tests/api/resources/test_pipeline_resource.py +11 -4
- mage_ai/tests/authentication/oauth/test_utils.py +1 -1
- mage_ai/tests/authentication/providers/test_oidc.py +59 -0
- mage_ai/tests/base_test.py +2 -2
- mage_ai/tests/data/__init__.py +0 -0
- mage_ai/tests/data/models/__init__.py +0 -0
- mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
- mage_ai/tests/data_preparation/git/test_git.py +4 -1
- mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
- mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
- mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
- mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
- mage_ai/tests/data_preparation/models/block/test_global_data_product.py +5 -2
- mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
- mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
- mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
- mage_ai/tests/data_preparation/models/test_block.py +115 -120
- mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
- mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
- mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
- mage_ai/tests/data_preparation/models/test_project.py +4 -1
- mage_ai/tests/data_preparation/models/test_utils.py +80 -0
- mage_ai/tests/data_preparation/models/test_variable.py +242 -69
- mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
- mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
- mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
- mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
- mage_ai/tests/factory.py +64 -43
- mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
- mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
- mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
- mage_ai/tests/orchestration/triggers/test_global_data_product.py +141 -138
- mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
- mage_ai/tests/server/test_server.py +19 -0
- mage_ai/tests/services/k8s/test_job_manager.py +27 -6
- mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
- mage_ai/tests/test_shared.py +61 -0
- mage_ai/usage_statistics/logger.py +7 -2
- mage_ai/utils/code.py +33 -19
- mage_ai/version_control/branch/utils.py +2 -1
- mage_ai/version_control/models.py +3 -2
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/METADATA +6 -3
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/RECORD +555 -454
- mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
- mage_ai/server/frontend_dist/_next/static/_krrrgup_C-dPOpX36S8I/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/KLL5mirre9d7_ZeEpaw3s/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-df144fbd8b2208c3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-f07b2913f7326b50.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9624-59b2f803f9c88cd6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-d9c89527266296f7.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage-852d403c7bda21b3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/[...slug]-ff4bd7a8ec3bab40.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-a8b61d8d239fd16f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e1dd1ed71d26c10d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/[...slug]-f028ef3880ed856c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-503049734a8b082f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-5b26eeda8aed8a7b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-8b793b3b696a2cd3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-5753fac7c1bfdc88.js +0 -1
- mage_ai/shared/memory.py +0 -90
- mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
- /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
- /mage_ai/server/frontend_dist/_next/static/{_krrrgup_C-dPOpX36S8I → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{KLL5mirre9d7_ZeEpaw3s → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
- /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.69.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
|
@@ -2,6 +2,7 @@ from mage_ai.api.operations import constants
|
|
|
2
2
|
from mage_ai.api.presenters.BasePresenter import BasePresenter
|
|
3
3
|
from mage_ai.data_preparation.models.constants import (
|
|
4
4
|
DATAFRAME_SAMPLE_COUNT_PREVIEW,
|
|
5
|
+
MAX_RESULTS_FOR_BLOCK_OUTPUTS_PREVIEW,
|
|
5
6
|
PipelineType,
|
|
6
7
|
)
|
|
7
8
|
from mage_ai.data_preparation.models.project import Project
|
|
@@ -68,10 +69,18 @@ class PipelinePresenter(BasePresenter):
|
|
|
68
69
|
if include_block_metadata:
|
|
69
70
|
include_block_metadata = include_block_metadata[0]
|
|
70
71
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
max_results_for_block_outputs = query.get(
|
|
73
|
+
'max_results_for_block_outputs', [MAX_RESULTS_FOR_BLOCK_OUTPUTS_PREVIEW]
|
|
74
|
+
)
|
|
75
|
+
if max_results_for_block_outputs:
|
|
76
|
+
max_results_for_block_outputs = max_results_for_block_outputs[0]
|
|
77
|
+
|
|
78
|
+
include_block_catalog = PipelineType.PYTHON == self.model.type and Project(
|
|
79
|
+
context_data=kwargs.get('context_data'),
|
|
80
|
+
repo_config=self.model.repo_config,
|
|
81
|
+
).is_feature_enabled(
|
|
82
|
+
FeatureUUID.DATA_INTEGRATION_IN_BATCH_PIPELINE,
|
|
83
|
+
)
|
|
75
84
|
|
|
76
85
|
data = await self.model.to_dict_async(
|
|
77
86
|
include_block_catalog=include_block_catalog,
|
|
@@ -85,6 +94,9 @@ class PipelinePresenter(BasePresenter):
|
|
|
85
94
|
include_outputs=include_outputs,
|
|
86
95
|
include_outputs_spark=include_outputs_spark,
|
|
87
96
|
sample_count=DATAFRAME_SAMPLE_COUNT_PREVIEW,
|
|
97
|
+
max_results=max_results_for_block_outputs,
|
|
98
|
+
disable_block_output_previews=True,
|
|
99
|
+
exclude_blank_variable_uuids=True,
|
|
88
100
|
)
|
|
89
101
|
elif constants.UPDATE == display_format:
|
|
90
102
|
data = self.model.to_dict(include_extensions=include_extensions)
|
|
@@ -103,7 +115,8 @@ class PipelinePresenter(BasePresenter):
|
|
|
103
115
|
|
|
104
116
|
PipelinePresenter.register_format(
|
|
105
117
|
constants.LIST,
|
|
106
|
-
PipelinePresenter.default_attributes
|
|
118
|
+
PipelinePresenter.default_attributes
|
|
119
|
+
+ [
|
|
107
120
|
'schedules',
|
|
108
121
|
],
|
|
109
122
|
)
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
from mage_ai.api.presenters.BasePresenter import BasePresenter
|
|
2
|
+
from mage_ai.data_preparation.shared.secrets import get_secret_value
|
|
3
|
+
from mage_ai.settings.repo import get_repo_path
|
|
2
4
|
|
|
3
5
|
|
|
4
6
|
class SyncPresenter(BasePresenter):
|
|
5
7
|
default_attributes = [
|
|
6
8
|
'access_token',
|
|
9
|
+
'access_token_secret_name',
|
|
7
10
|
'auth_type',
|
|
8
11
|
'branch',
|
|
9
12
|
'email',
|
|
10
13
|
'remote_repo_link',
|
|
11
14
|
'repo_path',
|
|
12
15
|
'ssh_private_key',
|
|
16
|
+
'ssh_private_key_secret_name',
|
|
13
17
|
'ssh_public_key',
|
|
18
|
+
'ssh_public_key_secret_name',
|
|
14
19
|
'sync_on_pipeline_run',
|
|
15
20
|
'sync_on_start',
|
|
16
21
|
'sync_submodules',
|
|
@@ -18,3 +23,23 @@ class SyncPresenter(BasePresenter):
|
|
|
18
23
|
'user_git_settings',
|
|
19
24
|
'username',
|
|
20
25
|
]
|
|
26
|
+
|
|
27
|
+
def present(self, **kwargs):
|
|
28
|
+
data = self.model
|
|
29
|
+
|
|
30
|
+
def filter_invalid_secret_values(data):
|
|
31
|
+
for attribute in self.default_attributes:
|
|
32
|
+
if attribute.endswith('_secret_name'):
|
|
33
|
+
secret_name = data.get(attribute)
|
|
34
|
+
if secret_name and not get_secret_value(
|
|
35
|
+
secret_name,
|
|
36
|
+
repo_name=get_repo_path(user=self.current_user),
|
|
37
|
+
suppress_warning=False,
|
|
38
|
+
):
|
|
39
|
+
data[attribute] = None
|
|
40
|
+
|
|
41
|
+
filter_invalid_secret_values(self.model)
|
|
42
|
+
if 'user_git_settings' in data:
|
|
43
|
+
filter_invalid_secret_values(data['user_git_settings'])
|
|
44
|
+
|
|
45
|
+
return self.model
|
|
@@ -13,7 +13,7 @@ class AutocompleteItemResource(GenericResource):
|
|
|
13
13
|
@classmethod
|
|
14
14
|
@safe_db_query
|
|
15
15
|
async def collection(self, query, meta, user, **kwargs):
|
|
16
|
-
repo_path = get_repo_path()
|
|
16
|
+
repo_path = get_repo_path(user=user)
|
|
17
17
|
|
|
18
18
|
collection = []
|
|
19
19
|
for file_group, mapping in [
|
|
@@ -21,6 +21,9 @@ from mage_ai.presenters.charts.data_sources.pipeline_schedules import (
|
|
|
21
21
|
ChartDataSourcePipelineSchedules,
|
|
22
22
|
)
|
|
23
23
|
from mage_ai.presenters.charts.data_sources.pipelines import ChartDataSourcePipelines
|
|
24
|
+
from mage_ai.presenters.charts.data_sources.system_metrics import (
|
|
25
|
+
ChartDataSourceSystemMetrics,
|
|
26
|
+
)
|
|
24
27
|
from mage_ai.shared.hash import extract, merge_dict
|
|
25
28
|
from mage_ai.usage_statistics.logger import UsageStatisticLogger
|
|
26
29
|
|
|
@@ -38,6 +41,13 @@ class BlockLayoutItemResource(GenericResource):
|
|
|
38
41
|
else:
|
|
39
42
|
variables[k] = v
|
|
40
43
|
|
|
44
|
+
content_override = query.get('content_override', [None])
|
|
45
|
+
if content_override:
|
|
46
|
+
content_override = content_override[0]
|
|
47
|
+
skip_render = query.get('skip_render', [False])
|
|
48
|
+
if skip_render:
|
|
49
|
+
skip_render = skip_render[0]
|
|
50
|
+
|
|
41
51
|
configuration_override = variables.pop('configuration_override', None)
|
|
42
52
|
if configuration_override:
|
|
43
53
|
configuration_override = json.loads(urllib.parse.unquote(configuration_override))
|
|
@@ -57,6 +67,7 @@ class BlockLayoutItemResource(GenericResource):
|
|
|
57
67
|
|
|
58
68
|
content = None
|
|
59
69
|
data = None
|
|
70
|
+
error = None
|
|
60
71
|
block_type = block_config.get('type')
|
|
61
72
|
|
|
62
73
|
if BlockType.CHART == block_type:
|
|
@@ -67,39 +78,58 @@ class BlockLayoutItemResource(GenericResource):
|
|
|
67
78
|
else:
|
|
68
79
|
data_source_config = data_source_override
|
|
69
80
|
|
|
70
|
-
configuration_to_use =
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
configuration_to_use = (
|
|
82
|
+
configuration_override or block_config.get('configuration') or {}
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
block = BlockFactory.get_block(
|
|
86
|
+
block_config.get('name') or file_path or uuid,
|
|
87
|
+
block_uuid,
|
|
88
|
+
block_type,
|
|
89
|
+
configuration=configuration_to_use,
|
|
90
|
+
**extract(
|
|
91
|
+
block_config,
|
|
92
|
+
[
|
|
82
93
|
'language',
|
|
83
|
-
]
|
|
84
|
-
)
|
|
94
|
+
],
|
|
95
|
+
),
|
|
96
|
+
)
|
|
85
97
|
|
|
98
|
+
content = content_override
|
|
99
|
+
if not content and block.file and block.file.exists():
|
|
86
100
|
content = block.content
|
|
87
101
|
|
|
102
|
+
if data_source_config or (
|
|
103
|
+
configuration_to_use and ChartType.CUSTOM == configuration_to_use.get('chart_type')
|
|
104
|
+
):
|
|
105
|
+
data_source_type = data_source_config.get('type')
|
|
106
|
+
pipeline_uuid = data_source_config.get('pipeline_uuid')
|
|
107
|
+
|
|
88
108
|
if ChartDataSourceType.CHART_CODE == data_source_type:
|
|
89
109
|
data_source = ChartDataSourceChartCode(
|
|
90
110
|
block_uuid=block.uuid,
|
|
91
111
|
pipeline_uuid=pipeline_uuid,
|
|
92
112
|
)
|
|
93
|
-
data = data_source.load_data(
|
|
113
|
+
data = data_source.load_data(
|
|
114
|
+
block=block,
|
|
115
|
+
configuration=configuration_to_use,
|
|
116
|
+
custom_code=content_override,
|
|
117
|
+
)
|
|
94
118
|
else:
|
|
95
119
|
data_source_block_uuid = data_source_config.get('block_uuid')
|
|
96
120
|
|
|
97
|
-
data_source_class_options = merge_dict(
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
121
|
+
data_source_class_options = merge_dict(
|
|
122
|
+
extract(
|
|
123
|
+
data_source_config,
|
|
124
|
+
[
|
|
125
|
+
'pipeline_schedule_id',
|
|
126
|
+
],
|
|
127
|
+
),
|
|
128
|
+
dict(
|
|
129
|
+
block_uuid=data_source_block_uuid,
|
|
130
|
+
pipeline_uuid=pipeline_uuid,
|
|
131
|
+
),
|
|
132
|
+
)
|
|
103
133
|
data_source_output = None
|
|
104
134
|
|
|
105
135
|
if ChartDataSourceType.BLOCK == data_source_type:
|
|
@@ -132,25 +162,32 @@ class BlockLayoutItemResource(GenericResource):
|
|
|
132
162
|
user=user,
|
|
133
163
|
**kwargs,
|
|
134
164
|
)
|
|
165
|
+
elif ChartDataSourceType.SYSTEM_METRICS == data_source_type:
|
|
166
|
+
data_source = ChartDataSourceSystemMetrics(**data_source_class_options)
|
|
167
|
+
data_source_output = data_source.load_data(
|
|
168
|
+
user=user,
|
|
169
|
+
**kwargs,
|
|
170
|
+
)
|
|
135
171
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
172
|
+
if not skip_render:
|
|
173
|
+
try:
|
|
174
|
+
input_args = None
|
|
175
|
+
if data_source_output is not None:
|
|
176
|
+
input_args = [data_source_output]
|
|
177
|
+
|
|
178
|
+
data = block.execute_with_callback(
|
|
179
|
+
custom_code=content_override,
|
|
180
|
+
disable_json_serialization=True,
|
|
181
|
+
input_args=input_args,
|
|
182
|
+
global_vars=merge_dict(
|
|
183
|
+
get_global_variables(pipeline_uuid) if pipeline_uuid else {},
|
|
184
|
+
variables or {},
|
|
185
|
+
),
|
|
186
|
+
).get('output', None)
|
|
187
|
+
except Exception as err:
|
|
188
|
+
error = ApiError(ApiError.RESOURCE_NOT_FOUND.copy())
|
|
189
|
+
error.message = str(err)
|
|
190
|
+
error.errors = traceback.format_exc()
|
|
154
191
|
|
|
155
192
|
block_config_to_show = {}
|
|
156
193
|
block_config_to_show.update(block_config)
|
|
@@ -162,8 +199,17 @@ class BlockLayoutItemResource(GenericResource):
|
|
|
162
199
|
|
|
163
200
|
await UsageStatisticLogger().chart_impression(block_config_to_show)
|
|
164
201
|
|
|
165
|
-
return self(
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
202
|
+
return self(
|
|
203
|
+
merge_dict(
|
|
204
|
+
block_config_to_show,
|
|
205
|
+
dict(
|
|
206
|
+
content=content,
|
|
207
|
+
data=data,
|
|
208
|
+
error=error,
|
|
209
|
+
skip_render=skip_render,
|
|
210
|
+
uuid=uuid,
|
|
211
|
+
),
|
|
212
|
+
),
|
|
213
|
+
user,
|
|
214
|
+
**kwargs,
|
|
215
|
+
)
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
from mage_ai.api.errors import ApiError
|
|
2
2
|
from mage_ai.api.resources.GenericResource import GenericResource
|
|
3
|
+
from mage_ai.data_preparation.models.constants import DATAFRAME_SAMPLE_COUNT
|
|
3
4
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
4
|
-
from mage_ai.data_preparation.models.
|
|
5
|
+
from mage_ai.data_preparation.models.variables.constants import VariableType
|
|
5
6
|
from mage_ai.orchestration.db import safe_db_query
|
|
7
|
+
from mage_ai.settings.repo import get_repo_path
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
class BlockOutputResource(GenericResource):
|
|
@@ -10,29 +12,60 @@ class BlockOutputResource(GenericResource):
|
|
|
10
12
|
Resource to fetch block output for the notebook. Created to support legacy
|
|
11
13
|
endpoint /api/pipelines/<pipeline_uuid>/blocks/<block_uuid>/outputs
|
|
12
14
|
"""
|
|
15
|
+
|
|
13
16
|
@classmethod
|
|
14
17
|
@safe_db_query
|
|
15
|
-
def member(
|
|
18
|
+
def member(cls, pk, user, **kwargs):
|
|
19
|
+
query = kwargs.get('query', {})
|
|
16
20
|
block_uuid = pk
|
|
17
21
|
|
|
18
|
-
query = kwargs.get('query', {})
|
|
19
22
|
pipeline_uuid = query.get('pipeline_uuid', [None])
|
|
20
23
|
if pipeline_uuid:
|
|
21
24
|
pipeline_uuid = pipeline_uuid[0]
|
|
25
|
+
|
|
26
|
+
outputs_query = {}
|
|
27
|
+
for key in [
|
|
28
|
+
'block_uuid',
|
|
29
|
+
'csv_lines_only',
|
|
30
|
+
'dynamic_block_index',
|
|
31
|
+
'exclude_blank_variable_uuids',
|
|
32
|
+
'execution_partition',
|
|
33
|
+
'include_print_outputs',
|
|
34
|
+
'sample',
|
|
35
|
+
'sample_count',
|
|
36
|
+
'variable_type',
|
|
37
|
+
]:
|
|
38
|
+
value = query.get(key, [None])
|
|
39
|
+
if value is not None:
|
|
40
|
+
value = value[0]
|
|
41
|
+
if value is not None:
|
|
42
|
+
outputs_query[key] = value
|
|
43
|
+
|
|
44
|
+
for key, value in [
|
|
45
|
+
('exclude_blank_variable_uuids', True),
|
|
46
|
+
('include_print_outputs', False),
|
|
47
|
+
('sample_count', DATAFRAME_SAMPLE_COUNT),
|
|
48
|
+
('variable_type', VariableType.DATAFRAME),
|
|
49
|
+
]:
|
|
50
|
+
if key not in outputs_query:
|
|
51
|
+
outputs_query[key] = value
|
|
52
|
+
|
|
22
53
|
outputs = []
|
|
23
54
|
if pipeline_uuid is not None:
|
|
24
|
-
|
|
55
|
+
repo_path = get_repo_path(user=user)
|
|
56
|
+
pipeline = Pipeline.get(pipeline_uuid, repo_path=repo_path)
|
|
25
57
|
block = pipeline.get_block(block_uuid)
|
|
58
|
+
|
|
26
59
|
error = ApiError.RESOURCE_ERROR.copy()
|
|
27
60
|
if block is None:
|
|
28
61
|
error.update(
|
|
29
|
-
message=f'Block {block_uuid} does not exist in pipeline {pipeline_uuid}'
|
|
62
|
+
message=f'Block {block_uuid} does not exist in pipeline {pipeline_uuid}'
|
|
63
|
+
)
|
|
30
64
|
raise ApiError(error)
|
|
65
|
+
|
|
31
66
|
# Only fetch dataframe variables by default
|
|
32
67
|
outputs = block.get_outputs(
|
|
33
|
-
|
|
34
|
-
sample_count=None,
|
|
35
|
-
variable_type=VariableType.DATAFRAME,
|
|
68
|
+
**outputs_query,
|
|
36
69
|
)
|
|
37
70
|
|
|
38
|
-
return
|
|
71
|
+
return cls(dict(outputs=outputs), user, **kwargs)
|
|
@@ -110,6 +110,7 @@ class BlockResource(GenericResource):
|
|
|
110
110
|
name = payload.get('name')
|
|
111
111
|
block_name = name or payload.get('uuid')
|
|
112
112
|
require_unique_name = payload.get('require_unique_name')
|
|
113
|
+
repo_path = get_repo_path(user=user)
|
|
113
114
|
|
|
114
115
|
payload_config = payload.get('config') or {}
|
|
115
116
|
replicated_block = None
|
|
@@ -208,7 +209,7 @@ class BlockResource(GenericResource):
|
|
|
208
209
|
|
|
209
210
|
if payload_config and payload_config.get('custom_template_uuid'):
|
|
210
211
|
template_uuid = payload_config.get('custom_template_uuid')
|
|
211
|
-
custom_template = CustomBlockTemplate.load(template_uuid=template_uuid)
|
|
212
|
+
custom_template = CustomBlockTemplate.load(repo_path, template_uuid=template_uuid)
|
|
212
213
|
block = custom_template.create_block(
|
|
213
214
|
block_name,
|
|
214
215
|
pipeline,
|
|
@@ -221,7 +222,7 @@ class BlockResource(GenericResource):
|
|
|
221
222
|
block = Block.create(
|
|
222
223
|
block_name,
|
|
223
224
|
block_type,
|
|
224
|
-
|
|
225
|
+
repo_path,
|
|
225
226
|
require_unique_name=require_unique_name,
|
|
226
227
|
**block_attributes,
|
|
227
228
|
)
|
|
@@ -234,7 +235,7 @@ class BlockResource(GenericResource):
|
|
|
234
235
|
|
|
235
236
|
if pipeline:
|
|
236
237
|
cache = await BlockCache.initialize_cache()
|
|
237
|
-
cache.add_pipeline(block.to_dict(), pipeline)
|
|
238
|
+
cache.add_pipeline(block.to_dict(), pipeline, repo_path)
|
|
238
239
|
|
|
239
240
|
cache_block_action_object = await BlockActionObjectCache.initialize_cache()
|
|
240
241
|
cache_block_action_object.update_block(block)
|
|
@@ -27,12 +27,22 @@ class BlockRunResource(DatabaseResource):
|
|
|
27
27
|
started_at = tup.started_at if hasattr(tup, 'started_at') else None
|
|
28
28
|
status = tup.status if hasattr(tup, 'status') else None
|
|
29
29
|
updated_at = tup.updated_at if hasattr(tup, 'updated_at') else None
|
|
30
|
-
pipeline_schedule_id =
|
|
31
|
-
tup
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
30
|
+
pipeline_schedule_id = (
|
|
31
|
+
tup.pipeline_schedule_id
|
|
32
|
+
if hasattr(
|
|
33
|
+
tup,
|
|
34
|
+
'pipeline_schedule_id',
|
|
35
|
+
)
|
|
36
|
+
else None
|
|
37
|
+
)
|
|
38
|
+
pipeline_schedule_name = (
|
|
39
|
+
tup.pipeline_schedule_name
|
|
40
|
+
if hasattr(
|
|
41
|
+
tup,
|
|
42
|
+
'pipeline_schedule_name',
|
|
43
|
+
)
|
|
44
|
+
else None
|
|
45
|
+
)
|
|
36
46
|
else:
|
|
37
47
|
(
|
|
38
48
|
block_uuid,
|
|
@@ -98,29 +108,22 @@ class BlockRunResource(DatabaseResource):
|
|
|
98
108
|
]
|
|
99
109
|
|
|
100
110
|
query = (
|
|
101
|
-
BlockRun.
|
|
102
|
-
|
|
103
|
-
join(
|
|
104
|
-
join(c, b.pipeline_schedule_id == c.id)
|
|
111
|
+
BlockRun.select(*columns)
|
|
112
|
+
.join(b, a.pipeline_run_id == b.id)
|
|
113
|
+
.join(c, b.pipeline_schedule_id == c.id)
|
|
105
114
|
)
|
|
106
115
|
|
|
107
116
|
pipeline_run_id = query_arg.get('pipeline_run_id', [None])
|
|
108
117
|
if pipeline_run_id:
|
|
109
118
|
pipeline_run_id = pipeline_run_id[0]
|
|
110
119
|
if pipeline_run_id and is_number(pipeline_run_id):
|
|
111
|
-
query = (
|
|
112
|
-
query.
|
|
113
|
-
filter(a.pipeline_run_id == int(pipeline_run_id))
|
|
114
|
-
)
|
|
120
|
+
query = query.filter(a.pipeline_run_id == int(pipeline_run_id))
|
|
115
121
|
|
|
116
122
|
pipeline_uuid = query_arg.get('pipeline_uuid', [None])
|
|
117
123
|
if pipeline_uuid:
|
|
118
124
|
pipeline_uuid = pipeline_uuid[0]
|
|
119
125
|
if pipeline_uuid:
|
|
120
|
-
query = (
|
|
121
|
-
query.
|
|
122
|
-
filter(c.pipeline_uuid == pipeline_uuid)
|
|
123
|
-
)
|
|
126
|
+
query = query.filter(c.pipeline_uuid == pipeline_uuid)
|
|
124
127
|
|
|
125
128
|
# The order_by value should be an attribute on the BlockRun model.
|
|
126
129
|
order_by_arg = query_arg.get('order_by', [None])
|
|
@@ -138,10 +141,12 @@ class BlockRunResource(DatabaseResource):
|
|
|
138
141
|
try:
|
|
139
142
|
br_col = getattr(a, col)
|
|
140
143
|
initial_results = query.order_by(getattr(br_col, asc_desc)())
|
|
141
|
-
except
|
|
142
|
-
raise Exception(
|
|
143
|
-
|
|
144
|
-
|
|
144
|
+
except AttributeError:
|
|
145
|
+
raise Exception(
|
|
146
|
+
'Block run sort column/query is invalid. The sort column '
|
|
147
|
+
+ 'must be an attribute of the BlockRun model. The sort direction '
|
|
148
|
+
+ 'is either "asc" (ascending order) or "desc" (descending order).'
|
|
149
|
+
)
|
|
145
150
|
else:
|
|
146
151
|
initial_results = query.order_by(
|
|
147
152
|
a.started_at.desc(),
|
|
@@ -4,6 +4,7 @@ from mage_ai.cluster_manager.constants import ClusterType
|
|
|
4
4
|
from mage_ai.orchestration.db import safe_db_query
|
|
5
5
|
from mage_ai.server.active_kernel import get_active_kernel_name
|
|
6
6
|
from mage_ai.server.kernels import KernelName
|
|
7
|
+
from mage_ai.settings.repo import get_repo_path
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
class ClusterResource(GenericResource):
|
|
@@ -41,7 +42,9 @@ class ClusterResource(GenericResource):
|
|
|
41
42
|
error['message'] = 'Please include cluster_id in the request payload.'
|
|
42
43
|
raise ApiError(error)
|
|
43
44
|
|
|
44
|
-
|
|
45
|
+
repo_path = get_repo_path(user=self.current_user)
|
|
46
|
+
|
|
47
|
+
emr_cluster_manager.set_active_cluster(repo_path=repo_path, cluster_id=cluster_id)
|
|
45
48
|
success = True
|
|
46
49
|
|
|
47
50
|
self.model.update(
|
|
@@ -20,6 +20,7 @@ from mage_ai.data_preparation.models.custom_templates.utils import (
|
|
|
20
20
|
)
|
|
21
21
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
22
22
|
from mage_ai.data_preparation.templates.template import fetch_template_source
|
|
23
|
+
from mage_ai.settings.repo import get_repo_path
|
|
23
24
|
from mage_ai.shared.hash import ignore_keys
|
|
24
25
|
from mage_ai.shared.utils import clean_name
|
|
25
26
|
from mage_ai.usage_statistics.logger import UsageStatisticLogger
|
|
@@ -64,14 +65,20 @@ class CustomTemplateResource(GenericResource):
|
|
|
64
65
|
template_uuid = clean_name(template_uuid)
|
|
65
66
|
payload['template_uuid'] = template_uuid
|
|
66
67
|
|
|
68
|
+
repo_path = get_repo_path(user=user)
|
|
67
69
|
if DIRECTORY_FOR_BLOCK_TEMPLATES == object_type:
|
|
68
|
-
custom_template = CustomBlockTemplate.load(template_uuid=template_uuid)
|
|
69
|
-
|
|
70
|
+
custom_template = CustomBlockTemplate.load(repo_path, template_uuid=template_uuid)
|
|
70
71
|
if not custom_template:
|
|
71
|
-
custom_template = CustomBlockTemplate(
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
custom_template = CustomBlockTemplate(
|
|
73
|
+
repo_path=repo_path,
|
|
74
|
+
**ignore_keys(
|
|
75
|
+
payload,
|
|
76
|
+
[
|
|
77
|
+
'uuid',
|
|
78
|
+
OBJECT_TYPE_KEY,
|
|
79
|
+
],
|
|
80
|
+
)
|
|
81
|
+
)
|
|
75
82
|
if user:
|
|
76
83
|
custom_template.user = dict(
|
|
77
84
|
username=user.username,
|
|
@@ -88,10 +95,12 @@ class CustomTemplateResource(GenericResource):
|
|
|
88
95
|
cache = await BlockActionObjectCache.initialize_cache()
|
|
89
96
|
cache.update_custom_block_template(custom_template)
|
|
90
97
|
elif DIRECTORY_FOR_PIPELINE_TEMPLATES == object_type:
|
|
91
|
-
custom_template = CustomPipelineTemplate.load(
|
|
98
|
+
custom_template = CustomPipelineTemplate.load(
|
|
99
|
+
template_uuid=template_uuid, repo_path=repo_path
|
|
100
|
+
)
|
|
92
101
|
|
|
93
102
|
if not custom_template:
|
|
94
|
-
pipeline = Pipeline.get(payload.get('pipeline_uuid'))
|
|
103
|
+
pipeline = Pipeline.get(payload.get('pipeline_uuid'), repo_path=repo_path)
|
|
95
104
|
custom_template = CustomPipelineTemplate.create_from_pipeline(
|
|
96
105
|
pipeline,
|
|
97
106
|
template_uuid,
|
|
@@ -120,12 +129,20 @@ class CustomTemplateResource(GenericResource):
|
|
|
120
129
|
|
|
121
130
|
template_uuid = urllib.parse.unquote(pk)
|
|
122
131
|
|
|
132
|
+
repo_path = get_repo_path(user=user)
|
|
123
133
|
try:
|
|
124
134
|
if DIRECTORY_FOR_BLOCK_TEMPLATES == object_type:
|
|
125
|
-
return self(
|
|
135
|
+
return self(
|
|
136
|
+
CustomBlockTemplate.load(repo_path, template_uuid=template_uuid),
|
|
137
|
+
user,
|
|
138
|
+
**kwargs,
|
|
139
|
+
)
|
|
126
140
|
elif DIRECTORY_FOR_PIPELINE_TEMPLATES == object_type:
|
|
127
141
|
return self(
|
|
128
|
-
CustomPipelineTemplate.load(
|
|
142
|
+
CustomPipelineTemplate.load(
|
|
143
|
+
repo_path,
|
|
144
|
+
template_uuid=template_uuid,
|
|
145
|
+
),
|
|
129
146
|
user,
|
|
130
147
|
**kwargs,
|
|
131
148
|
)
|
|
@@ -152,10 +169,13 @@ class CustomTemplateResource(GenericResource):
|
|
|
152
169
|
cache = await BlockActionObjectCache.initialize_cache()
|
|
153
170
|
cache.update_custom_block_template(self.model, remove=True)
|
|
154
171
|
|
|
155
|
-
for key, value in ignore_keys(
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
172
|
+
for key, value in ignore_keys(
|
|
173
|
+
payload,
|
|
174
|
+
[
|
|
175
|
+
'uuid',
|
|
176
|
+
OBJECT_TYPE_KEY,
|
|
177
|
+
],
|
|
178
|
+
).items():
|
|
159
179
|
setattr(self.model, key, value)
|
|
160
180
|
self.model.save()
|
|
161
181
|
|
|
@@ -43,7 +43,7 @@ class DataProviderResource(GenericResource):
|
|
|
43
43
|
@classmethod
|
|
44
44
|
@safe_db_query
|
|
45
45
|
async def collection(self, query, meta, user, **kwargs):
|
|
46
|
-
file_path = f'{get_repo_path()}/io_config.yaml'
|
|
46
|
+
file_path = f'{get_repo_path(user=user)}/io_config.yaml'
|
|
47
47
|
profiles = []
|
|
48
48
|
if os.path.exists(file_path):
|
|
49
49
|
async with aiofiles.open(file_path, mode='r') as file:
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from mage_ai.api.resources.GenericResource import GenericResource
|
|
2
2
|
from mage_ai.data_preparation.models.constants import PipelineType
|
|
3
3
|
from mage_ai.data_preparation.models.pipeline import Pipeline
|
|
4
|
+
from mage_ai.settings.repo import get_repo_path
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
class ExecutionStateResource(GenericResource):
|
|
@@ -17,7 +18,8 @@ class ExecutionStateResource(GenericResource):
|
|
|
17
18
|
block_uuid = block_uuid[0]
|
|
18
19
|
|
|
19
20
|
if pipeline_uuid and block_uuid:
|
|
20
|
-
|
|
21
|
+
repo_path = get_repo_path(user=user)
|
|
22
|
+
pipeline = await Pipeline.get_async(pipeline_uuid, repo_path=repo_path)
|
|
21
23
|
if pipeline and pipeline.type in [PipelineType.PYTHON, PipelineType.PYSPARK]:
|
|
22
24
|
block = pipeline.get_block(block_uuid)
|
|
23
25
|
if block and block.compute_management_enabled() and block.is_using_spark():
|
|
@@ -22,7 +22,10 @@ class FileContentResource(GenericResource):
|
|
|
22
22
|
if 'payload' in kwargs and 'file_content' in kwargs['payload']:
|
|
23
23
|
payload = kwargs['payload']['file_content']
|
|
24
24
|
if 'block_uuid' in payload and 'pipeline_uuid' in payload:
|
|
25
|
-
|
|
25
|
+
repo_path = get_repo_path(user=user)
|
|
26
|
+
pipeline = await Pipeline.get_async(
|
|
27
|
+
payload.get('pipeline_uuid'), repo_path=repo_path
|
|
28
|
+
)
|
|
26
29
|
if pipeline:
|
|
27
30
|
block = pipeline.get_block(payload.get('block_uuid'))
|
|
28
31
|
if block:
|
|
@@ -57,7 +60,10 @@ class FileContentResource(GenericResource):
|
|
|
57
60
|
|
|
58
61
|
await self.model.update_content_async(content)
|
|
59
62
|
|
|
60
|
-
block_type = Block.block_type_from_path(
|
|
63
|
+
block_type = Block.block_type_from_path(
|
|
64
|
+
self.model.dir_path,
|
|
65
|
+
repo_path=get_repo_path(user=self.current_user),
|
|
66
|
+
)
|
|
61
67
|
if block_type:
|
|
62
68
|
cache_block_action_object = await BlockActionObjectCache.initialize_cache()
|
|
63
69
|
cache_block_action_object.update_block(block_file_absolute_path=self.model.file_path)
|