mage-ai 0.9.70__py3-none-any.whl → 0.9.71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mage-ai might be problematic. Click here for more details.
- mage_ai/ai/utils/xgboost.py +222 -0
- mage_ai/api/errors.py +37 -25
- mage_ai/api/operations/base.py +13 -1
- mage_ai/api/parsers/PipelineScheduleParser.py +1 -1
- mage_ai/api/policies/BlockOutputPolicy.py +40 -17
- mage_ai/api/policies/GlobalDataProductPolicy.py +91 -41
- mage_ai/api/policies/KernelPolicy.py +55 -32
- mage_ai/api/policies/KernelProcessPolicy.py +56 -0
- mage_ai/api/policies/OutputPolicy.py +73 -41
- mage_ai/api/policies/PipelinePolicy.py +206 -138
- mage_ai/api/presenters/BlockLayoutItemPresenter.py +9 -7
- mage_ai/api/presenters/BlockPresenter.py +1 -1
- mage_ai/api/presenters/GlobalDataProductPresenter.py +6 -1
- mage_ai/api/presenters/KernelPresenter.py +5 -26
- mage_ai/api/presenters/KernelProcessPresenter.py +28 -0
- mage_ai/api/presenters/PipelinePresenter.py +18 -5
- mage_ai/api/presenters/StatusPresenter.py +2 -0
- mage_ai/api/presenters/SyncPresenter.py +25 -0
- mage_ai/api/resources/AutocompleteItemResource.py +1 -1
- mage_ai/api/resources/BlockLayoutItemResource.py +90 -44
- mage_ai/api/resources/BlockOutputResource.py +42 -9
- mage_ai/api/resources/BlockResource.py +4 -3
- mage_ai/api/resources/BlockRunResource.py +27 -22
- mage_ai/api/resources/ClusterResource.py +4 -1
- mage_ai/api/resources/CustomTemplateResource.py +34 -14
- mage_ai/api/resources/DataProviderResource.py +1 -1
- mage_ai/api/resources/ExecutionStateResource.py +3 -1
- mage_ai/api/resources/FileContentResource.py +8 -2
- mage_ai/api/resources/FileResource.py +10 -4
- mage_ai/api/resources/FileVersionResource.py +3 -1
- mage_ai/api/resources/GitBranchResource.py +46 -9
- mage_ai/api/resources/GlobalDataProductResource.py +44 -7
- mage_ai/api/resources/GlobalHookResource.py +4 -1
- mage_ai/api/resources/IntegrationDestinationResource.py +6 -2
- mage_ai/api/resources/IntegrationSourceResource.py +8 -4
- mage_ai/api/resources/IntegrationSourceStreamResource.py +6 -2
- mage_ai/api/resources/KernelProcessResource.py +44 -0
- mage_ai/api/resources/KernelResource.py +25 -3
- mage_ai/api/resources/OutputResource.py +33 -11
- mage_ai/api/resources/PageBlockLayoutResource.py +34 -23
- mage_ai/api/resources/PipelineInteractionResource.py +31 -15
- mage_ai/api/resources/PipelineResource.py +250 -123
- mage_ai/api/resources/PipelineRunResource.py +11 -3
- mage_ai/api/resources/PipelineScheduleResource.py +7 -2
- mage_ai/api/resources/PipelineTriggerResource.py +6 -1
- mage_ai/api/resources/ProjectResource.py +18 -7
- mage_ai/api/resources/SecretResource.py +1 -1
- mage_ai/api/resources/SeedResource.py +8 -1
- mage_ai/api/resources/StatusResource.py +21 -6
- mage_ai/api/resources/SyncResource.py +6 -8
- mage_ai/api/resources/VariableResource.py +46 -26
- mage_ai/api/resources/VersionControlProjectResource.py +9 -2
- mage_ai/api/resources/WidgetResource.py +1 -1
- mage_ai/api/resources/WorkspaceResource.py +1 -1
- mage_ai/api/views.py +47 -40
- mage_ai/authentication/permissions/seed.py +16 -2
- mage_ai/authentication/providers/oidc.py +21 -1
- mage_ai/autocomplete/utils.py +13 -9
- mage_ai/cache/base.py +1 -1
- mage_ai/cache/block.py +18 -12
- mage_ai/cache/block_action_object/__init__.py +32 -4
- mage_ai/cache/file.py +22 -19
- mage_ai/cache/pipeline.py +18 -12
- mage_ai/cli/main.py +1 -0
- mage_ai/cluster_manager/aws/emr_cluster_manager.py +9 -5
- mage_ai/cluster_manager/config.py +2 -2
- mage_ai/cluster_manager/manage.py +1 -1
- mage_ai/cluster_manager/workspace/base.py +1 -1
- mage_ai/command_center/applications/factory.py +10 -7
- mage_ai/command_center/files/factory.py +17 -15
- mage_ai/command_center/utils.py +25 -13
- mage_ai/data/__init__.py +0 -0
- mage_ai/data/constants.py +45 -0
- mage_ai/data/models/__init__.py +0 -0
- mage_ai/data/models/base.py +119 -0
- mage_ai/data/models/constants.py +1 -0
- mage_ai/data/models/generator.py +115 -0
- mage_ai/data/models/manager.py +168 -0
- mage_ai/data/models/pyarrow/__init__.py +0 -0
- mage_ai/data/models/pyarrow/record_batch.py +55 -0
- mage_ai/data/models/pyarrow/shared.py +21 -0
- mage_ai/data/models/pyarrow/table.py +8 -0
- mage_ai/data/models/reader.py +103 -0
- mage_ai/data/models/utils.py +59 -0
- mage_ai/data/models/writer.py +91 -0
- mage_ai/data/tabular/__init__.py +0 -0
- mage_ai/data/tabular/constants.py +23 -0
- mage_ai/data/tabular/mocks.py +19 -0
- mage_ai/data/tabular/models.py +126 -0
- mage_ai/data/tabular/reader.py +602 -0
- mage_ai/data/tabular/utils.py +102 -0
- mage_ai/data/tabular/writer.py +266 -0
- mage_ai/data/variables/__init__.py +0 -0
- mage_ai/data/variables/wrapper.py +54 -0
- mage_ai/data_cleaner/analysis/charts.py +61 -39
- mage_ai/data_cleaner/column_types/column_type_detector.py +53 -31
- mage_ai/data_cleaner/estimators/encoders.py +5 -2
- mage_ai/data_integrations/utils/scheduler.py +16 -11
- mage_ai/data_preparation/decorators.py +1 -0
- mage_ai/data_preparation/executors/block_executor.py +237 -155
- mage_ai/data_preparation/executors/streaming_pipeline_executor.py +1 -1
- mage_ai/data_preparation/git/__init__.py +27 -7
- mage_ai/data_preparation/git/api.py +7 -1
- mage_ai/data_preparation/git/utils.py +22 -16
- mage_ai/data_preparation/logging/logger_manager.py +4 -3
- mage_ai/data_preparation/models/block/__init__.py +1542 -878
- mage_ai/data_preparation/models/block/data_integration/mixins.py +4 -3
- mage_ai/data_preparation/models/block/dynamic/__init__.py +17 -6
- mage_ai/data_preparation/models/block/dynamic/child.py +41 -102
- mage_ai/data_preparation/models/block/dynamic/constants.py +1 -0
- mage_ai/data_preparation/models/block/dynamic/counter.py +296 -0
- mage_ai/data_preparation/models/block/dynamic/data.py +16 -0
- mage_ai/data_preparation/models/block/dynamic/factory.py +163 -0
- mage_ai/data_preparation/models/block/dynamic/models.py +19 -0
- mage_ai/data_preparation/models/block/dynamic/shared.py +92 -0
- mage_ai/data_preparation/models/block/dynamic/utils.py +291 -168
- mage_ai/data_preparation/models/block/dynamic/variables.py +384 -144
- mage_ai/data_preparation/models/block/dynamic/wrappers.py +77 -0
- mage_ai/data_preparation/models/block/extension/utils.py +10 -1
- mage_ai/data_preparation/models/block/global_data_product/__init__.py +10 -1
- mage_ai/data_preparation/models/block/integration/__init__.py +6 -2
- mage_ai/data_preparation/models/block/outputs.py +722 -0
- mage_ai/data_preparation/models/block/platform/mixins.py +7 -8
- mage_ai/data_preparation/models/block/r/__init__.py +56 -38
- mage_ai/data_preparation/models/block/settings/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/dynamic/constants.py +7 -0
- mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +118 -0
- mage_ai/data_preparation/models/block/settings/dynamic/models.py +31 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/mixins.py +20 -0
- mage_ai/data_preparation/models/block/settings/global_data_products/models.py +46 -0
- mage_ai/data_preparation/models/block/settings/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/block/settings/variables/mixins.py +74 -0
- mage_ai/data_preparation/models/block/settings/variables/models.py +49 -0
- mage_ai/data_preparation/models/block/spark/mixins.py +2 -1
- mage_ai/data_preparation/models/block/sql/__init__.py +30 -5
- mage_ai/data_preparation/models/block/sql/utils/shared.py +21 -3
- mage_ai/data_preparation/models/block/utils.py +127 -70
- mage_ai/data_preparation/models/constants.py +19 -14
- mage_ai/data_preparation/models/custom_templates/custom_block_template.py +18 -13
- mage_ai/data_preparation/models/custom_templates/custom_pipeline_template.py +33 -16
- mage_ai/data_preparation/models/custom_templates/utils.py +1 -1
- mage_ai/data_preparation/models/file.py +41 -28
- mage_ai/data_preparation/models/global_data_product/__init__.py +88 -58
- mage_ai/data_preparation/models/global_hooks/models.py +1 -0
- mage_ai/data_preparation/models/interfaces.py +29 -0
- mage_ai/data_preparation/models/pipeline.py +365 -180
- mage_ai/data_preparation/models/pipelines/integration_pipeline.py +1 -2
- mage_ai/data_preparation/models/pipelines/seed.py +1 -1
- mage_ai/data_preparation/models/project/__init__.py +66 -18
- mage_ai/data_preparation/models/project/constants.py +2 -0
- mage_ai/data_preparation/models/triggers/__init__.py +120 -24
- mage_ai/data_preparation/models/utils.py +467 -17
- mage_ai/data_preparation/models/variable.py +1028 -137
- mage_ai/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/data_preparation/models/variables/cache.py +149 -0
- mage_ai/data_preparation/models/variables/constants.py +72 -0
- mage_ai/data_preparation/models/variables/summarizer.py +336 -0
- mage_ai/data_preparation/models/variables/utils.py +77 -0
- mage_ai/data_preparation/models/widget/__init__.py +63 -41
- mage_ai/data_preparation/models/widget/charts.py +40 -27
- mage_ai/data_preparation/models/widget/constants.py +2 -0
- mage_ai/data_preparation/models/widget/utils.py +3 -3
- mage_ai/data_preparation/preferences.py +3 -3
- mage_ai/data_preparation/repo_manager.py +55 -21
- mage_ai/data_preparation/storage/base_storage.py +2 -2
- mage_ai/data_preparation/storage/gcs_storage.py +7 -4
- mage_ai/data_preparation/storage/local_storage.py +6 -3
- mage_ai/data_preparation/storage/s3_storage.py +5 -2
- mage_ai/data_preparation/templates/data_exporters/streaming/oracledb.yaml +8 -0
- mage_ai/data_preparation/variable_manager.py +281 -76
- mage_ai/io/base.py +3 -2
- mage_ai/io/bigquery.py +1 -0
- mage_ai/io/redshift.py +7 -5
- mage_ai/kernels/__init__.py +0 -0
- mage_ai/kernels/models.py +188 -0
- mage_ai/kernels/utils.py +169 -0
- mage_ai/orchestration/concurrency.py +6 -2
- mage_ai/orchestration/db/__init__.py +1 -0
- mage_ai/orchestration/db/migrations/versions/0227396a216c_add_userproject_table.py +38 -0
- mage_ai/orchestration/db/models/dynamic/__init__.py +0 -0
- mage_ai/orchestration/db/models/dynamic/controller.py +67 -0
- mage_ai/orchestration/db/models/oauth.py +2 -9
- mage_ai/orchestration/db/models/projects.py +10 -0
- mage_ai/orchestration/db/models/schedules.py +204 -187
- mage_ai/orchestration/db/models/schedules_project_platform.py +18 -12
- mage_ai/orchestration/db/models/utils.py +46 -5
- mage_ai/orchestration/metrics/pipeline_run.py +8 -9
- mage_ai/orchestration/notification/sender.py +1 -0
- mage_ai/orchestration/pipeline_scheduler_original.py +32 -8
- mage_ai/orchestration/pipeline_scheduler_project_platform.py +1 -1
- mage_ai/orchestration/run_status_checker.py +11 -4
- mage_ai/orchestration/triggers/api.py +12 -1
- mage_ai/presenters/charts/data_sources/base.py +4 -2
- mage_ai/presenters/charts/data_sources/block.py +15 -9
- mage_ai/presenters/charts/data_sources/chart_code.py +8 -5
- mage_ai/presenters/charts/data_sources/constants.py +1 -0
- mage_ai/presenters/charts/data_sources/system_metrics.py +22 -0
- mage_ai/presenters/interactions/models.py +11 -7
- mage_ai/presenters/pages/loaders/pipelines.py +5 -3
- mage_ai/presenters/pages/models/page_components/pipeline_schedules.py +3 -1
- mage_ai/presenters/utils.py +2 -0
- mage_ai/server/api/blocks.py +2 -1
- mage_ai/server/api/downloads.py +5 -1
- mage_ai/server/api/triggers.py +3 -1
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +5 -5
- mage_ai/server/frontend_dist/_next/static/UZLabyPgcxtZvp0O0EUUS/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{1557-b3502f3f1aa92ac7.js → 1557-25a7d985d5564fd3.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/3763-61b542dafdbf5754.js → frontend_dist/_next/static/chunks/3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{main-77fe248a6fbd12d8.js → main-b99d4e30a88d9dc7.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-9fe2d9d07c94e968.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/overview-597b74828bf105db.js → frontend_dist/_next/static/chunks/pages/overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-7181b086c93784d2.js → [...slug]-7e737f6fc7e83e9b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js → frontend_dist/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{webpack-d079359c241db804.js → webpack-ac7fdc472bedf682.js} +1 -1
- mage_ai/server/frontend_dist/block-layout.html +3 -3
- mage_ai/server/frontend_dist/compute.html +6 -6
- mage_ai/server/frontend_dist/files.html +6 -6
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-data-products.html +6 -6
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/index.html +3 -3
- mage_ai/server/frontend_dist/manage/files.html +6 -6
- mage_ai/server/frontend_dist/manage/settings.html +6 -6
- mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist/manage/users.html +6 -6
- mage_ai/server/frontend_dist/manage.html +6 -6
- mage_ai/server/frontend_dist/oauth.html +5 -5
- mage_ai/server/frontend_dist/overview.html +6 -6
- mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist/pipelines.html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist/settings.html +3 -3
- mage_ai/server/frontend_dist/sign-in.html +12 -12
- mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist/templates.html +6 -6
- mage_ai/server/frontend_dist/terminal.html +6 -6
- mage_ai/server/frontend_dist/test.html +3 -3
- mage_ai/server/frontend_dist/triggers.html +6 -6
- mage_ai/server/frontend_dist/version-control.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/404.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1376-22de38b4ad008d8a.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{1557-b3502f3f1aa92ac7.js → 1557-25a7d985d5564fd3.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1668-30b4619b9534519b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1799-c42db95a015689ee.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2996-2108b53b9d371d8d.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/3763-61b542dafdbf5754.js → frontend_dist_base_path_template/_next/static/chunks/3763-40780c6d1e4b261d.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-129dd2a2448a2e36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3958-bcdfa414ccfa1eb2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4168-97fd1578d1a38315.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4982-fa5a238b139fbdd2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-176f445e1313f001.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7162-7dd03f0f605de721.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7779-68d2b72a90c5f925.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-5446a8e43711e2f9.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8023-6c2f172f48dcb99b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-c351b8a735d73e0c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{main-70b78159c2bb3fe1.js → main-384298e9133cec76.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-13a578bce3b7f30c.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{block-layout-14f952f66964022f.js → block-layout-7f4b735c67115df5.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-e7d48e6b0c3068ac.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-b943f31f050fc3a4.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/overview-597b74828bf105db.js → frontend_dist_base_path_template/_next/static/chunks/pages/overview-9f1ac4ec003884f3.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-7181b086c93784d2.js → [...slug]-7e737f6fc7e83e9b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-d94488e3f2eeef36.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-cc641a7fa8473796.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/{block-runs-a5c0362763a21fa8.js → block-runs-284309877f3c5a5a.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-26250e5335194ade.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors-7acc7afc00df17c2.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors-5f4c8128b2413fd8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-4ebfc8e400315dda.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-e5e0150a256aadb3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-3f0df3decc856ee9.js → profile-3ae43c932537b254.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/preferences-32985f3f7c7dd3ab.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-b603d7fe4b175256.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-c2e9ef989c8bfa73.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-319ddbabc239e91b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-47b64ced27c24985.js → [...slug]-5c360f72e4498855.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e5a4d3d815cec25d.js → permissions-fb29fa6c2bd90bb0.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-3b76fa959ffa09d3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-379e1ee292504842.js → [...slug]-3b787b42f1093b1f.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-0b83fbdd39e85f5b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-a1e6950974d643a8.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-2af9afbe727d88aa.js → [...slug]-0aa019d87db8b0b8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{users-a4db8710f703c729.js → users-88c694d19207f2ec.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-31d0d50f7f30462b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-68c003fb6a175cd7.js → webpack-481689d9989710cd.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/kcptwoOU-JJJg6Vwpkfmx/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +12 -12
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
- mage_ai/server/kernel_output_parser.py +4 -1
- mage_ai/server/scheduler_manager.py +9 -0
- mage_ai/server/server.py +35 -31
- mage_ai/server/utils/custom_output.py +284 -0
- mage_ai/server/utils/execute_custom_code.py +245 -0
- mage_ai/server/utils/output_display.py +123 -289
- mage_ai/server/websocket_server.py +116 -69
- mage_ai/services/k8s/config.py +23 -0
- mage_ai/services/k8s/job_manager.py +6 -1
- mage_ai/services/ssh/aws/emr/utils.py +8 -8
- mage_ai/settings/keys/auth.py +1 -0
- mage_ai/settings/platform/__init__.py +159 -38
- mage_ai/settings/platform/constants.py +5 -0
- mage_ai/settings/platform/utils.py +53 -10
- mage_ai/settings/repo.py +26 -12
- mage_ai/settings/server.py +128 -37
- mage_ai/shared/array.py +24 -1
- mage_ai/shared/complex.py +45 -0
- mage_ai/shared/config.py +2 -1
- mage_ai/shared/custom_logger.py +11 -0
- mage_ai/shared/dates.py +10 -6
- mage_ai/shared/files.py +63 -8
- mage_ai/shared/hash.py +33 -9
- mage_ai/shared/io.py +9 -5
- mage_ai/shared/models.py +82 -24
- mage_ai/shared/outputs.py +87 -0
- mage_ai/shared/parsers.py +141 -15
- mage_ai/shared/path_fixer.py +11 -7
- mage_ai/shared/singletons/__init__.py +0 -0
- mage_ai/shared/singletons/base.py +47 -0
- mage_ai/shared/singletons/memory.py +38 -0
- mage_ai/shared/strings.py +34 -1
- mage_ai/shared/yaml.py +24 -0
- mage_ai/streaming/sinks/oracledb.py +57 -0
- mage_ai/streaming/sinks/sink_factory.py +4 -0
- mage_ai/system/__init__.py +0 -0
- mage_ai/system/constants.py +14 -0
- mage_ai/system/memory/__init__.py +0 -0
- mage_ai/system/memory/constants.py +1 -0
- mage_ai/system/memory/manager.py +174 -0
- mage_ai/system/memory/presenters.py +158 -0
- mage_ai/system/memory/process.py +216 -0
- mage_ai/system/memory/samples.py +13 -0
- mage_ai/system/memory/utils.py +656 -0
- mage_ai/system/memory/wrappers.py +177 -0
- mage_ai/system/models.py +58 -0
- mage_ai/system/storage/__init__.py +0 -0
- mage_ai/system/storage/utils.py +29 -0
- mage_ai/tests/api/endpoints/mixins.py +2 -2
- mage_ai/tests/api/endpoints/test_blocks.py +2 -1
- mage_ai/tests/api/endpoints/test_custom_designs.py +4 -4
- mage_ai/tests/api/endpoints/test_pipeline_runs.py +2 -2
- mage_ai/tests/api/endpoints/test_projects.py +2 -1
- mage_ai/tests/api/operations/base/test_base.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_authentication.py +27 -27
- mage_ai/tests/api/operations/base/test_base_with_user_permissions.py +23 -23
- mage_ai/tests/api/operations/test_syncs.py +6 -4
- mage_ai/tests/api/resources/test_pipeline_resource.py +9 -2
- mage_ai/tests/authentication/providers/test_oidc.py +59 -0
- mage_ai/tests/base_test.py +2 -2
- mage_ai/tests/data/__init__.py +0 -0
- mage_ai/tests/data/models/__init__.py +0 -0
- mage_ai/tests/data_preparation/executors/test_block_executor.py +23 -16
- mage_ai/tests/data_preparation/git/test_git.py +4 -1
- mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +305 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +212 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_factory.py +360 -0
- mage_ai/tests/data_preparation/models/block/dynamic/test_variables.py +332 -0
- mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +2 -2
- mage_ai/tests/data_preparation/models/block/platform/test_mixins.py +1 -1
- mage_ai/tests/data_preparation/models/block/sql/utils/test_shared.py +26 -1
- mage_ai/tests/data_preparation/models/block/test_global_data_product.py +3 -2
- mage_ai/tests/data_preparation/models/custom_templates/test_utils.py +5 -4
- mage_ai/tests/data_preparation/models/global_hooks/test_hook.py +3 -0
- mage_ai/tests/data_preparation/models/global_hooks/test_predicates.py +9 -3
- mage_ai/tests/data_preparation/models/test_block.py +115 -120
- mage_ai/tests/data_preparation/models/test_blocks_helper.py +114 -0
- mage_ai/tests/data_preparation/models/test_global_data_product.py +41 -24
- mage_ai/tests/data_preparation/models/test_pipeline.py +9 -6
- mage_ai/tests/data_preparation/models/test_project.py +4 -1
- mage_ai/tests/data_preparation/models/test_utils.py +80 -0
- mage_ai/tests/data_preparation/models/test_variable.py +242 -69
- mage_ai/tests/data_preparation/models/variables/__init__.py +0 -0
- mage_ai/tests/data_preparation/models/variables/test_summarizer.py +481 -0
- mage_ai/tests/data_preparation/storage/shared/__init__.py +0 -0
- mage_ai/tests/data_preparation/test_repo_manager.py +6 -7
- mage_ai/tests/data_preparation/test_variable_manager.py +57 -48
- mage_ai/tests/factory.py +64 -43
- mage_ai/tests/orchestration/db/models/test_schedules.py +3 -3
- mage_ai/tests/orchestration/db/models/test_schedules_dynamic_blocks.py +279 -0
- mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
- mage_ai/tests/orchestration/triggers/test_global_data_product.py +3 -2
- mage_ai/tests/orchestration/triggers/test_utils.py +3 -2
- mage_ai/tests/services/k8s/test_job_manager.py +18 -0
- mage_ai/tests/streaming/sinks/test_oracledb.py +38 -0
- mage_ai/tests/test_shared.py +61 -0
- mage_ai/usage_statistics/logger.py +7 -2
- mage_ai/utils/code.py +33 -19
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/METADATA +5 -2
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/RECORD +513 -417
- mage_ai/data_preparation/models/global_data_product/constants.py +0 -6
- mage_ai/server/frontend_dist/_next/static/RhDiJSkcjCsh4xxX4BFBk/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/7966-b9b85ba10667e654.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/TdpLLFome13qvM0gXvpHs/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2631-b9f9bea3f1cf906d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-ef4cd4f0b52072d0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/4783-422429203610c318.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/5699-6d708c6b2153ea08.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/635-0d6b7c8804bcd2dc.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7022-0d52dd8868621fb0.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7361-8a23dd8360593e7a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/7966-b9b85ba10667e654.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8095-bdce03896ef9639a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/8146-6bed4e7401e067e6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9265-d2a1aaec75ec69b8.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9440-4069842b90d4b801.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/9832-67896490f6e8a014.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-2a69553d8c6eeb53.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/[...slug]-591abd392dc50ed4.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products-78e8e88f2a757a18.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/dashboard-95ffcd3e2b27e567.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-b645a6d13ab9fe3a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-1ed9045b2f1dfd65.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-1417ad1c821d720a.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-59aca25a5b1d3998.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-349af617d05f001b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles-36fa165a48af586b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/sync-data-60d01d3887e31136.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/version-control-3433c8b22e8342aa.js +0 -1
- mage_ai/shared/memory.py +0 -90
- mage_ai/tests/data_preparation/models/block/dynamic/test_dynamic_helpers.py +0 -48
- /mage_ai/{tests/data_preparation/shared → ai/utils}/__init__.py +0 -0
- /mage_ai/server/frontend_dist/_next/static/{RhDiJSkcjCsh4xxX4BFBk → UZLabyPgcxtZvp0O0EUUS}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{TdpLLFome13qvM0gXvpHs → kcptwoOU-JJJg6Vwpkfmx}/_ssgManifest.js +0 -0
- /mage_ai/tests/data_preparation/{shared → storage/shared}/test_secrets.py +0 -0
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/WHEEL +0 -0
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.70.dist-info → mage_ai-0.9.71.dist-info}/top_level.txt +0 -0
|
@@ -8,6 +8,7 @@ import pandas as pd
|
|
|
8
8
|
from mage_ai.data_preparation.models.block import Block
|
|
9
9
|
from mage_ai.data_preparation.models.constants import DATAFRAME_SAMPLE_COUNT_PREVIEW
|
|
10
10
|
from mage_ai.shared.hash import merge_dict
|
|
11
|
+
from mage_ai.shared.parsers import convert_matrix_to_dataframe
|
|
11
12
|
from mage_ai.shared.strings import is_number
|
|
12
13
|
|
|
13
14
|
from .charts import (
|
|
@@ -22,9 +23,11 @@ from .constants import (
|
|
|
22
23
|
VARIABLE_NAME_INDEX,
|
|
23
24
|
VARIABLE_NAME_LIMIT,
|
|
24
25
|
VARIABLE_NAME_METRICS,
|
|
26
|
+
VARIABLE_NAME_ORDER_BY,
|
|
25
27
|
VARIABLE_NAME_TIME_INTERVAL,
|
|
26
28
|
VARIABLE_NAME_X,
|
|
27
29
|
VARIABLE_NAME_Y,
|
|
30
|
+
VARIABLE_NAME_Y_SORT_ORDER,
|
|
28
31
|
VARIABLE_NAMES_BY_CHART_TYPE,
|
|
29
32
|
ChartType,
|
|
30
33
|
)
|
|
@@ -99,8 +102,17 @@ class Widget(Block):
|
|
|
99
102
|
else:
|
|
100
103
|
dfs.append(input_var)
|
|
101
104
|
|
|
102
|
-
|
|
103
|
-
|
|
105
|
+
arr = []
|
|
106
|
+
for d in dfs:
|
|
107
|
+
if isinstance(d, list):
|
|
108
|
+
arr += d
|
|
109
|
+
else:
|
|
110
|
+
arr.append(d)
|
|
111
|
+
dfs = [convert_matrix_to_dataframe(d) for d in arr]
|
|
112
|
+
|
|
113
|
+
should_use_no_code = (
|
|
114
|
+
x_values is None and y_values is None and (group_by_columns or metrics)
|
|
115
|
+
)
|
|
104
116
|
|
|
105
117
|
if should_use_no_code and len(dfs) == 0:
|
|
106
118
|
return data
|
|
@@ -153,13 +165,11 @@ class Widget(Block):
|
|
|
153
165
|
data = build_x_y(df, group_by_columns, metrics)
|
|
154
166
|
else:
|
|
155
167
|
for var_name_orig, _var_name in self.output_variable_names:
|
|
156
|
-
data.update(
|
|
157
|
-
|
|
158
|
-
var_name_orig
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
}
|
|
162
|
-
)
|
|
168
|
+
data.update({
|
|
169
|
+
var_name_orig: encode_values_in_list(
|
|
170
|
+
convert_to_list(variables[var_name_orig])
|
|
171
|
+
),
|
|
172
|
+
})
|
|
163
173
|
elif ChartType.PIE_CHART == chart_type:
|
|
164
174
|
arr1 = []
|
|
165
175
|
data_key = VARIABLE_NAME_X
|
|
@@ -188,20 +198,30 @@ class Widget(Block):
|
|
|
188
198
|
value_counts[key] = 0
|
|
189
199
|
value_counts[key] += 1
|
|
190
200
|
|
|
191
|
-
buckets = int(self.configuration.get(VARIABLE_NAME_BUCKETS
|
|
201
|
+
buckets = int(self.configuration.get(VARIABLE_NAME_BUCKETS) or MAX_BUCKETS)
|
|
192
202
|
arr = sorted(
|
|
193
203
|
list(zip(value_counts.values(), value_counts.keys())),
|
|
194
204
|
reverse=True,
|
|
195
205
|
)[:buckets]
|
|
196
206
|
data[data_key] = {k: v for v, k in arr}
|
|
197
207
|
elif ChartType.TABLE == chart_type:
|
|
198
|
-
limit_config =
|
|
199
|
-
DATAFRAME_SAMPLE_COUNT_PREVIEW
|
|
208
|
+
limit_config = (
|
|
209
|
+
self.configuration.get(VARIABLE_NAME_LIMIT) or DATAFRAME_SAMPLE_COUNT_PREVIEW
|
|
210
|
+
)
|
|
200
211
|
if is_number(limit_config):
|
|
201
212
|
limit_config = int(limit_config)
|
|
202
213
|
|
|
203
214
|
if should_use_no_code:
|
|
204
|
-
df = dfs[0]
|
|
215
|
+
df = dfs[0]
|
|
216
|
+
order_by = self.configuration.get(VARIABLE_NAME_ORDER_BY)
|
|
217
|
+
if order_by:
|
|
218
|
+
df.sort_values(
|
|
219
|
+
by=order_by,
|
|
220
|
+
ascending=self.configuration.get(VARIABLE_NAME_Y_SORT_ORDER)
|
|
221
|
+
!= 'descending',
|
|
222
|
+
inplace=True,
|
|
223
|
+
)
|
|
224
|
+
df = df.iloc[:limit_config]
|
|
205
225
|
if group_by_columns:
|
|
206
226
|
data[VARIABLE_NAME_X] = group_by_columns
|
|
207
227
|
data[VARIABLE_NAME_Y] = df[group_by_columns].to_numpy()
|
|
@@ -214,14 +234,15 @@ class Widget(Block):
|
|
|
214
234
|
if var_name_orig in [VARIABLE_NAME_Y, VARIABLE_NAME_INDEX]:
|
|
215
235
|
limit = limit_config
|
|
216
236
|
|
|
217
|
-
data.update(
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
237
|
+
data.update({
|
|
238
|
+
var_name_orig: encode_values_in_list(
|
|
239
|
+
convert_to_list(arr, limit=limit)
|
|
240
|
+
),
|
|
241
|
+
})
|
|
242
|
+
elif chart_type in [
|
|
243
|
+
ChartType.TIME_SERIES_BAR_CHART,
|
|
244
|
+
ChartType.TIME_SERIES_LINE_CHART,
|
|
245
|
+
]:
|
|
225
246
|
if should_use_no_code:
|
|
226
247
|
df = dfs[0]
|
|
227
248
|
if group_by_columns and metrics:
|
|
@@ -269,16 +290,19 @@ class Widget(Block):
|
|
|
269
290
|
decorated_functions_y = []
|
|
270
291
|
test_functions = []
|
|
271
292
|
|
|
272
|
-
results = merge_dict(
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
293
|
+
results = merge_dict(
|
|
294
|
+
dict(
|
|
295
|
+
columns=self._block_decorator(decorated_functions_columns),
|
|
296
|
+
configuration=self._block_decorator(decorated_functions_configuration),
|
|
297
|
+
data_source=self._block_decorator(decorated_functions_data_source),
|
|
298
|
+
render=self._block_decorator_render(decorated_functions_render),
|
|
299
|
+
test=self._block_decorator(test_functions),
|
|
300
|
+
x=self._block_decorator(decorated_functions_x),
|
|
301
|
+
xy=self._block_decorator(decorated_functions_xy),
|
|
302
|
+
y=self._block_decorator(decorated_functions_y),
|
|
303
|
+
),
|
|
304
|
+
outputs_from_input_vars,
|
|
305
|
+
)
|
|
282
306
|
|
|
283
307
|
inputs_vars_use = list()
|
|
284
308
|
if input_vars is not None:
|
|
@@ -290,8 +314,7 @@ class Widget(Block):
|
|
|
290
314
|
metrics = chart_configuration_settings['metrics']
|
|
291
315
|
|
|
292
316
|
if custom_code is not None and custom_code.strip():
|
|
293
|
-
|
|
294
|
-
exec(custom_code, results)
|
|
317
|
+
exec(custom_code, results)
|
|
295
318
|
elif self.content is not None:
|
|
296
319
|
exec(self.content, results)
|
|
297
320
|
elif os.path.exists(self.file_path):
|
|
@@ -354,18 +377,16 @@ class Widget(Block):
|
|
|
354
377
|
else:
|
|
355
378
|
item = None
|
|
356
379
|
|
|
357
|
-
if
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
380
|
+
if (
|
|
381
|
+
input_vars_from_data_source is not None
|
|
382
|
+
and isinstance(input_vars_from_data_source, list)
|
|
383
|
+
and len(input_vars_from_data_source) >= 1
|
|
384
|
+
):
|
|
361
385
|
item = input_vars_from_data_source[0]
|
|
362
386
|
else:
|
|
363
387
|
item = input_vars_from_data_source
|
|
364
388
|
|
|
365
|
-
if item is not None and
|
|
366
|
-
isinstance(item, list) and \
|
|
367
|
-
len(item) >= 1:
|
|
368
|
-
|
|
389
|
+
if item is not None and isinstance(item, list) and len(item) >= 1:
|
|
369
390
|
item = item[0]
|
|
370
391
|
|
|
371
392
|
if item is not None:
|
|
@@ -431,6 +452,7 @@ class Widget(Block):
|
|
|
431
452
|
render = function(*args_inner, **kwargs_inner)
|
|
432
453
|
|
|
433
454
|
return merge_dict(dict(render=render), kwargs or {})
|
|
455
|
+
|
|
434
456
|
decorated_functions.append(func)
|
|
435
457
|
|
|
436
458
|
return inner
|
|
@@ -44,11 +44,13 @@ def build_buckets(min_value, max_value, max_buckets):
|
|
|
44
44
|
min_v = min_value + (i * bucket_interval)
|
|
45
45
|
max_v = min_value + ((i + 1) * bucket_interval)
|
|
46
46
|
if max_value >= min_v:
|
|
47
|
-
buckets.append(
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
buckets.append(
|
|
48
|
+
dict(
|
|
49
|
+
max_value=max_v,
|
|
50
|
+
min_value=min_v,
|
|
51
|
+
values=[],
|
|
52
|
+
)
|
|
53
|
+
)
|
|
52
54
|
|
|
53
55
|
return buckets, bucket_interval
|
|
54
56
|
|
|
@@ -69,10 +71,12 @@ def build_histogram_data(arr, max_buckets):
|
|
|
69
71
|
y = []
|
|
70
72
|
|
|
71
73
|
for idx, bucket in enumerate(buckets):
|
|
72
|
-
x.append(
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
74
|
+
x.append(
|
|
75
|
+
dict(
|
|
76
|
+
max=bucket['max_value'],
|
|
77
|
+
min=bucket['min_value'],
|
|
78
|
+
)
|
|
79
|
+
)
|
|
76
80
|
y.append(dict(value=count[idx]))
|
|
77
81
|
|
|
78
82
|
return dict(
|
|
@@ -81,6 +85,14 @@ def build_histogram_data(arr, max_buckets):
|
|
|
81
85
|
)
|
|
82
86
|
|
|
83
87
|
|
|
88
|
+
def convert_to_datetime(dt):
|
|
89
|
+
if len(str(dt)) == 10 and str(dt).isdigit():
|
|
90
|
+
return datetime.fromtimestamp(int(dt))
|
|
91
|
+
elif type(dt) is np.datetime64:
|
|
92
|
+
return pd.to_datetime(dt.astype(datetime)).to_pydatetime()
|
|
93
|
+
return dt
|
|
94
|
+
|
|
95
|
+
|
|
84
96
|
def build_time_series_buckets(
|
|
85
97
|
df,
|
|
86
98
|
datetime_column,
|
|
@@ -94,8 +106,9 @@ def build_time_series_buckets(
|
|
|
94
106
|
return []
|
|
95
107
|
|
|
96
108
|
datetimes = datetimes.unique()
|
|
97
|
-
|
|
98
|
-
|
|
109
|
+
datetimes_are_timestamps = all([len(str(dt)) == 10 and str(dt).isdigit() for dt in datetimes])
|
|
110
|
+
min_value_datetime = convert_to_datetime(datetimes.min())
|
|
111
|
+
max_value_datetime = convert_to_datetime(datetimes.max())
|
|
99
112
|
|
|
100
113
|
if type(min_value_datetime) is str:
|
|
101
114
|
min_value_datetime = dateutil.parser.parse(min_value_datetime)
|
|
@@ -104,12 +117,7 @@ def build_time_series_buckets(
|
|
|
104
117
|
|
|
105
118
|
# If you manually convert the datetime column to a datetime, Pandas will use numpy.datetime64
|
|
106
119
|
# type. This type does not have the methods year, month, day, etc that is used down below.
|
|
107
|
-
datetimes_temp = []
|
|
108
|
-
for dt in datetimes:
|
|
109
|
-
if type(dt) is np.datetime64:
|
|
110
|
-
datetimes_temp.append(pd.to_datetime(dt.astype(datetime)).to_pydatetime())
|
|
111
|
-
else:
|
|
112
|
-
datetimes_temp.append(dt)
|
|
120
|
+
datetimes_temp = [convert_to_datetime(dt) for dt in datetimes]
|
|
113
121
|
datetimes = datetimes_temp
|
|
114
122
|
if type(min_value_datetime) is np.datetime64:
|
|
115
123
|
min_value_datetime = pd.to_datetime(min_value_datetime.astype(datetime)).to_pydatetime()
|
|
@@ -169,8 +177,14 @@ def build_time_series_buckets(
|
|
|
169
177
|
start_datetime = datetime(year, 1, 1, 0, 0, 0)
|
|
170
178
|
|
|
171
179
|
df_copy = df.copy()
|
|
172
|
-
|
|
173
|
-
|
|
180
|
+
if datetimes_are_timestamps:
|
|
181
|
+
df_copy[datetime_column] = df[datetime_column].apply(
|
|
182
|
+
lambda x: x if pd.isnull(x) else int(x)
|
|
183
|
+
)
|
|
184
|
+
else:
|
|
185
|
+
df_copy[datetime_column] = pd.to_datetime(df[datetime_column]).apply(
|
|
186
|
+
lambda x: x if pd.isnull(x) else x.timestamp()
|
|
187
|
+
)
|
|
174
188
|
|
|
175
189
|
values = [[] for _ in metrics]
|
|
176
190
|
buckets = []
|
|
@@ -182,9 +196,9 @@ def build_time_series_buckets(
|
|
|
182
196
|
now + TIME_INTERVAL_TO_TIME_DELTA[time_interval]
|
|
183
197
|
).timestamp() - now.timestamp()
|
|
184
198
|
|
|
185
|
-
number_of_buckets =
|
|
186
|
-
max_value_datetime_ts - min_value_datetime.timestamp()
|
|
187
|
-
)
|
|
199
|
+
number_of_buckets = (
|
|
200
|
+
math.ceil(max_value_datetime_ts - min_value_datetime.timestamp()) / interval_seconds
|
|
201
|
+
)
|
|
188
202
|
|
|
189
203
|
max_buckets_to_use = max_buckets or MAX_BUCKETS
|
|
190
204
|
|
|
@@ -204,11 +218,10 @@ def build_time_series_buckets(
|
|
|
204
218
|
)
|
|
205
219
|
buckets.append(max_date.timestamp())
|
|
206
220
|
|
|
207
|
-
df_in_range = df_copy[
|
|
208
|
-
df_copy[datetime_column] >= min_date_ts
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
)]
|
|
221
|
+
df_in_range = df_copy[
|
|
222
|
+
(df_copy[datetime_column] >= min_date_ts)
|
|
223
|
+
& (df_copy[datetime_column] < max_date.timestamp())
|
|
224
|
+
]
|
|
212
225
|
|
|
213
226
|
for idx, metric in enumerate(metrics):
|
|
214
227
|
aggregation = metric['aggregation']
|
|
@@ -4,9 +4,11 @@ from dateutil.relativedelta import relativedelta
|
|
|
4
4
|
|
|
5
5
|
VARIABLE_NAME_BUCKETS = 'buckets'
|
|
6
6
|
VARIABLE_NAME_GROUP_BY = 'group_by'
|
|
7
|
+
VARIABLE_NAME_ORDER_BY = 'order_by'
|
|
7
8
|
VARIABLE_NAME_INDEX = 'index'
|
|
8
9
|
VARIABLE_NAME_LIMIT = 'limit'
|
|
9
10
|
VARIABLE_NAME_METRICS = 'metrics'
|
|
11
|
+
VARIABLE_NAME_Y_SORT_ORDER = 'y_sort_order'
|
|
10
12
|
VARIABLE_NAME_TIME_INTERVAL = 'time_interval'
|
|
11
13
|
VARIABLE_NAME_X = 'x'
|
|
12
14
|
VARIABLE_NAME_Y = 'y'
|
|
@@ -13,13 +13,13 @@ def clean_series(series, column_type=None, dropna=True):
|
|
|
13
13
|
if dropna:
|
|
14
14
|
series_cleaned = series_cleaned.dropna()
|
|
15
15
|
|
|
16
|
-
if column_type is
|
|
16
|
+
if column_type is float:
|
|
17
|
+
series_cleaned = series_cleaned.astype(float)
|
|
18
|
+
else:
|
|
17
19
|
try:
|
|
18
20
|
series_cleaned = series_cleaned.astype(float).astype(np.int64)
|
|
19
21
|
except ValueError:
|
|
20
22
|
series_cleaned = series_cleaned.astype(float)
|
|
21
|
-
elif column_type is float:
|
|
22
|
-
series_cleaned = series_cleaned.astype(float)
|
|
23
23
|
|
|
24
24
|
return series_cleaned
|
|
25
25
|
|
|
@@ -101,12 +101,12 @@ class Preferences:
|
|
|
101
101
|
config_dict: Dict = None,
|
|
102
102
|
user: User = None,
|
|
103
103
|
):
|
|
104
|
-
self.repo_path = repo_path or get_repo_path()
|
|
104
|
+
self.repo_path = repo_path or get_repo_path(user=user)
|
|
105
105
|
self.preferences_file_path = os.path.join(self.repo_path, PREFERENCES_FILE)
|
|
106
106
|
self.user = user
|
|
107
107
|
project_preferences = dict()
|
|
108
108
|
try:
|
|
109
|
-
if user and user.preferences and user.
|
|
109
|
+
if user and user.preferences and user.get_git_settings(self.repo_path) is None:
|
|
110
110
|
project_preferences = user.preferences
|
|
111
111
|
elif config_dict:
|
|
112
112
|
project_preferences = config_dict
|
|
@@ -120,7 +120,7 @@ class Preferences:
|
|
|
120
120
|
# Git settings
|
|
121
121
|
project_sync_config = project_preferences.get('sync_config', dict())
|
|
122
122
|
if user:
|
|
123
|
-
user_git_settings = user.
|
|
123
|
+
user_git_settings = user.get_git_settings(self.repo_path) or {}
|
|
124
124
|
project_sync_config = merge_dict(project_sync_config, user_git_settings)
|
|
125
125
|
|
|
126
126
|
self.sync_config = build_sync_config(project_sync_config)
|
|
@@ -8,20 +8,25 @@ from typing import Dict, Optional
|
|
|
8
8
|
from warnings import warn
|
|
9
9
|
|
|
10
10
|
import ruamel.yaml
|
|
11
|
-
import yaml
|
|
12
11
|
from jinja2 import Template
|
|
13
12
|
|
|
14
13
|
from mage_ai.cluster_manager.constants import ClusterType
|
|
15
14
|
from mage_ai.data_preparation.templates.utils import copy_template_directory
|
|
16
15
|
from mage_ai.settings import INITIAL_METADATA, settings
|
|
17
|
-
from mage_ai.settings.repo import
|
|
16
|
+
from mage_ai.settings.repo import (
|
|
17
|
+
DEFAULT_MAGE_DATA_DIR,
|
|
18
|
+
MAGE_DATA_DIR_ENV_VAR,
|
|
19
|
+
PROJECT_METADATA_FILENAME,
|
|
20
|
+
)
|
|
18
21
|
from mage_ai.settings.repo import get_data_dir as get_data_dir_new
|
|
19
22
|
from mage_ai.settings.repo import get_metadata_path
|
|
20
23
|
from mage_ai.settings.repo import get_repo_name as get_repo_name_new
|
|
21
24
|
from mage_ai.settings.repo import get_repo_path as get_repo_path_new
|
|
22
25
|
from mage_ai.settings.repo import get_variables_dir
|
|
23
26
|
from mage_ai.settings.repo import set_repo_path as set_repo_path_new
|
|
27
|
+
from mage_ai.settings.utils import base_repo_path
|
|
24
28
|
from mage_ai.shared.environments import is_debug
|
|
29
|
+
from mage_ai.shared.yaml import load_yaml, trim_strings
|
|
25
30
|
|
|
26
31
|
yml = ruamel.yaml.YAML()
|
|
27
32
|
yml.preserve_quotes = True
|
|
@@ -42,10 +47,16 @@ class RepoConfig:
|
|
|
42
47
|
self,
|
|
43
48
|
repo_path: str = None,
|
|
44
49
|
config_dict: Dict = None,
|
|
50
|
+
context_data: Dict = None,
|
|
45
51
|
root_project: bool = False,
|
|
52
|
+
user=None,
|
|
46
53
|
):
|
|
47
54
|
self.root_project = root_project
|
|
48
|
-
self.repo_path = repo_path or get_repo_path_new(
|
|
55
|
+
self.repo_path = repo_path or get_repo_path_new(
|
|
56
|
+
context_data=context_data,
|
|
57
|
+
root_project=self.root_project,
|
|
58
|
+
user=user,
|
|
59
|
+
)
|
|
49
60
|
self.repo_name = os.path.basename(self.repo_path)
|
|
50
61
|
self.project_uuid = None
|
|
51
62
|
self.project_type = None
|
|
@@ -77,14 +88,14 @@ class RepoConfig:
|
|
|
77
88
|
self.workspace_shared_config = None
|
|
78
89
|
|
|
79
90
|
from mage_ai.data_preparation.shared.utils import get_template_vars
|
|
91
|
+
|
|
80
92
|
try:
|
|
81
93
|
if not config_dict:
|
|
82
94
|
if os.path.exists(self.metadata_path):
|
|
83
95
|
with open(self.metadata_path) as f:
|
|
84
|
-
config_file = Template(f.read()).render(
|
|
85
|
-
|
|
86
|
-
)
|
|
87
|
-
repo_config = yaml.full_load(config_file) or {}
|
|
96
|
+
config_file = Template(f.read()).render(**get_template_vars())
|
|
97
|
+
repo_config = load_yaml(config_file) or {}
|
|
98
|
+
repo_config = trim_strings(repo_config)
|
|
88
99
|
else:
|
|
89
100
|
repo_config = dict()
|
|
90
101
|
else:
|
|
@@ -117,8 +128,9 @@ class RepoConfig:
|
|
|
117
128
|
|
|
118
129
|
# Executor configs
|
|
119
130
|
self.ai_config = repo_config.get('ai_config', dict())
|
|
120
|
-
self.azure_container_instance_config =
|
|
121
|
-
|
|
131
|
+
self.azure_container_instance_config = repo_config.get(
|
|
132
|
+
'azure_container_instance_config'
|
|
133
|
+
)
|
|
122
134
|
self.ecs_config = repo_config.get('ecs_config')
|
|
123
135
|
self.emr_config = repo_config.get('emr_config') or dict()
|
|
124
136
|
self.features = repo_config.get('features', {})
|
|
@@ -132,15 +144,21 @@ class RepoConfig:
|
|
|
132
144
|
self.openai_api_key = repo_config.get('openai_api_key')
|
|
133
145
|
self.pipelines = repo_config.get('pipelines')
|
|
134
146
|
self.retry_config = repo_config.get('retry_config')
|
|
135
|
-
self.workspace_config_defaults = repo_config.get(
|
|
136
|
-
|
|
147
|
+
self.workspace_config_defaults = repo_config.get(
|
|
148
|
+
'workspace_config_defaults'
|
|
149
|
+
)
|
|
150
|
+
self.workspace_initial_metadata = repo_config.get(
|
|
151
|
+
'workspace_initial_metadata'
|
|
152
|
+
)
|
|
137
153
|
|
|
138
154
|
self.ldap_config = repo_config.get('ldap_config')
|
|
139
155
|
|
|
140
156
|
self.s3_bucket = None
|
|
141
157
|
self.s3_path_prefix = None
|
|
142
|
-
if
|
|
143
|
-
|
|
158
|
+
if (
|
|
159
|
+
self.remote_variables_dir is not None
|
|
160
|
+
and self.remote_variables_dir.startswith('s3://')
|
|
161
|
+
):
|
|
144
162
|
path_parts = self.remote_variables_dir.replace('s3://', '').split('/')
|
|
145
163
|
self.s3_bucket = path_parts.pop(0)
|
|
146
164
|
self.s3_path_prefix = '/'.join(path_parts)
|
|
@@ -149,7 +167,9 @@ class RepoConfig:
|
|
|
149
167
|
|
|
150
168
|
self.logging_config = repo_config.get('logging_config', dict())
|
|
151
169
|
|
|
152
|
-
self.variables_retention_period = repo_config.get(
|
|
170
|
+
self.variables_retention_period = repo_config.get(
|
|
171
|
+
'variables_retention_period'
|
|
172
|
+
)
|
|
153
173
|
except Exception as err:
|
|
154
174
|
traceback.print_exc()
|
|
155
175
|
if is_debug():
|
|
@@ -208,7 +228,7 @@ class RepoConfig:
|
|
|
208
228
|
spark_config=self.spark_config,
|
|
209
229
|
variables_dir=self.remote_variables_dir if remote else self.variables_dir,
|
|
210
230
|
variables_retention_period=self.variables_retention_period,
|
|
211
|
-
workspace_config_defaults=self.workspace_config_defaults
|
|
231
|
+
workspace_config_defaults=self.workspace_config_defaults,
|
|
212
232
|
)
|
|
213
233
|
|
|
214
234
|
def save(self, **kwargs) -> None:
|
|
@@ -278,18 +298,31 @@ def init_repo(
|
|
|
278
298
|
get_repo_config(repo_path, root_project=root_project).save(**new_config)
|
|
279
299
|
|
|
280
300
|
|
|
281
|
-
def get_repo_config(
|
|
282
|
-
|
|
301
|
+
def get_repo_config(
|
|
302
|
+
repo_path: str = None,
|
|
303
|
+
context_data: Dict = None,
|
|
304
|
+
root_project: bool = False,
|
|
305
|
+
user=None,
|
|
306
|
+
) -> RepoConfig:
|
|
307
|
+
return RepoConfig(
|
|
308
|
+
repo_path=repo_path,
|
|
309
|
+
context_data=context_data,
|
|
310
|
+
root_project=root_project,
|
|
311
|
+
user=user,
|
|
312
|
+
)
|
|
283
313
|
|
|
284
314
|
|
|
285
|
-
def get_project_type(repo_path=None) -> ProjectType:
|
|
315
|
+
def get_project_type(repo_config: RepoConfig = None, repo_path: str = None) -> ProjectType:
|
|
286
316
|
from mage_ai.settings.repo import MAGE_PROJECT_TYPE_ENV_VAR
|
|
317
|
+
|
|
287
318
|
try:
|
|
288
319
|
project_type_from_env = os.getenv(MAGE_PROJECT_TYPE_ENV_VAR)
|
|
289
320
|
if project_type_from_env:
|
|
290
321
|
return ProjectType(project_type_from_env)
|
|
291
322
|
else:
|
|
292
|
-
|
|
323
|
+
if repo_config is None:
|
|
324
|
+
repo_config = get_repo_config(repo_path=repo_path)
|
|
325
|
+
return repo_config.project_type
|
|
293
326
|
except Exception:
|
|
294
327
|
# default to standalone project type
|
|
295
328
|
return ProjectType.STANDALONE
|
|
@@ -297,6 +330,7 @@ def get_project_type(repo_path=None) -> ProjectType:
|
|
|
297
330
|
|
|
298
331
|
def get_cluster_type(repo_path=None) -> Optional[ClusterType]:
|
|
299
332
|
from mage_ai.settings.repo import MAGE_CLUSTER_TYPE_ENV_VAR
|
|
333
|
+
|
|
300
334
|
try:
|
|
301
335
|
cluster_type_from_env = os.getenv(MAGE_CLUSTER_TYPE_ENV_VAR)
|
|
302
336
|
if cluster_type_from_env:
|
|
@@ -310,7 +344,7 @@ def get_cluster_type(repo_path=None) -> Optional[ClusterType]:
|
|
|
310
344
|
|
|
311
345
|
def set_project_uuid_from_metadata() -> None:
|
|
312
346
|
global project_uuid
|
|
313
|
-
metadata_path =
|
|
347
|
+
metadata_path = os.path.join(base_repo_path(), PROJECT_METADATA_FILENAME)
|
|
314
348
|
if os.path.exists(metadata_path):
|
|
315
349
|
with open(metadata_path, 'r', encoding='utf-8') as f:
|
|
316
350
|
config = yml.load(f) or {}
|
|
@@ -366,7 +400,7 @@ def get_repo_path() -> str:
|
|
|
366
400
|
warn(
|
|
367
401
|
'repo_manager.get_repo_path is deprecated. Please use mage_ai.settings.repo.get_repo_path',
|
|
368
402
|
DeprecationWarning,
|
|
369
|
-
stacklevel=2
|
|
403
|
+
stacklevel=2,
|
|
370
404
|
)
|
|
371
405
|
return get_repo_path_new()
|
|
372
406
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
from contextlib import contextmanager
|
|
3
|
-
from typing import Dict, List
|
|
3
|
+
from typing import Dict, List, Optional, Union
|
|
4
4
|
|
|
5
5
|
import pandas as pd
|
|
6
6
|
import polars as pl
|
|
@@ -67,7 +67,7 @@ class BaseStorage(ABC):
|
|
|
67
67
|
def read_json_file(
|
|
68
68
|
self,
|
|
69
69
|
file_path: str,
|
|
70
|
-
default_value: Dict = None,
|
|
70
|
+
default_value: Optional[Union[Dict, List]] = None,
|
|
71
71
|
raise_exception: bool = False,
|
|
72
72
|
) -> Dict:
|
|
73
73
|
"""
|
|
@@ -55,10 +55,13 @@ class GCSStorage(BaseStorage):
|
|
|
55
55
|
Example block output path in GCS:
|
|
56
56
|
gs://mage_demo/pipelines/example_pipeline/.variables/example_block/output_0/
|
|
57
57
|
"""
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
try:
|
|
59
|
+
blobs = self.bucket.list_blobs(
|
|
60
|
+
prefix=path,
|
|
61
|
+
max_results=max_results,
|
|
62
|
+
)
|
|
63
|
+
except Exception:
|
|
64
|
+
return []
|
|
62
65
|
keys = []
|
|
63
66
|
for blob in blobs:
|
|
64
67
|
# Avoid finding files recursevively in the dir path.
|
|
@@ -2,7 +2,7 @@ import json
|
|
|
2
2
|
import os
|
|
3
3
|
import shutil
|
|
4
4
|
from contextlib import contextmanager
|
|
5
|
-
from typing import Dict, List
|
|
5
|
+
from typing import Dict, List, Optional, Union
|
|
6
6
|
|
|
7
7
|
import aiofiles
|
|
8
8
|
import pandas as pd
|
|
@@ -11,6 +11,7 @@ import simplejson
|
|
|
11
11
|
|
|
12
12
|
from mage_ai.data_preparation.models.file import File
|
|
13
13
|
from mage_ai.data_preparation.storage.base_storage import BaseStorage
|
|
14
|
+
from mage_ai.settings.server import DEBUG_FILE_IO
|
|
14
15
|
from mage_ai.shared.environments import is_debug
|
|
15
16
|
from mage_ai.shared.parsers import encode_complex
|
|
16
17
|
|
|
@@ -26,7 +27,7 @@ class LocalStorage(BaseStorage):
|
|
|
26
27
|
max_results: int = None,
|
|
27
28
|
) -> List[str]:
|
|
28
29
|
paths = []
|
|
29
|
-
if not os.path.exists(path):
|
|
30
|
+
if not os.path.exists(path) or not os.path.isdir(path):
|
|
30
31
|
return paths
|
|
31
32
|
|
|
32
33
|
if max_results is not None:
|
|
@@ -57,9 +58,11 @@ class LocalStorage(BaseStorage):
|
|
|
57
58
|
def read_json_file(
|
|
58
59
|
self,
|
|
59
60
|
file_path: str,
|
|
60
|
-
default_value: Dict = None,
|
|
61
|
+
default_value: Optional[Union[Dict, List]] = None,
|
|
61
62
|
raise_exception: bool = False,
|
|
62
63
|
) -> Dict:
|
|
64
|
+
if DEBUG_FILE_IO and '.variables' in file_path:
|
|
65
|
+
print(f'[READ JSON FILE]: {file_path}')
|
|
63
66
|
if not self.path_exists(file_path):
|
|
64
67
|
return default_value or {}
|
|
65
68
|
with open(file_path) as file:
|
|
@@ -39,8 +39,11 @@ class S3Storage(BaseStorage):
|
|
|
39
39
|
if not path.endswith('/'):
|
|
40
40
|
path += '/'
|
|
41
41
|
path = s3_url_path(path)
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
try:
|
|
43
|
+
keys = self.client.listdir(path, suffix=suffix, max_results=max_results)
|
|
44
|
+
return [k[len(path):].rstrip('/') for k in keys]
|
|
45
|
+
except Exception:
|
|
46
|
+
return []
|
|
44
47
|
|
|
45
48
|
def makedirs(self, path: str, **kwargs) -> None:
|
|
46
49
|
pass
|