mage-ai 0.9.74__py3-none-any.whl → 0.9.79__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mage_ai/ai/llm_pipeline_wizard.py +6 -4
- mage_ai/ai/openai_client.py +7 -5
- mage_ai/api/policies/PipelineSchedulePolicy.py +1 -0
- mage_ai/api/presenters/PipelineSchedulePresenter.py +11 -2
- mage_ai/api/resources/GitFileResource.py +8 -0
- mage_ai/api/resources/PipelineScheduleResource.py +20 -14
- mage_ai/api/resources/PipelineTriggerResource.py +3 -1
- mage_ai/api/resources/SessionResource.py +2 -2
- mage_ai/api/resources/SyncResource.py +1 -1
- mage_ai/api/resources/UserResource.py +1 -1
- mage_ai/cli/main.py +8 -1
- mage_ai/data_cleaner/analysis/charts.py +1 -1
- mage_ai/data_cleaner/cleaning_rules/reformat_values.py +1 -1
- mage_ai/data_integrations/destinations/constants.py +3 -0
- mage_ai/data_integrations/sources/constants.py +2 -0
- mage_ai/data_preparation/executors/block_executor.py +8 -3
- mage_ai/data_preparation/executors/pipeline_executor.py +35 -19
- mage_ai/data_preparation/git/utils.py +2 -2
- mage_ai/data_preparation/logging/logger_manager.py +31 -2
- mage_ai/data_preparation/models/block/__init__.py +33 -27
- mage_ai/data_preparation/models/block/dbt/dbt_adapter.py +20 -8
- mage_ai/data_preparation/models/block/dynamic/constants.py +0 -1
- mage_ai/data_preparation/models/block/dynamic/counter.py +1 -3
- mage_ai/data_preparation/models/block/outputs.py +7 -1
- mage_ai/data_preparation/models/block/r/__init__.py +16 -5
- mage_ai/data_preparation/models/block/sql/__init__.py +2 -0
- mage_ai/data_preparation/models/block/sql/mssql.py +8 -0
- mage_ai/data_preparation/models/block/sql/utils/shared.py +6 -2
- mage_ai/data_preparation/models/constants.py +4 -1
- mage_ai/data_preparation/models/pipeline.py +11 -2
- mage_ai/data_preparation/models/project/__init__.py +3 -1
- mage_ai/data_preparation/models/triggers/__init__.py +1 -1
- mage_ai/data_preparation/storage/local_storage.py +4 -1
- mage_ai/data_preparation/templates/constants.py +7 -0
- mage_ai/data_preparation/templates/data_exporters/streaming/elasticsearch.yaml +3 -0
- mage_ai/data_preparation/templates/data_loaders/airtable.py +28 -0
- mage_ai/data_preparation/templates/data_loaders/streaming/nats.yaml +6 -3
- mage_ai/data_preparation/templates/repo/io_config.yaml +2 -0
- mage_ai/io/airtable.py +104 -0
- mage_ai/io/base.py +30 -1
- mage_ai/io/bigquery.py +36 -0
- mage_ai/io/config.py +6 -0
- mage_ai/io/mssql.py +21 -9
- mage_ai/io/mysql.py +6 -1
- mage_ai/io/oracledb.py +2 -4
- mage_ai/io/postgres.py +41 -19
- mage_ai/io/qdrant.py +1 -1
- mage_ai/io/redshift.py +13 -0
- mage_ai/io/sql.py +1 -0
- mage_ai/io/utils.py +18 -0
- mage_ai/orchestration/db/__init__.py +23 -3
- mage_ai/orchestration/db/migrations/versions/39d36f1dab73_create_genericjob.py +47 -0
- mage_ai/orchestration/db/models/oauth.py +2 -1
- mage_ai/orchestration/db/models/schedules.py +108 -6
- mage_ai/orchestration/db/models/schedules_project_platform.py +1 -1
- mage_ai/orchestration/db/models/secrets.py +11 -1
- mage_ai/orchestration/job_manager.py +19 -0
- mage_ai/orchestration/metrics/pipeline_run.py +1 -1
- mage_ai/orchestration/notification/sender.py +2 -2
- mage_ai/orchestration/pipeline_scheduler_original.py +150 -6
- mage_ai/orchestration/pipeline_scheduler_project_platform.py +4 -5
- mage_ai/orchestration/queue/config.py +11 -1
- mage_ai/orchestration/queue/process_queue.py +4 -0
- mage_ai/orchestration/utils/distributed_lock.py +8 -1
- mage_ai/orchestration/utils/resources.py +56 -2
- mage_ai/sample_datasets/salary_survey.csv +52 -52
- mage_ai/server/api/base.py +41 -0
- mage_ai/server/api/constants.py +1 -0
- mage_ai/server/api/triggers.py +9 -0
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +3 -3
- mage_ai/server/frontend_dist/_next/static/TUo4RceCdMufBTBTq8CAq/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{1187-839336d276186105.js → 1187-4560c3895e1d7099.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{1598-0adca9dce3ba4c60.js → 1598-cbf3f5a6078fc3f5.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2717-638a944d24d5abde.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/3548-36f746b1824004f2.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/{3763-39a5174f6a3924db.js → 3763-aabe2703076636b0.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-3e2acb5ed45b582b.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/449-5e2253c6aba42557.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/5627-d5e559859dd0e1e0.js → frontend_dist/_next/static/chunks/5627-10e76bafa5a26f5f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{5699-e49718dfc9eb2854.js → 5699-e99379e332bd0b41.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/{7966-163da2621b8c987c.js → 7966-a5a7db345ce81263.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-b697b35dfc4e6e26.js +2 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/compute-ed67fa8e81662e8b.js → frontend_dist/_next/static/chunks/pages/compute-9e2dea78024e3bb4.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/files-e0ecd7ced09a63b2.js → frontend_dist/_next/static/chunks/pages/files-e08c7fe76f968f9c.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/global-data-products/{[...slug]-c7a729477ecda50e.js → [...slug]-30c3807057a4e65b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{global-data-products-fd6ae6a358a60a0c.js → global-data-products-8dcb3b31af9e0e39.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/global-hooks/[...slug]-8e50243797a7fe59.js → frontend_dist/_next/static/chunks/pages/global-hooks/[...slug]-85a64b64d27214b6.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{global-hooks-d0c003446332dc0d.js → global-hooks-4ff959d51b8a9502.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{files-a69ed8e9f814490c.js → files-d08a460641d0efaa.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{overview-1aad7093c6d39257.js → overview-aae747f487e08d51.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{pipeline-runs-528d30e0d13b0cc7.js → pipeline-runs-09a842d64a6ada62.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/{settings-fb9201d9cf63031d.js → settings-2e98e57d9376a458.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users/{[user]-000f5a980a07da39.js → [user]-7be6e41ad66089bb.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/manage/users/new-e4e613f6e817a733.js → frontend_dist/_next/static/chunks/pages/manage/users/new-4c088833063bfa07.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users-5db54821a3059c69.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/manage-34d718b8a4066c23.js → frontend_dist/_next/static/chunks/pages/manage-868fcd8cbeb265f0.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{oauth-3bfd1b8d7f036726.js → oauth-6ceceb62191dfe8a.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/overview-f65416f6dbe30ad3.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{pipeline-runs-5f8c100e648efa8a.js → pipeline-runs-2d0136b51b57de93.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-688c652f3296bb9c.js → [...slug]-1ad5238742e25b4c.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-bd11e87d026bfbf9.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{dashboard-1236e36d39b1637d.js → dashboard-0f4f47f721b0723f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-5ae8efe9e0530212.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-fe91dfb0091f6bc6.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-0d68d4bf6290fefb.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-cf794b2d22a80f31.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-9254358d58f07714.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-a964caef91bed9e1.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/{monitors-821001e690caebe2.js → monitors-80bebb4401eefe25.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-2eae7cb017027682.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs-2d4b2a0800a66b33.js → frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-776b2e5b0b6ceba8.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-03d9bca3bc5e6088.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/syncs-707ed8ca942ca802.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers/{[...slug]-259143ed3cf59e31.js → [...slug]-8429f17d4146e1ec.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers-193045d9836d8d80.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-d25d07db166cbb04.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/platform/global-hooks/{[...slug]-5eeec927e4202b63.js → [...slug]-6834ae87bd668cb2.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/platform/global-hooks-fbe9ad995d46d837.js → frontend_dist/_next/static/chunks/pages/platform/global-hooks-b3f7309a23e592b2.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/account/{profile-fc659962d4015cb3.js → profile-f8b7374385e1f1bf.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-8de68502a9afa299.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-a4f88c334414402b.js → frontend_dist/_next/static/chunks/pages/settings/platform/settings-50fb6a34f3913f1f.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-4deb9579ef99a3c6.js → [...slug]-2e5c098c21ea32b7.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{permissions-e0cda2f2bfce8d61.js → permissions-54e4b15b9585bfc4.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-040f83d75d0f6537.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-910257d16c604ebd.js → [...slug]-95088f43034e3c95.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{roles-4f7a0756806cee34.js → roles-e9149e1fcf218f42.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/{sync-data-208d6f955204d704.js → sync-data-75b67ae4a00818ef.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users/{[...slug]-c89dc67e5a1706a8.js → [...slug]-557dda05ca6c6124.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users-fa61dc6c1370e6a5.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{sign-in-054b33312d3193c3.js → sign-in-593c40985d63fcf7.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/templates/{[...slug]-b6ed6a5d818bfd20.js → [...slug]-252c4b6b818345d5.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{templates-852357bc983af2ea.js → templates-ca528bc607753ab8.js} +1 -1
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/terminal-1f9c56d671bbc67d.js → frontend_dist/_next/static/chunks/pages/terminal-287362c1defcc96b.js} +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/test-2f83af8c9f1378fe.js +1 -0
- mage_ai/server/frontend_dist/_next/static/chunks/pages/triggers-d9de73fb799efed8.js +1 -0
- mage_ai/server/{frontend_dist_base_path_template/_next/static/chunks/pages/version-control-ae3469b992a341d6.js → frontend_dist/_next/static/chunks/pages/version-control-573f0225d7a703ed.js} +1 -1
- mage_ai/server/frontend_dist/block-layout.html +3 -3
- mage_ai/server/frontend_dist/compute.html +6 -6
- mage_ai/server/frontend_dist/files.html +6 -6
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-data-products.html +6 -6
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/index.html +3 -3
- mage_ai/server/frontend_dist/manage/files.html +6 -6
- mage_ai/server/frontend_dist/manage/overview.html +6 -6
- mage_ai/server/frontend_dist/manage/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist/manage/settings.html +6 -6
- mage_ai/server/frontend_dist/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist/manage/users.html +6 -6
- mage_ai/server/frontend_dist/manage.html +6 -6
- mage_ai/server/frontend_dist/oauth.html +5 -5
- mage_ai/server/frontend_dist/overview.html +6 -6
- mage_ai/server/frontend_dist/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist/pipelines.html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist/settings.html +3 -3
- mage_ai/server/frontend_dist/sign-in.html +7 -7
- mage_ai/server/frontend_dist/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist/templates.html +6 -6
- mage_ai/server/frontend_dist/terminal.html +6 -6
- mage_ai/server/frontend_dist/test.html +3 -3
- mage_ai/server/frontend_dist/triggers.html +6 -6
- mage_ai/server/frontend_dist/v2/canvas.html +2 -2
- mage_ai/server/frontend_dist/v2.html +2 -2
- mage_ai/server/frontend_dist/version-control.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/404.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/_next/static/2QL-FT4lFR0a9bDZ7lNd9/_buildManifest.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{1187-839336d276186105.js → 1187-4560c3895e1d7099.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{1598-0adca9dce3ba4c60.js → 1598-cbf3f5a6078fc3f5.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2717-638a944d24d5abde.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3548-36f746b1824004f2.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{3763-39a5174f6a3924db.js → 3763-aabe2703076636b0.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-3e2acb5ed45b582b.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/449-5e2253c6aba42557.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/5627-d5e559859dd0e1e0.js → frontend_dist_base_path_template/_next/static/chunks/5627-10e76bafa5a26f5f.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{5699-e49718dfc9eb2854.js → 5699-e99379e332bd0b41.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{7966-163da2621b8c987c.js → 7966-a5a7db345ce81263.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-f205accb03b9ff43.js +2 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/compute-ed67fa8e81662e8b.js → frontend_dist_base_path_template/_next/static/chunks/pages/compute-9e2dea78024e3bb4.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/files-e0ecd7ced09a63b2.js → frontend_dist_base_path_template/_next/static/chunks/pages/files-e08c7fe76f968f9c.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/global-data-products/{[...slug]-c7a729477ecda50e.js → [...slug]-30c3807057a4e65b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{global-data-products-fd6ae6a358a60a0c.js → global-data-products-8dcb3b31af9e0e39.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/global-hooks/[...slug]-8e50243797a7fe59.js → frontend_dist_base_path_template/_next/static/chunks/pages/global-hooks/[...slug]-85a64b64d27214b6.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{global-hooks-d0c003446332dc0d.js → global-hooks-4ff959d51b8a9502.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/{files-a69ed8e9f814490c.js → files-d08a460641d0efaa.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/{overview-1aad7093c6d39257.js → overview-aae747f487e08d51.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/{pipeline-runs-528d30e0d13b0cc7.js → pipeline-runs-09a842d64a6ada62.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/{settings-fb9201d9cf63031d.js → settings-2e98e57d9376a458.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/users/{[user]-000f5a980a07da39.js → [user]-7be6e41ad66089bb.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/manage/users/new-e4e613f6e817a733.js → frontend_dist_base_path_template/_next/static/chunks/pages/manage/users/new-4c088833063bfa07.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/users-5db54821a3059c69.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/manage-34d718b8a4066c23.js → frontend_dist_base_path_template/_next/static/chunks/pages/manage-868fcd8cbeb265f0.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{oauth-3bfd1b8d7f036726.js → oauth-6ceceb62191dfe8a.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/overview-f65416f6dbe30ad3.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{pipeline-runs-5f8c100e648efa8a.js → pipeline-runs-2d0136b51b57de93.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills/{[...slug]-688c652f3296bb9c.js → [...slug]-1ad5238742e25b4c.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-bd11e87d026bfbf9.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{dashboard-1236e36d39b1637d.js → dashboard-0f4f47f721b0723f.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-5ae8efe9e0530212.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-fe91dfb0091f6bc6.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-0d68d4bf6290fefb.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runs-cf794b2d22a80f31.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-9254358d58f07714.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/monitors/block-runtime-a964caef91bed9e1.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/{monitors-821001e690caebe2.js → monitors-80bebb4401eefe25.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-2eae7cb017027682.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs-2d4b2a0800a66b33.js → frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs-776b2e5b0b6ceba8.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-03d9bca3bc5e6088.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/syncs-707ed8ca942ca802.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers/{[...slug]-259143ed3cf59e31.js → [...slug]-8429f17d4146e1ec.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers-193045d9836d8d80.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines-d25d07db166cbb04.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/platform/global-hooks/{[...slug]-5eeec927e4202b63.js → [...slug]-6834ae87bd668cb2.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/platform/global-hooks-fbe9ad995d46d837.js → frontend_dist_base_path_template/_next/static/chunks/pages/platform/global-hooks-b3f7309a23e592b2.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/account/{profile-fc659962d4015cb3.js → profile-f8b7374385e1f1bf.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-8de68502a9afa299.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/settings/platform/settings-a4f88c334414402b.js → frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/settings-50fb6a34f3913f1f.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/permissions/{[...slug]-4deb9579ef99a3c6.js → [...slug]-2e5c098c21ea32b7.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{permissions-e0cda2f2bfce8d61.js → permissions-54e4b15b9585bfc4.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-040f83d75d0f6537.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/roles/{[...slug]-910257d16c604ebd.js → [...slug]-95088f43034e3c95.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{roles-4f7a0756806cee34.js → roles-e9149e1fcf218f42.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/{sync-data-208d6f955204d704.js → sync-data-75b67ae4a00818ef.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users/{[...slug]-c89dc67e5a1706a8.js → [...slug]-557dda05ca6c6124.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users-fa61dc6c1370e6a5.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{sign-in-054b33312d3193c3.js → sign-in-593c40985d63fcf7.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/templates/{[...slug]-b6ed6a5d818bfd20.js → [...slug]-252c4b6b818345d5.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{templates-852357bc983af2ea.js → templates-ca528bc607753ab8.js} +1 -1
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/terminal-1f9c56d671bbc67d.js → frontend_dist_base_path_template/_next/static/chunks/pages/terminal-287362c1defcc96b.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/test-2f83af8c9f1378fe.js +1 -0
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/triggers-d9de73fb799efed8.js +1 -0
- mage_ai/server/{frontend_dist/_next/static/chunks/pages/version-control-ae3469b992a341d6.js → frontend_dist_base_path_template/_next/static/chunks/pages/version-control-573f0225d7a703ed.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-12ad70eb5c31aa92.js → webpack-5f4be622608d9267.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/compute.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/index.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/overview.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/manage.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/oauth.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/overview.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +3 -3
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/settings.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +7 -7
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +6 -6
- mage_ai/server/frontend_dist_base_path_template/templates.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/terminal.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/test.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/triggers.html +6 -6
- mage_ai/server/frontend_dist_base_path_template/v2/canvas.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/v2.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/version-control.html +6 -6
- mage_ai/server/scheduler_manager.py +2 -0
- mage_ai/server/server.py +13 -0
- mage_ai/server/terminal_server.py +3 -0
- mage_ai/server/utils/output_display.py +5 -3
- mage_ai/services/aws/events/events.py +2 -2
- mage_ai/services/gcp/cloud_run/cloud_run.py +1 -1
- mage_ai/services/teams/config.py +13 -2
- mage_ai/services/teams/teams.py +13 -11
- mage_ai/settings/server.py +12 -1
- mage_ai/shared/constants.py +3 -1
- mage_ai/shared/croniter.py +1398 -0
- mage_ai/shared/enum.py +2 -5
- mage_ai/shared/environments.py +27 -3
- mage_ai/streaming/sinks/elasticsearch.py +15 -5
- mage_ai/streaming/sinks/kafka.py +21 -3
- mage_ai/streaming/sources/kafka.py +64 -7
- mage_ai/streaming/sources/kafka_oauth.py +182 -0
- mage_ai/tests/api/endpoints/test_blocks.py +1 -101
- mage_ai/tests/api/endpoints/test_configuration_options.py +1 -48
- mage_ai/tests/api/endpoints/test_configuration_options_project_platform.py +68 -0
- mage_ai/tests/api/endpoints/test_custom_designs.py +1 -106
- mage_ai/tests/api/endpoints/test_custom_designs_project_platform.py +132 -0
- mage_ai/tests/api/endpoints/test_dbt_blocks.py +111 -0
- mage_ai/tests/api/endpoints/test_file_contents.py +0 -48
- mage_ai/tests/api/endpoints/test_file_contents_with_project_platform.py +66 -0
- mage_ai/tests/api/endpoints/test_pipelines.py +0 -134
- mage_ai/tests/api/endpoints/test_pipelines_with_project_platform.py +143 -0
- mage_ai/tests/data_preparation/executors/test_block_executor.py +3 -3
- mage_ai/tests/data_preparation/logging/test_logger_manager.py +24 -5
- mage_ai/tests/data_preparation/models/block/dynamic/test_counter.py +1 -3
- mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +3 -3
- mage_ai/tests/data_preparation/models/test_block.py +7 -0
- mage_ai/tests/data_preparation/models/test_pipeline.py +55 -0
- mage_ai/tests/data_preparation/models/test_variable.py +2 -0
- mage_ai/tests/data_preparation/test_repo_manager.py +0 -63
- mage_ai/tests/data_preparation/test_repo_manager_project_platform.py +67 -0
- mage_ai/tests/data_preparation/test_variable_manager.py +0 -51
- mage_ai/tests/data_preparation/test_variable_manager_project_platform.py +41 -0
- mage_ai/tests/io/create_table/test_postgresql.py +28 -0
- mage_ai/tests/orchestration/db/models/test_schedules.py +1 -1
- mage_ai/tests/orchestration/notification/test_config.py +3 -3
- mage_ai/tests/orchestration/notification/test_sender.py +5 -1
- mage_ai/tests/orchestration/utils/__init__.py +0 -0
- mage_ai/tests/orchestration/utils/test_resources.py +235 -0
- mage_ai/tests/shared/test_croniter.py +2541 -0
- mage_ai/tests/streaming/sinks/test_kafka.py +130 -0
- mage_ai/tests/streaming/sources/test_kafka.py +125 -3
- mage_ai/tests/streaming/sources/test_kafka_oauth.py +208 -0
- mage_ai/tests/streaming/sources/test_kafka_raw_value.py +105 -0
- mage_ai/usage_statistics/logger.py +99 -15
- mage_ai-0.9.79.dist-info/METADATA +358 -0
- {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info}/RECORD +377 -359
- {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info}/WHEEL +1 -1
- mage_ai/server/frontend_dist/_next/static/chunks/1557-1ad0c64c2d08e569.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/2717-d65056b6b5e124eb.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3548-b2c5edfb710886a6.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/3782-4b3091e550f809a2.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/_app-5bdff745074fb350.js +0 -2
- mage_ai/server/frontend_dist/_next/static/chunks/pages/manage/users-a5e9d77ed5b50205.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/overview-69af3253ad0d0d89.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/backfills-fe112809feb25b05.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/edit-e9ca358209cdf93d.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/logs-a29f1615d2e7d330.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-6d382ae5bad9745c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/settings-cd49372ae1702963.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/syncs-135be8974f7f5f2b.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines/[pipeline]/triggers-3af13e89adff4d6c.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/pipelines-b578b075a8d857e3.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/platform/preferences-058d283ee178c038.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/preferences-7b02bb70462144cb.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/settings/workspace/users-5212c01a9dc558da.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/test-86b12cc12d4a625f.js +0 -1
- mage_ai/server/frontend_dist/_next/static/chunks/pages/triggers-2481c40b18d5b6d4.js +0 -1
- mage_ai/server/frontend_dist/_next/static/pLWT6Sqd09xYpufCVIqnz/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/JQewSAObpbhO0wrdAM6Ng/_buildManifest.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/1557-1ad0c64c2d08e569.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/2717-d65056b6b5e124eb.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3548-b2c5edfb710886a6.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/3782-4b3091e550f809a2.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/_app-90de19bc03f1484b.js +0 -2
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/manage/users-a5e9d77ed5b50205.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/overview-69af3253ad0d0d89.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/backfills-fe112809feb25b05.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/edit-e9ca358209cdf93d.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/logs-a29f1615d2e7d330.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/runs/[run]-6d382ae5bad9745c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/settings-cd49372ae1702963.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/syncs-135be8974f7f5f2b.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines/[pipeline]/triggers-3af13e89adff4d6c.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/pipelines-b578b075a8d857e3.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/platform/preferences-058d283ee178c038.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/preferences-7b02bb70462144cb.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/settings/workspace/users-5212c01a9dc558da.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/test-86b12cc12d4a625f.js +0 -1
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/triggers-2481c40b18d5b6d4.js +0 -1
- mage_ai-0.9.74.dist-info/METADATA +0 -544
- /mage_ai/server/frontend_dist/_next/static/{pLWT6Sqd09xYpufCVIqnz → TUo4RceCdMufBTBTq8CAq}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-5bdff745074fb350.js.LICENSE.txt → _app-b697b35dfc4e6e26.js.LICENSE.txt} +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{JQewSAObpbhO0wrdAM6Ng → 2QL-FT4lFR0a9bDZ7lNd9}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{_app-90de19bc03f1484b.js.LICENSE.txt → _app-f205accb03b9ff43.js.LICENSE.txt} +0 -0
- {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info/licenses}/LICENSE +0 -0
- {mage_ai-0.9.74.dist-info → mage_ai-0.9.79.dist-info}/top_level.txt +0 -0
mage_ai/io/config.py
CHANGED
|
@@ -16,6 +16,8 @@ class ConfigKey(StrEnum):
|
|
|
16
16
|
List of configuration settings for use with data IO clients.
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
|
+
AIRTABLE_ACCESS_TOKEN = "AIRTABLE_ACCESS_TOKEN"
|
|
20
|
+
|
|
19
21
|
ALGOLIA_APP_ID = 'ALGOLIA_APP_ID'
|
|
20
22
|
ALGOLIA_API_KEY = 'ALGOLIA_API_KEY'
|
|
21
23
|
ALGOLIA_INDEX_NAME = 'ALGOLIA_INDEX_NAME'
|
|
@@ -64,6 +66,7 @@ class ConfigKey(StrEnum):
|
|
|
64
66
|
MONGODB_PORT = 'MONGODB_PORT'
|
|
65
67
|
MONGODB_USER = 'MONGODB_USER'
|
|
66
68
|
|
|
69
|
+
MSSQL_AUTHENTICATION = 'MSSQL_AUTHENTICATION'
|
|
67
70
|
MSSQL_DATABASE = 'MSSQL_DATABASE'
|
|
68
71
|
MSSQL_DRIVER = 'MSSQL_DRIVER'
|
|
69
72
|
MSSQL_HOST = 'MSSQL_HOST'
|
|
@@ -78,6 +81,7 @@ class ConfigKey(StrEnum):
|
|
|
78
81
|
MYSQL_PASSWORD = 'MYSQL_PASSWORD'
|
|
79
82
|
MYSQL_PORT = 'MYSQL_PORT'
|
|
80
83
|
MYSQL_USER = 'MYSQL_USER'
|
|
84
|
+
MYSQL_ALLOW_LOCAL_INFILE = 'MYSQL_ALLOW_LOCAL_INFILE'
|
|
81
85
|
|
|
82
86
|
ORACLEDB_USER = 'ORACLEDB_USER'
|
|
83
87
|
ORACLEDB_PASSWORD = 'ORACLEDB_PASSWORD'
|
|
@@ -338,6 +342,7 @@ class VerboseConfigKey(StrEnum):
|
|
|
338
342
|
Config key headers for the verbose configuration file format.
|
|
339
343
|
"""
|
|
340
344
|
|
|
345
|
+
AIRTABLE = 'Airtable'
|
|
341
346
|
ALGOLIA = 'Algolia'
|
|
342
347
|
AWS = 'AWS'
|
|
343
348
|
BIGQUERY = 'BigQuery'
|
|
@@ -356,6 +361,7 @@ class VerboseConfigKey(StrEnum):
|
|
|
356
361
|
|
|
357
362
|
class ConfigFileLoader(BaseConfigLoader):
|
|
358
363
|
KEY_MAP = {
|
|
364
|
+
ConfigKey.AIRTABLE_ACCESS_TOKEN: VerboseConfigKey.AIRTABLE,
|
|
359
365
|
ConfigKey.ALGOLIA_APP_ID: (
|
|
360
366
|
VerboseConfigKey.ALGOLIA, 'app_id'),
|
|
361
367
|
ConfigKey.ALGOLIA_API_KEY: (
|
mage_ai/io/mssql.py
CHANGED
|
@@ -14,6 +14,7 @@ from mage_ai.io.config import BaseConfigLoader, ConfigKey
|
|
|
14
14
|
from mage_ai.io.constants import UNIQUE_CONFLICT_METHOD_UPDATE
|
|
15
15
|
from mage_ai.io.export_utils import PandasTypes
|
|
16
16
|
from mage_ai.io.sql import BaseSQL
|
|
17
|
+
from mage_ai.shared.hash import extract
|
|
17
18
|
from mage_ai.shared.parsers import encode_complex
|
|
18
19
|
|
|
19
20
|
MERGE_TABLE_SQL = '''MERGE {table_name} AS t
|
|
@@ -37,12 +38,14 @@ class MSSQL(BaseSQL):
|
|
|
37
38
|
host: str,
|
|
38
39
|
password: str,
|
|
39
40
|
user: str,
|
|
41
|
+
authentication: str = None,
|
|
40
42
|
schema: str = None,
|
|
41
43
|
port: int = 1433,
|
|
42
44
|
verbose: bool = True,
|
|
43
45
|
**kwargs,
|
|
44
46
|
) -> None:
|
|
45
47
|
super().__init__(
|
|
48
|
+
authentication=authentication,
|
|
46
49
|
database=database,
|
|
47
50
|
server=host,
|
|
48
51
|
user=user,
|
|
@@ -60,15 +63,19 @@ class MSSQL(BaseSQL):
|
|
|
60
63
|
database = self.settings['database']
|
|
61
64
|
username = self.settings['user']
|
|
62
65
|
password = self.settings['password']
|
|
63
|
-
|
|
64
|
-
f'DRIVER={{{driver}}}
|
|
65
|
-
f'SERVER={server}
|
|
66
|
-
f'DATABASE={database}
|
|
67
|
-
f'UID={username}
|
|
68
|
-
f'PWD={password}
|
|
69
|
-
'ENCRYPT=yes
|
|
70
|
-
'TrustServerCertificate=yes
|
|
71
|
-
|
|
66
|
+
conn_opts = [
|
|
67
|
+
f'DRIVER={{{driver}}}',
|
|
68
|
+
f'SERVER={server}',
|
|
69
|
+
f'DATABASE={database}',
|
|
70
|
+
f'UID={username}',
|
|
71
|
+
f'PWD={password}',
|
|
72
|
+
'ENCRYPT=yes',
|
|
73
|
+
'TrustServerCertificate=yes',
|
|
74
|
+
]
|
|
75
|
+
if self.settings.get('authentication'):
|
|
76
|
+
authentication = self.settings.get('authentication')
|
|
77
|
+
conn_opts.append(f'Authentication={authentication}')
|
|
78
|
+
return ';'.join(conn_opts)
|
|
72
79
|
|
|
73
80
|
def default_schema(self) -> str:
|
|
74
81
|
return self.settings.get('schema') or 'dbo'
|
|
@@ -197,9 +204,11 @@ class MSSQL(BaseSQL):
|
|
|
197
204
|
TrustServerCertificate='yes',
|
|
198
205
|
),
|
|
199
206
|
)
|
|
207
|
+
conn_kwargs = extract(kwargs, ['pool_size', 'max_overflow'])
|
|
200
208
|
engine = create_engine(
|
|
201
209
|
connection_url,
|
|
202
210
|
fast_executemany=True,
|
|
211
|
+
**conn_kwargs,
|
|
203
212
|
)
|
|
204
213
|
|
|
205
214
|
unique_conflict_method = kwargs.get('unique_conflict_method')
|
|
@@ -242,6 +251,7 @@ class MSSQL(BaseSQL):
|
|
|
242
251
|
method=merge_table,
|
|
243
252
|
)
|
|
244
253
|
return
|
|
254
|
+
sql_kwargs = extract(kwargs, ['chunksize', 'method'])
|
|
245
255
|
|
|
246
256
|
df.to_sql(
|
|
247
257
|
table_name,
|
|
@@ -249,6 +259,7 @@ class MSSQL(BaseSQL):
|
|
|
249
259
|
schema=schema_name,
|
|
250
260
|
if_exists=if_exists or ExportWritePolicy.REPLACE,
|
|
251
261
|
index=False,
|
|
262
|
+
**sql_kwargs,
|
|
252
263
|
)
|
|
253
264
|
|
|
254
265
|
def get_type(self, column: Series, dtype: str) -> str:
|
|
@@ -313,6 +324,7 @@ class MSSQL(BaseSQL):
|
|
|
313
324
|
@classmethod
|
|
314
325
|
def with_config(cls, config: BaseConfigLoader) -> 'MSSQL':
|
|
315
326
|
return cls(
|
|
327
|
+
authentication=config[ConfigKey.MSSQL_AUTHENTICATION],
|
|
316
328
|
database=config[ConfigKey.MSSQL_DATABASE],
|
|
317
329
|
schema=config[ConfigKey.MSSQL_SCHEMA],
|
|
318
330
|
driver=config[ConfigKey.MSSQL_DRIVER],
|
mage_ai/io/mysql.py
CHANGED
|
@@ -25,6 +25,7 @@ class MySQL(BaseSQL):
|
|
|
25
25
|
password: str,
|
|
26
26
|
user: str,
|
|
27
27
|
port: int = 3306,
|
|
28
|
+
allow_local_infile: bool = False,
|
|
28
29
|
verbose: bool = True,
|
|
29
30
|
**kwargs,
|
|
30
31
|
) -> None:
|
|
@@ -35,18 +36,22 @@ class MySQL(BaseSQL):
|
|
|
35
36
|
port=port or 3306,
|
|
36
37
|
user=user,
|
|
37
38
|
verbose=verbose,
|
|
39
|
+
allow_local_infile=allow_local_infile,
|
|
38
40
|
**kwargs,
|
|
39
41
|
)
|
|
40
42
|
|
|
41
43
|
@classmethod
|
|
42
44
|
def with_config(cls, config: BaseConfigLoader) -> 'MySQL':
|
|
43
|
-
|
|
45
|
+
conn_kwargs = dict(
|
|
44
46
|
database=config[ConfigKey.MYSQL_DATABASE],
|
|
45
47
|
host=config[ConfigKey.MYSQL_HOST],
|
|
46
48
|
password=config[ConfigKey.MYSQL_PASSWORD],
|
|
47
49
|
port=config[ConfigKey.MYSQL_PORT],
|
|
48
50
|
user=config[ConfigKey.MYSQL_USER],
|
|
49
51
|
)
|
|
52
|
+
if config[ConfigKey.MYSQL_ALLOW_LOCAL_INFILE] is not None:
|
|
53
|
+
conn_kwargs['allow_local_infile'] = config[ConfigKey.MYSQL_ALLOW_LOCAL_INFILE]
|
|
54
|
+
return cls(**conn_kwargs)
|
|
50
55
|
|
|
51
56
|
def build_create_table_command(
|
|
52
57
|
self,
|
mage_ai/io/oracledb.py
CHANGED
|
@@ -244,10 +244,8 @@ FETCH FIRST {limit} ROWS ONLY
|
|
|
244
244
|
# Remove extraneous surrounding double quotes
|
|
245
245
|
# that get added while performing conversion to string.
|
|
246
246
|
df_[col] = df_[col].apply(lambda x: x.strip('"') if x and isinstance(x, str) else x)
|
|
247
|
-
df_.
|
|
248
|
-
values =
|
|
249
|
-
for i in range(0, len(df_)):
|
|
250
|
-
values.append(tuple(df_.fillna('').values[i]))
|
|
247
|
+
df_.fillna('', inplace=True)
|
|
248
|
+
values = list(df_.itertuples(index=False, name=None))
|
|
251
249
|
|
|
252
250
|
# Create values placeholder
|
|
253
251
|
colmn_names = df.columns.tolist()
|
mage_ai/io/postgres.py
CHANGED
|
@@ -99,6 +99,8 @@ class Postgres(BaseSQL):
|
|
|
99
99
|
password = self.settings['password']
|
|
100
100
|
port = self.settings['port']
|
|
101
101
|
user = self.settings['user']
|
|
102
|
+
keepalives = self.settings.get('keepalives', 1)
|
|
103
|
+
keepalives_idle = self.settings.get('keepalives_idle', 300)
|
|
102
104
|
if self.settings['connection_method'] == 'ssh_tunnel':
|
|
103
105
|
ssh_setting = dict(ssh_username=self.settings['ssh_username'])
|
|
104
106
|
if self.settings['ssh_pkey'] is not None:
|
|
@@ -128,19 +130,35 @@ class Postgres(BaseSQL):
|
|
|
128
130
|
host = '127.0.0.1'
|
|
129
131
|
port = self.ssh_tunnel.local_bind_port
|
|
130
132
|
|
|
131
|
-
connect_opts =
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
133
|
+
connect_opts = self.settings.copy()
|
|
134
|
+
# See recognized keyword parameters for psycopg2.connect()
|
|
135
|
+
# https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS
|
|
136
|
+
unrecognized_keys = [
|
|
137
|
+
'dbname',
|
|
138
|
+
'schema',
|
|
139
|
+
'connection_method',
|
|
140
|
+
'ssh_host',
|
|
141
|
+
'ssh_port',
|
|
142
|
+
'ssh_username',
|
|
143
|
+
'ssh_password',
|
|
144
|
+
'ssh_pkey',
|
|
145
|
+
"verbose"
|
|
146
|
+
]
|
|
147
|
+
for key in unrecognized_keys:
|
|
148
|
+
connect_opts.pop(key, None)
|
|
149
|
+
|
|
150
|
+
connect_opts.update(
|
|
151
|
+
{
|
|
152
|
+
'database': database,
|
|
153
|
+
'host': host,
|
|
154
|
+
'password': password,
|
|
155
|
+
'port': port,
|
|
156
|
+
'user': user,
|
|
157
|
+
'keepalives': keepalives,
|
|
158
|
+
'keepalives_idle': keepalives_idle,
|
|
159
|
+
}
|
|
139
160
|
)
|
|
140
161
|
|
|
141
|
-
if self.settings.get('connect_timeout'):
|
|
142
|
-
connect_opts['connect_timeout'] = self.settings['connect_timeout']
|
|
143
|
-
|
|
144
162
|
try:
|
|
145
163
|
self._ctx = connect(**connect_opts)
|
|
146
164
|
except Exception:
|
|
@@ -277,6 +295,14 @@ class Postgres(BaseSQL):
|
|
|
277
295
|
|
|
278
296
|
return 'text'
|
|
279
297
|
|
|
298
|
+
@staticmethod
|
|
299
|
+
def _clean_array_value(val: str) -> str:
|
|
300
|
+
if val is None or type(val) is not str or len(val) < 2:
|
|
301
|
+
return val
|
|
302
|
+
if val[0] == '[' and val[-1] == ']':
|
|
303
|
+
return str(val).replace('[', '{').replace(']', '}')
|
|
304
|
+
return val
|
|
305
|
+
|
|
280
306
|
def upload_dataframe(
|
|
281
307
|
self,
|
|
282
308
|
cursor: _psycopg.cursor,
|
|
@@ -298,30 +324,26 @@ class Postgres(BaseSQL):
|
|
|
298
324
|
# Use COPY command
|
|
299
325
|
use_insert_command = False
|
|
300
326
|
|
|
301
|
-
def clean_array_value(val):
|
|
302
|
-
if val is None or type(val) is not str or len(val) < 2:
|
|
303
|
-
return val
|
|
304
|
-
if val[0] == '[' and val[-1] == ']':
|
|
305
|
-
return '{' + val[1:-1] + '}'
|
|
306
|
-
return val
|
|
307
|
-
|
|
308
327
|
def serialize_obj(val):
|
|
309
328
|
if type(val) is dict or type(val) is np.ndarray:
|
|
310
329
|
return simplejson.dumps(
|
|
311
330
|
val,
|
|
312
331
|
default=encode_complex,
|
|
332
|
+
ensure_ascii=False,
|
|
313
333
|
ignore_nan=True,
|
|
314
334
|
)
|
|
315
335
|
elif type(val) is list and len(val) >= 1 and type(val[0]) is dict:
|
|
316
336
|
return simplejson.dumps(
|
|
317
337
|
val,
|
|
318
338
|
default=encode_complex,
|
|
339
|
+
ensure_ascii=False,
|
|
319
340
|
ignore_nan=True,
|
|
320
341
|
)
|
|
321
342
|
elif not use_insert_command and type(val) is list:
|
|
322
|
-
return
|
|
343
|
+
return self._clean_array_value(simplejson.dumps(
|
|
323
344
|
val,
|
|
324
345
|
default=encode_complex,
|
|
346
|
+
ensure_ascii=False,
|
|
325
347
|
ignore_nan=True,
|
|
326
348
|
))
|
|
327
349
|
return val
|
mage_ai/io/qdrant.py
CHANGED
|
@@ -107,7 +107,7 @@ class Qdrant(BaseIO):
|
|
|
107
107
|
Save data into Qdrant.
|
|
108
108
|
Args:
|
|
109
109
|
df (DataFrame): Data to export.
|
|
110
|
-
document_column (str): Column name
|
|
110
|
+
document_column (str): Column name containing documents to export.
|
|
111
111
|
id_column (str): Column name of the id. Default will use index in df.
|
|
112
112
|
vector_column (str): Column name of the vector. Will use default
|
|
113
113
|
encoder to auto generate query vector to auto generate query vector.
|
mage_ai/io/redshift.py
CHANGED
|
@@ -5,6 +5,7 @@ from typing import Dict, Union
|
|
|
5
5
|
from pandas import DataFrame
|
|
6
6
|
from redshift_connector import connect
|
|
7
7
|
|
|
8
|
+
from mage_ai.data_preparation.models.block.sql.utils.shared import split_query_string
|
|
8
9
|
from mage_ai.io.base import QUERY_ROW_LIMIT, ExportWritePolicy
|
|
9
10
|
from mage_ai.io.config import BaseConfigLoader, ConfigKey
|
|
10
11
|
from mage_ai.io.export_utils import clean_df_for_export, infer_dtypes
|
|
@@ -67,6 +68,18 @@ class Redshift(BaseSQL):
|
|
|
67
68
|
with self.conn.cursor() as cur:
|
|
68
69
|
cur.execute(query_string, **kwargs)
|
|
69
70
|
|
|
71
|
+
def execute_query_raw(self, query: str, **kwargs) -> None:
|
|
72
|
+
"""
|
|
73
|
+
Overwrite execute query to process multiple queries in one string.
|
|
74
|
+
"""
|
|
75
|
+
results = []
|
|
76
|
+
with self.conn.cursor() as cursor:
|
|
77
|
+
for query_string in split_query_string(query):
|
|
78
|
+
result = cursor.execute(query_string)
|
|
79
|
+
results.append(result)
|
|
80
|
+
self.conn.commit()
|
|
81
|
+
return results
|
|
82
|
+
|
|
70
83
|
def load(
|
|
71
84
|
self,
|
|
72
85
|
query_string: str,
|
mage_ai/io/sql.py
CHANGED
mage_ai/io/utils.py
CHANGED
|
@@ -29,3 +29,21 @@ def escape_quotes(line: str, single: bool = True, double: bool = True) -> str:
|
|
|
29
29
|
if double:
|
|
30
30
|
new_line = new_line.replace('\"', '\\"')
|
|
31
31
|
return new_line
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def map_json_to_airtable(data_types):
|
|
35
|
+
# Extract the non-null type (ignoring 'null')
|
|
36
|
+
data_type = next((t for t in data_types if t != 'null'), 'string')
|
|
37
|
+
|
|
38
|
+
# Mapping from JSON types to Airtable types
|
|
39
|
+
type_mapping = {
|
|
40
|
+
'string': 'multilineText',
|
|
41
|
+
'integer': 'number',
|
|
42
|
+
'boolean': 'checkbox',
|
|
43
|
+
'array': 'multipleSelects',
|
|
44
|
+
'object': 'singleCollaborator',
|
|
45
|
+
'number': 'number',
|
|
46
|
+
'date-time': 'dateTime'
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return type_mapping.get(data_type, 'str')
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
|
+
from functools import wraps
|
|
3
4
|
from urllib.parse import parse_qs, quote_plus, urlparse
|
|
4
5
|
|
|
5
6
|
import sqlalchemy
|
|
@@ -13,7 +14,7 @@ from mage_ai.orchestration.db.setup import get_postgres_connection_url
|
|
|
13
14
|
from mage_ai.orchestration.db.utils import get_user_info_from_db_connection_url
|
|
14
15
|
from mage_ai.settings import OTEL_EXPORTER_OTLP_ENDPOINT
|
|
15
16
|
from mage_ai.settings.repo import get_variables_dir
|
|
16
|
-
from mage_ai.shared.environments import is_debug,
|
|
17
|
+
from mage_ai.shared.environments import is_debug, is_test_mage
|
|
17
18
|
|
|
18
19
|
DB_RETRY_COUNT = 2
|
|
19
20
|
TEST_DB = 'test.db'
|
|
@@ -28,7 +29,7 @@ db_kwargs = dict(
|
|
|
28
29
|
if OTEL_EXPORTER_OTLP_ENDPOINT:
|
|
29
30
|
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
|
|
30
31
|
|
|
31
|
-
if
|
|
32
|
+
if is_test_mage():
|
|
32
33
|
db_connection_url = f'sqlite:///{TEST_DB}'
|
|
33
34
|
db_kwargs['connect_args']['check_same_thread'] = False
|
|
34
35
|
elif not db_connection_url:
|
|
@@ -37,7 +38,7 @@ elif not db_connection_url:
|
|
|
37
38
|
if pg_db_connection_url:
|
|
38
39
|
db_connection_url = pg_db_connection_url
|
|
39
40
|
else:
|
|
40
|
-
if
|
|
41
|
+
if is_test_mage():
|
|
41
42
|
db_connection_url = f'sqlite:///{TEST_DB}'
|
|
42
43
|
elif os.path.exists(os.path.join('mage_ai', 'orchestration', 'db')):
|
|
43
44
|
# For local dev environment
|
|
@@ -169,6 +170,25 @@ def safe_db_query(func):
|
|
|
169
170
|
return func_with_rollback
|
|
170
171
|
|
|
171
172
|
|
|
173
|
+
def safe_db_query_async(func):
|
|
174
|
+
@wraps(func)
|
|
175
|
+
async def func_with_rollback_async(*args, **kwargs):
|
|
176
|
+
retry_count = 0
|
|
177
|
+
while True:
|
|
178
|
+
try:
|
|
179
|
+
return await func(*args, **kwargs)
|
|
180
|
+
except (
|
|
181
|
+
sqlalchemy.exc.OperationalError,
|
|
182
|
+
sqlalchemy.exc.PendingRollbackError,
|
|
183
|
+
sqlalchemy.exc.InternalError,
|
|
184
|
+
) as e:
|
|
185
|
+
db_connection.session.rollback()
|
|
186
|
+
if retry_count >= DB_RETRY_COUNT:
|
|
187
|
+
raise e
|
|
188
|
+
retry_count += 1
|
|
189
|
+
return func_with_rollback_async
|
|
190
|
+
|
|
191
|
+
|
|
172
192
|
logging.basicConfig()
|
|
173
193
|
|
|
174
194
|
if is_debug() and not os.getenv('DISABLE_DATABASE_TERMINAL_OUTPUT'):
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Create GenericJob
|
|
2
|
+
|
|
3
|
+
Revision ID: 39d36f1dab73
|
|
4
|
+
Revises: 0227396a216c
|
|
5
|
+
Create Date: 2025-08-27 20:45:29.756869
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
from alembic import op
|
|
9
|
+
import sqlalchemy as sa
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# revision identifiers, used by Alembic.
|
|
13
|
+
revision = '39d36f1dab73'
|
|
14
|
+
down_revision = '0227396a216c'
|
|
15
|
+
branch_labels = None
|
|
16
|
+
depends_on = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def upgrade() -> None:
|
|
20
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
21
|
+
op.create_table('generic_job',
|
|
22
|
+
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
|
|
23
|
+
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
|
|
24
|
+
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
|
|
25
|
+
sa.Column('job_type', sa.Enum('CANCEL_PIPELINE_RUN', name='jobtype'), nullable=False),
|
|
26
|
+
sa.Column('status', sa.Enum('INITIAL', 'QUEUED', 'RUNNING', 'COMPLETED', 'FAILED', name='jobstatus'), nullable=True),
|
|
27
|
+
sa.Column('payload', sa.JSON(), nullable=True),
|
|
28
|
+
sa.Column('extra_metadata', sa.JSON(), nullable=True),
|
|
29
|
+
sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
|
|
30
|
+
sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True),
|
|
31
|
+
sa.Column('repo_path', sa.String(length=255), nullable=True),
|
|
32
|
+
sa.PrimaryKeyConstraint('id')
|
|
33
|
+
)
|
|
34
|
+
with op.batch_alter_table('generic_job', schema=None) as batch_op:
|
|
35
|
+
batch_op.create_index(batch_op.f('ix_generic_job_job_type'), ['job_type'], unique=False)
|
|
36
|
+
batch_op.create_index(batch_op.f('ix_generic_job_status'), ['status'], unique=False)
|
|
37
|
+
# ### end Alembic commands ###
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def downgrade() -> None:
|
|
41
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
42
|
+
with op.batch_alter_table('generic_job', schema=None) as batch_op:
|
|
43
|
+
batch_op.drop_index(batch_op.f('ix_generic_job_status'))
|
|
44
|
+
batch_op.drop_index(batch_op.f('ix_generic_job_job_type'))
|
|
45
|
+
|
|
46
|
+
op.drop_table('generic_job')
|
|
47
|
+
# ### end Alembic commands ###
|
|
@@ -882,4 +882,5 @@ class Oauth2AccessToken(BaseModel):
|
|
|
882
882
|
def is_valid(self) -> bool:
|
|
883
883
|
return self.token and \
|
|
884
884
|
self.expires and \
|
|
885
|
-
self.expires >= datetime.utcnow().replace(tzinfo=self.expires.tzinfo)
|
|
885
|
+
self.expires >= datetime.utcnow().replace(tzinfo=self.expires.tzinfo) and \
|
|
886
|
+
self.user is not None
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import asyncio
|
|
2
1
|
import collections
|
|
3
2
|
import traceback
|
|
4
3
|
import uuid
|
|
@@ -9,7 +8,6 @@ from typing import DefaultDict, Dict, List
|
|
|
9
8
|
|
|
10
9
|
import dateutil.parser
|
|
11
10
|
import pytz
|
|
12
|
-
from croniter import croniter
|
|
13
11
|
from dateutil.relativedelta import relativedelta
|
|
14
12
|
from sqlalchemy import (
|
|
15
13
|
JSON,
|
|
@@ -71,6 +69,7 @@ from mage_ai.server.kernel_output_parser import DataType
|
|
|
71
69
|
from mage_ai.settings.platform import project_platform_activated
|
|
72
70
|
from mage_ai.settings.repo import get_repo_path
|
|
73
71
|
from mage_ai.shared.constants import ENV_PROD
|
|
72
|
+
from mage_ai.shared.croniter import croniter
|
|
74
73
|
from mage_ai.shared.dates import compare
|
|
75
74
|
from mage_ai.shared.enum import StrEnum
|
|
76
75
|
from mage_ai.shared.hash import ignore_keys, index_by, merge_dict
|
|
@@ -1400,10 +1399,8 @@ class PipelineRun(PipelineRunProjectPlatformMixin, BaseModel):
|
|
|
1400
1399
|
|
|
1401
1400
|
from mage_ai.usage_statistics.logger import UsageStatisticLogger
|
|
1402
1401
|
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
lambda: self.query.filter(self.status == self.PipelineRunStatus.COMPLETED).count(),
|
|
1406
|
-
)
|
|
1402
|
+
UsageStatisticLogger().pipeline_runs_impression_sync(
|
|
1403
|
+
lambda: self.query.filter(self.status == self.PipelineRunStatus.COMPLETED).count(),
|
|
1407
1404
|
)
|
|
1408
1405
|
|
|
1409
1406
|
@safe_db_query
|
|
@@ -1807,6 +1804,111 @@ class BlockRun(BlockRunProjectPlatformMixin, BaseModel):
|
|
|
1807
1804
|
)
|
|
1808
1805
|
|
|
1809
1806
|
|
|
1807
|
+
class GenericJob(BaseModel):
|
|
1808
|
+
class JobStatus(StrEnum):
|
|
1809
|
+
INITIAL = 'initial'
|
|
1810
|
+
QUEUED = 'queued'
|
|
1811
|
+
RUNNING = 'running'
|
|
1812
|
+
COMPLETED = 'completed'
|
|
1813
|
+
FAILED = 'failed'
|
|
1814
|
+
|
|
1815
|
+
class JobType(StrEnum):
|
|
1816
|
+
CANCEL_PIPELINE_RUN = 'cancel_pipeline_run'
|
|
1817
|
+
|
|
1818
|
+
job_type = Column(Enum(JobType), nullable=False, index=True)
|
|
1819
|
+
status = Column(Enum(JobStatus), default=JobStatus.INITIAL, index=True)
|
|
1820
|
+
payload = Column(JSON) # Job-specific data (e.g., pipeline_run_id)
|
|
1821
|
+
extra_metadata = Column(JSON) # Additional metadata for the job
|
|
1822
|
+
started_at = Column(DateTime(timezone=True))
|
|
1823
|
+
completed_at = Column(DateTime(timezone=True))
|
|
1824
|
+
repo_path = Column(String(255))
|
|
1825
|
+
|
|
1826
|
+
@classproperty
|
|
1827
|
+
def repo_query(cls):
|
|
1828
|
+
return cls.query.filter(
|
|
1829
|
+
or_(
|
|
1830
|
+
GenericJob.repo_path == get_repo_path(),
|
|
1831
|
+
GenericJob.repo_path.is_(None),
|
|
1832
|
+
)
|
|
1833
|
+
)
|
|
1834
|
+
|
|
1835
|
+
@classmethod
|
|
1836
|
+
@safe_db_query
|
|
1837
|
+
def enqueue_cancel_pipeline_run(
|
|
1838
|
+
cls,
|
|
1839
|
+
pipeline_run_id: int,
|
|
1840
|
+
cancelled_block_run_ids: List,
|
|
1841
|
+
repo_path: str = None,
|
|
1842
|
+
) -> 'GenericJob':
|
|
1843
|
+
"""Enqueue a job to cancel a pipeline run."""
|
|
1844
|
+
if repo_path is None:
|
|
1845
|
+
repo_path = get_repo_path()
|
|
1846
|
+
|
|
1847
|
+
job = cls.create(
|
|
1848
|
+
job_type=cls.JobType.CANCEL_PIPELINE_RUN,
|
|
1849
|
+
payload=dict(
|
|
1850
|
+
pipeline_run_id=pipeline_run_id,
|
|
1851
|
+
cancelled_block_run_ids=cancelled_block_run_ids,
|
|
1852
|
+
),
|
|
1853
|
+
repo_path=repo_path,
|
|
1854
|
+
status=cls.JobStatus.INITIAL
|
|
1855
|
+
)
|
|
1856
|
+
return job
|
|
1857
|
+
|
|
1858
|
+
@classmethod
|
|
1859
|
+
@safe_db_query
|
|
1860
|
+
def get_jobs_with_initial_status(
|
|
1861
|
+
cls,
|
|
1862
|
+
limit: int = 10,
|
|
1863
|
+
repo_path: str = None,
|
|
1864
|
+
) -> List['GenericJob']:
|
|
1865
|
+
"""Get up to N jobs with INITIAL status for processing."""
|
|
1866
|
+
query = cls.repo_query.filter(
|
|
1867
|
+
cls.status == cls.JobStatus.INITIAL,
|
|
1868
|
+
)
|
|
1869
|
+
|
|
1870
|
+
if repo_path:
|
|
1871
|
+
query = query.filter(cls.repo_path == repo_path)
|
|
1872
|
+
|
|
1873
|
+
jobs = query.order_by(cls.created_at.asc()).limit(limit).all()
|
|
1874
|
+
|
|
1875
|
+
return jobs
|
|
1876
|
+
|
|
1877
|
+
@safe_db_query
|
|
1878
|
+
def mark_completed(self):
|
|
1879
|
+
"""Mark the job as completed successfully."""
|
|
1880
|
+
self.update(
|
|
1881
|
+
status=self.JobStatus.COMPLETED,
|
|
1882
|
+
completed_at=datetime.now(tz=pytz.UTC)
|
|
1883
|
+
)
|
|
1884
|
+
|
|
1885
|
+
@safe_db_query
|
|
1886
|
+
def mark_failed(self, metadata: dict = None):
|
|
1887
|
+
"""Mark the job as failed."""
|
|
1888
|
+
self.update(
|
|
1889
|
+
status=self.JobStatus.FAILED,
|
|
1890
|
+
completed_at=datetime.now(tz=pytz.UTC),
|
|
1891
|
+
extra_metadata=metadata or {}
|
|
1892
|
+
)
|
|
1893
|
+
|
|
1894
|
+
@safe_db_query
|
|
1895
|
+
def mark_queued(self):
|
|
1896
|
+
"""Mark the job as queued."""
|
|
1897
|
+
self.update(
|
|
1898
|
+
status=self.JobStatus.QUEUED,
|
|
1899
|
+
)
|
|
1900
|
+
|
|
1901
|
+
@safe_db_query
|
|
1902
|
+
def mark_running(self):
|
|
1903
|
+
"""Mark the job as queued."""
|
|
1904
|
+
self.update(
|
|
1905
|
+
status=self.JobStatus.RUNNING,
|
|
1906
|
+
)
|
|
1907
|
+
|
|
1908
|
+
def __repr__(self):
|
|
1909
|
+
return f'GenericJob(id={self.id}, type={self.job_type}, status={self.status})'
|
|
1910
|
+
|
|
1911
|
+
|
|
1810
1912
|
class EventMatcher(BaseModel):
|
|
1811
1913
|
class EventType(StrEnum):
|
|
1812
1914
|
AWS_EVENT = 'aws_event'
|
|
@@ -5,7 +5,6 @@ from typing import Dict, List
|
|
|
5
5
|
|
|
6
6
|
import dateutil.parser
|
|
7
7
|
import pytz
|
|
8
|
-
from croniter import croniter
|
|
9
8
|
from dateutil.relativedelta import relativedelta
|
|
10
9
|
from sqlalchemy import or_
|
|
11
10
|
from sqlalchemy.sql import func
|
|
@@ -28,6 +27,7 @@ from mage_ai.settings.platform.utils import (
|
|
|
28
27
|
get_pipeline_from_platform_async,
|
|
29
28
|
)
|
|
30
29
|
from mage_ai.shared.constants import ENV_PROD
|
|
30
|
+
from mage_ai.shared.croniter import croniter
|
|
31
31
|
from mage_ai.shared.dates import compare
|
|
32
32
|
from mage_ai.shared.hash import merge_dict
|
|
33
33
|
from mage_ai.shared.utils import clean_name
|
|
@@ -3,7 +3,8 @@ from typing import Optional
|
|
|
3
3
|
from sqlalchemy import Column, String, Text, UniqueConstraint, or_
|
|
4
4
|
|
|
5
5
|
from mage_ai.orchestration.db import safe_db_query
|
|
6
|
-
from mage_ai.orchestration.db.models.base import BaseModel
|
|
6
|
+
from mage_ai.orchestration.db.models.base import BaseModel, classproperty
|
|
7
|
+
from mage_ai.settings.repo import get_repo_path
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
class Secret(BaseModel):
|
|
@@ -15,6 +16,15 @@ class Secret(BaseModel):
|
|
|
15
16
|
key_uuid = Column(String(255), nullable=True)
|
|
16
17
|
__table_args__ = (UniqueConstraint('name', 'key_uuid', name='name_key_uuid_uc'),)
|
|
17
18
|
|
|
19
|
+
@classproperty
|
|
20
|
+
def repo_query(cls):
|
|
21
|
+
return cls.query.filter(
|
|
22
|
+
or_(
|
|
23
|
+
Secret.repo_name == get_repo_path(),
|
|
24
|
+
Secret.repo_name.is_(None),
|
|
25
|
+
)
|
|
26
|
+
)
|
|
27
|
+
|
|
18
28
|
@classmethod
|
|
19
29
|
@safe_db_query
|
|
20
30
|
def get_secret(cls, name: str, key_uuid: str) -> Optional['Secret']:
|