mage-ai 0.9.73__py3-none-any.whl → 0.9.74__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mage_ai/ai/constants.py +2 -2
- mage_ai/ai/llm_pipeline_wizard.py +5 -3
- mage_ai/ai/openai_client.py +11 -0
- mage_ai/api/constants.py +4 -5
- mage_ai/api/oauth_scope.py +2 -2
- mage_ai/api/operations/constants.py +2 -2
- mage_ai/api/presenters/SyncPresenter.py +4 -1
- mage_ai/authentication/oauth/constants.py +2 -2
- mage_ai/authentication/operation_history/constants.py +2 -2
- mage_ai/authentication/permissions/constants.py +5 -6
- mage_ai/cache/constants.py +2 -2
- mage_ai/cache/dbt/constants.py +2 -3
- mage_ai/cluster_manager/constants.py +2 -2
- mage_ai/cluster_manager/kubernetes/workload_manager.py +3 -2
- mage_ai/command_center/constants.py +13 -13
- mage_ai/data_cleaner/column_types/constants.py +2 -2
- mage_ai/data_cleaner/transformer_actions/column.py +19 -7
- mage_ai/data_cleaner/transformer_actions/constants.py +10 -9
- mage_ai/data_integrations/sources/constants.py +2 -0
- mage_ai/data_preparation/git/utils.py +4 -1
- mage_ai/data_preparation/logging/__init__.py +2 -2
- mage_ai/data_preparation/logging/logger.py +3 -3
- mage_ai/data_preparation/models/block/__init__.py +9 -3
- mage_ai/data_preparation/models/block/data_integration/constants.py +2 -2
- mage_ai/data_preparation/models/block/dbt/constants.py +3 -3
- mage_ai/data_preparation/models/block/dynamic/factory.py +40 -3
- mage_ai/data_preparation/models/block/dynamic/utils.py +31 -33
- mage_ai/data_preparation/models/block/dynamic/variables.py +1 -1
- mage_ai/data_preparation/models/block/settings/dynamic/constants.py +3 -3
- mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +63 -4
- mage_ai/data_preparation/models/block/settings/global_data_products/models.py +2 -2
- mage_ai/data_preparation/models/constants.py +10 -10
- mage_ai/data_preparation/models/global_hooks/constants.py +7 -8
- mage_ai/data_preparation/models/global_hooks/models.py +5 -5
- mage_ai/data_preparation/models/project/constants.py +2 -2
- mage_ai/data_preparation/models/triggers/__init__.py +6 -4
- mage_ai/data_preparation/models/variables/constants.py +5 -5
- mage_ai/data_preparation/models/widget/constants.py +5 -5
- mage_ai/data_preparation/preferences.py +9 -16
- mage_ai/data_preparation/repo_manager.py +2 -2
- mage_ai/data_preparation/sync/__init__.py +2 -2
- mage_ai/data_preparation/templates/data_exporters/streaming/generic_python.py +1 -1
- mage_ai/data_preparation/templates/data_loaders/streaming/nats.yaml +3 -0
- mage_ai/errors/constants.py +2 -2
- mage_ai/io/base.py +28 -15
- mage_ai/io/config.py +3 -3
- mage_ai/io/export_utils.py +2 -2
- mage_ai/io/google_cloud_storage.py +3 -2
- mage_ai/io/io_config.py +3 -2
- mage_ai/io/postgres.py +2 -1
- mage_ai/kernels/magic/constants.py +4 -4
- mage_ai/orchestration/constants.py +2 -2
- mage_ai/orchestration/db/constants.py +2 -2
- mage_ai/orchestration/db/models/oauth.py +5 -5
- mage_ai/orchestration/db/models/schedules.py +28 -8
- mage_ai/orchestration/db/models/schedules_project_platform.py +10 -8
- mage_ai/orchestration/job_manager.py +2 -2
- mage_ai/orchestration/monitor/monitor_stats.py +2 -2
- mage_ai/orchestration/notification/config.py +2 -2
- mage_ai/orchestration/pipeline_scheduler_original.py +4 -12
- mage_ai/orchestration/queue/config.py +2 -2
- mage_ai/orchestration/queue/process_queue.py +3 -3
- mage_ai/presenters/charts/data_sources/constants.py +2 -2
- mage_ai/presenters/interactions/constants.py +4 -4
- mage_ai/presenters/pages/models/constants.py +4 -4
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +2 -2
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-663c909dcda4c23a.js → _app-5bdff745074fb350.js} +2 -2
- mage_ai/server/frontend_dist/_next/static/chunks/{webpack-43534cc51fce8644.js → webpack-b9a067f3bd0a3a05.js} +1 -1
- mage_ai/server/frontend_dist/block-layout.html +2 -2
- mage_ai/server/frontend_dist/compute.html +2 -2
- mage_ai/server/frontend_dist/files.html +2 -2
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +2 -2
- mage_ai/server/frontend_dist/global-data-products.html +2 -2
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist/global-hooks.html +2 -2
- mage_ai/server/frontend_dist/index.html +2 -2
- mage_ai/server/frontend_dist/manage/files.html +2 -2
- mage_ai/server/frontend_dist/manage/overview.html +2 -2
- mage_ai/server/frontend_dist/manage/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist/manage/settings.html +2 -2
- mage_ai/server/frontend_dist/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist/manage/users.html +2 -2
- mage_ai/server/frontend_dist/manage.html +2 -2
- mage_ai/server/frontend_dist/oauth.html +3 -3
- mage_ai/server/frontend_dist/overview.html +2 -2
- mage_ai/server/frontend_dist/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist/pipelines.html +2 -2
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist/platform/global-hooks.html +2 -2
- mage_ai/server/frontend_dist/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist/settings/platform/preferences.html +2 -2
- mage_ai/server/frontend_dist/settings/platform/settings.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/roles.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist/settings.html +2 -2
- mage_ai/server/frontend_dist/sign-in.html +5 -5
- mage_ai/server/frontend_dist/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist/templates.html +2 -2
- mage_ai/server/frontend_dist/terminal.html +2 -2
- mage_ai/server/frontend_dist/test.html +2 -2
- mage_ai/server/frontend_dist/triggers.html +2 -2
- mage_ai/server/frontend_dist/v2/canvas.html +2 -2
- mage_ai/server/frontend_dist/v2.html +2 -2
- mage_ai/server/frontend_dist/version-control.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/404.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{_app-bb4a0e0d783622a8.js → _app-90de19bc03f1484b.js} +2 -2
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-9eb1dd2ee735aaac.js → webpack-12ad70eb5c31aa92.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/compute.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/files.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/index.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/overview.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/oauth.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/overview.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/templates.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/terminal.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/test.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/triggers.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/v2/canvas.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/v2.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/version-control.html +2 -2
- mage_ai/server/kernel_output_parser.py +2 -3
- mage_ai/server/kernels.py +2 -3
- mage_ai/server/scheduler_manager.py +2 -2
- mage_ai/server/server.py +12 -0
- mage_ai/server/websockets/constants.py +4 -4
- mage_ai/services/compute/aws/constants.py +2 -2
- mage_ai/services/compute/aws/steps.py +16 -10
- mage_ai/services/compute/constants.py +4 -4
- mage_ai/services/compute/models.py +4 -4
- mage_ai/services/k8s/config.py +4 -0
- mage_ai/services/k8s/job_manager.py +2 -0
- mage_ai/services/spark/constants.py +3 -3
- mage_ai/services/spark/models/jobs.py +2 -2
- mage_ai/services/spark/models/sqls.py +2 -2
- mage_ai/services/spark/models/stages.py +4 -4
- mage_ai/services/spark/models/threads.py +2 -2
- mage_ai/settings/backends.py +3 -2
- mage_ai/settings/models/configuration_option.py +3 -3
- mage_ai/settings/server.py +5 -0
- mage_ai/shared/constants.py +3 -3
- mage_ai/shared/custom_logger.py +13 -13
- mage_ai/shared/enum.py +11 -0
- mage_ai/shared/environments.py +1 -1
- mage_ai/shared/logger.py +2 -2
- mage_ai/shared/models.py +2 -1
- mage_ai/streaming/constants.py +3 -3
- mage_ai/streaming/sinks/kafka.py +2 -2
- mage_ai/streaming/sinks/postgres.py +6 -0
- mage_ai/streaming/sources/amazon_sqs.py +2 -2
- mage_ai/streaming/sources/base.py +2 -2
- mage_ai/streaming/sources/kafka.py +20 -4
- mage_ai/streaming/sources/nats_js.py +10 -3
- mage_ai/streaming/sources/shared.py +3 -2
- mage_ai/system/constants.py +2 -2
- mage_ai/tests/api/policies/permissions/test_base_policy_with_permissions.py +2 -2
- mage_ai/tests/data_cleaner/transformer_actions/test_trim_transformer.py +161 -0
- mage_ai/tests/data_preparation/models/block/dbt/test_profiles.py +1 -1
- mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +1 -1
- mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +3 -2
- mage_ai/tests/data_preparation/models/test_blocks_helper.py +1 -1
- mage_ai/tests/data_preparation/models/variables/test_summarizer.py +1 -1
- mage_ai/tests/data_preparation/sync/test_git_sync.py +6 -6
- mage_ai/tests/orchestration/queue/test_process_queue.py +3 -2
- mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
- mage_ai/tests/settings/test_platform.py +2 -2
- mage_ai/tests/streaming/sinks/test_generic_io.py +25 -21
- mage_ai/usage_statistics/constants.py +4 -4
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/METADATA +172 -171
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/RECORD +247 -245
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/WHEEL +1 -1
- /mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-663c909dcda4c23a.js.LICENSE.txt → _app-5bdff745074fb350.js.LICENSE.txt} +0 -0
- /mage_ai/server/frontend_dist/_next/static/{kxGpiudO3f9aX6FAiqydf → pLWT6Sqd09xYpufCVIqnz}/_buildManifest.js +0 -0
- /mage_ai/server/frontend_dist/_next/static/{kxGpiudO3f9aX6FAiqydf → pLWT6Sqd09xYpufCVIqnz}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{H5pcGxWf1BkhXDRs2BqiI → JQewSAObpbhO0wrdAM6Ng}/_buildManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{H5pcGxWf1BkhXDRs2BqiI → JQewSAObpbhO0wrdAM6Ng}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{_app-bb4a0e0d783622a8.js.LICENSE.txt → _app-90de19bc03f1484b.js.LICENSE.txt} +0 -0
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/top_level.txt +0 -0
|
@@ -39,6 +39,26 @@ def calculate_item_count(
|
|
|
39
39
|
check_upstream_block_uuids_only: Optional[List[str]] = None,
|
|
40
40
|
update_upstream_item_count_callback: Optional[Callable] = None,
|
|
41
41
|
) -> int:
|
|
42
|
+
"""
|
|
43
|
+
Calculates the item count for a given block based on its upstream blocks' output.
|
|
44
|
+
|
|
45
|
+
This method builds counters for the block and multiplies their item counts to determine
|
|
46
|
+
the total item count. If a counter has an item count of zero and is included in
|
|
47
|
+
`check_upstream_block_uuids_only`, the method recursively calculates the item count
|
|
48
|
+
for the corresponding upstream block. An optional callback can be provided to update
|
|
49
|
+
the item count based on upstream block outputs.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
block: The block for which to calculate the item count.
|
|
53
|
+
execution_partition (Optional[str]): The execution partition identifier.
|
|
54
|
+
check_upstream_block_uuids_only (Optional[List[str]]): A list of UUIDs for upstream blocks
|
|
55
|
+
to check when their item count is zero.
|
|
56
|
+
update_upstream_item_count_callback (Optional[Callable]): A callback function that can
|
|
57
|
+
update the item count based on the upstream block's output.
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
int: The calculated item count for the block.
|
|
61
|
+
"""
|
|
42
62
|
counters = build_counters(block, execution_partition)
|
|
43
63
|
|
|
44
64
|
uuid_counts = [
|
|
@@ -95,7 +115,9 @@ class DynamicBlockFactory(DynamicBlockWrapperBase):
|
|
|
95
115
|
])
|
|
96
116
|
|
|
97
117
|
# If item count is 0 because the upstream blocks haven’t output anything yet
|
|
98
|
-
item_count_total =
|
|
118
|
+
item_count_total = self.__calculate_item_count()
|
|
119
|
+
if item_count_total == 0:
|
|
120
|
+
item_count_total = upstream_blocks_count
|
|
99
121
|
|
|
100
122
|
upstream_block_runs = self.__upstream_block_runs()
|
|
101
123
|
upstream_block_runs_completed = len([
|
|
@@ -174,6 +196,19 @@ class DynamicBlockFactory(DynamicBlockWrapperBase):
|
|
|
174
196
|
return self._counters
|
|
175
197
|
|
|
176
198
|
def __calculate_item_count(self) -> int:
|
|
199
|
+
"""
|
|
200
|
+
Calculates the item count for the current block, ensuring downstream execution even if an
|
|
201
|
+
upstream block has no output.
|
|
202
|
+
|
|
203
|
+
This method calculates the item count by checking the output of upstream blocks. If an
|
|
204
|
+
upstream block has been spawned multiple times but has no output, it ensures that the
|
|
205
|
+
downstream block is still executed by returning an item count of 1. An internal callback
|
|
206
|
+
updates the item count based on the upstream block's completed runs.
|
|
207
|
+
|
|
208
|
+
Returns:
|
|
209
|
+
int: The calculated item count for the block, ensuring downstream block execution.
|
|
210
|
+
"""
|
|
211
|
+
|
|
177
212
|
# If an upstream block indeed has no output (e.g. no return statement),
|
|
178
213
|
# we need to ensure that the downstream block is still executed.
|
|
179
214
|
|
|
@@ -226,10 +261,12 @@ class DynamicBlockFactory(DynamicBlockWrapperBase):
|
|
|
226
261
|
if completed_count == spawn_count:
|
|
227
262
|
return 1
|
|
228
263
|
|
|
229
|
-
# If no upstream block has been spawned,
|
|
264
|
+
# If no upstream block has been spawned and the upstream block runs are completed,
|
|
230
265
|
# we need to return 1 so that the downstream block is still executed.
|
|
231
266
|
if spawn_count == 0:
|
|
232
|
-
|
|
267
|
+
upstream_block_runs = self.__upstream_block_runs()
|
|
268
|
+
if all(b.status == BlockRun.BlockRunStatus.COMPLETED for b in upstream_block_runs):
|
|
269
|
+
return 1
|
|
233
270
|
|
|
234
271
|
return output_item_count
|
|
235
272
|
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
3
|
from dataclasses import dataclass, field
|
|
4
|
-
from enum import Enum
|
|
5
4
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
6
5
|
|
|
7
6
|
import pandas as pd
|
|
@@ -38,11 +37,12 @@ from mage_ai.data_preparation.models.utils import (
|
|
|
38
37
|
from mage_ai.server.kernel_output_parser import DataType
|
|
39
38
|
from mage_ai.shared.array import find
|
|
40
39
|
from mage_ai.shared.custom_logger import DX_PRINTER
|
|
40
|
+
from mage_ai.shared.enum import StrEnum
|
|
41
41
|
from mage_ai.shared.hash import ignore_keys_with_blank_values
|
|
42
42
|
from mage_ai.shared.models import BaseDataClass
|
|
43
43
|
|
|
44
44
|
|
|
45
|
-
class DynamicBlockFlag(
|
|
45
|
+
class DynamicBlockFlag(StrEnum):
|
|
46
46
|
CLONE_OF_ORIGINAL = 'clone_of_original'
|
|
47
47
|
DYNAMIC = 'dynamic'
|
|
48
48
|
DYNAMIC_CHILD = 'dynamic_child'
|
|
@@ -1012,42 +1012,40 @@ def check_all_dynamic_upstreams_completed(
|
|
|
1012
1012
|
execution_partition=execution_partition,
|
|
1013
1013
|
pipeline=pipeline,
|
|
1014
1014
|
):
|
|
1015
|
-
if not upstream_block.upstream_blocks:
|
|
1016
|
-
return True
|
|
1017
|
-
|
|
1018
1015
|
from mage_ai.orchestration.db.models.schedules import BlockRun
|
|
1019
1016
|
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1017
|
+
if upstream_block.upstream_blocks:
|
|
1018
|
+
upstreams_done = all([
|
|
1019
|
+
check_all_dynamic_upstreams_completed(
|
|
1020
|
+
b,
|
|
1021
|
+
block_runs=block_runs,
|
|
1022
|
+
execution_partition=execution_partition,
|
|
1023
|
+
)
|
|
1024
|
+
for b in upstream_block.upstream_blocks
|
|
1025
|
+
])
|
|
1028
1026
|
|
|
1029
|
-
|
|
1030
|
-
|
|
1027
|
+
if not upstreams_done:
|
|
1028
|
+
return False
|
|
1031
1029
|
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1030
|
+
is_dynamic_child = is_dynamic_block_child(upstream_block)
|
|
1031
|
+
if is_dynamic_child:
|
|
1032
|
+
combos = build_combinations_for_dynamic_child(
|
|
1033
|
+
upstream_block,
|
|
1034
|
+
execution_partition=execution_partition,
|
|
1035
|
+
)
|
|
1038
1036
|
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1037
|
+
selected = [
|
|
1038
|
+
br
|
|
1039
|
+
for br in block_runs
|
|
1040
|
+
if pipeline.get_block(
|
|
1041
|
+
br.block_uuid,
|
|
1042
|
+
).uuid
|
|
1043
|
+
== upstream_block.uuid
|
|
1044
|
+
and br.status == BlockRun.BlockRunStatus.COMPLETED
|
|
1045
|
+
]
|
|
1046
|
+
|
|
1047
|
+
if len(list(selected)) < len(combos) + 1:
|
|
1048
|
+
return False
|
|
1051
1049
|
|
|
1052
1050
|
return all([
|
|
1053
1051
|
br.status == BlockRun.BlockRunStatus.COMPLETED
|
|
@@ -734,7 +734,7 @@ def fetch_input_variables_for_dynamic_upstream_blocks(
|
|
|
734
734
|
)
|
|
735
735
|
if index is not None:
|
|
736
736
|
pair = lazy_variable_controller.render(
|
|
737
|
-
child_dynamic_block_index=
|
|
737
|
+
child_dynamic_block_index=index,
|
|
738
738
|
)
|
|
739
739
|
child_data, metadata = pair
|
|
740
740
|
|
|
@@ -45,7 +45,12 @@ class DynamicMixin:
|
|
|
45
45
|
|
|
46
46
|
@property
|
|
47
47
|
def is_dynamic_parent(self) -> bool:
|
|
48
|
-
|
|
48
|
+
parent = self.__dynamic_configuration().parent
|
|
49
|
+
if isinstance(parent, bool):
|
|
50
|
+
return parent
|
|
51
|
+
elif isinstance(parent, list):
|
|
52
|
+
return len(parent) > 0
|
|
53
|
+
return parent is not None
|
|
49
54
|
|
|
50
55
|
@property
|
|
51
56
|
def is_dynamic_child(self) -> bool:
|
|
@@ -57,6 +62,20 @@ class DynamicMixin:
|
|
|
57
62
|
|
|
58
63
|
@property
|
|
59
64
|
def is_dynamic_child_streaming(self) -> bool:
|
|
65
|
+
"""
|
|
66
|
+
Indicates whether current block is dynamic child with stream mode.
|
|
67
|
+
|
|
68
|
+
This property checks if the current block is in dynamic v2 mode (`self.is_dynamic_v2`)
|
|
69
|
+
and whether any of its upstream blocks either:
|
|
70
|
+
- Should dynamically generate the current block and have dynamic stream mode enabled, or
|
|
71
|
+
- recursively check whether upstream block is dynamic child block with stream mode.
|
|
72
|
+
|
|
73
|
+
If any of these conditions are met for any upstream block, the current block is dynamic
|
|
74
|
+
child with stream mode.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
bool: `True` if current block is dynamic child with stream mode., `False` otherwise.
|
|
78
|
+
"""
|
|
60
79
|
return self.is_dynamic_v2 and any(
|
|
61
80
|
(
|
|
62
81
|
upstream_block.should_dynamically_generate_block(self)
|
|
@@ -68,10 +87,24 @@ class DynamicMixin:
|
|
|
68
87
|
|
|
69
88
|
@property
|
|
70
89
|
def should_reduce_output(self) -> bool:
|
|
71
|
-
|
|
90
|
+
reduce_output = self.__dynamic_configuration().reduce_output
|
|
91
|
+
if isinstance(reduce_output, bool):
|
|
92
|
+
return reduce_output
|
|
93
|
+
elif isinstance(reduce_output, list):
|
|
94
|
+
return len(reduce_output) > 0
|
|
95
|
+
return reduce_output is not None
|
|
72
96
|
|
|
73
97
|
@property
|
|
74
98
|
def is_dynamic_stream_mode_enabled(self) -> bool:
|
|
99
|
+
"""
|
|
100
|
+
Indicates whether dynamic stream mode is enabled.
|
|
101
|
+
|
|
102
|
+
This property checks if the setting "type: stream" is in the modes list.
|
|
103
|
+
If the settings are present, dynamic stream mode is considered enabled.
|
|
104
|
+
|
|
105
|
+
Returns:
|
|
106
|
+
bool: `True` if dynamic stream mode is enabled, `False` otherwise.
|
|
107
|
+
"""
|
|
75
108
|
return self.settings_for_mode(ModeType.STREAM) is not None
|
|
76
109
|
|
|
77
110
|
def build_dynamic_uuid(self, index: int) -> str:
|
|
@@ -98,6 +131,22 @@ class DynamicMixin:
|
|
|
98
131
|
return False
|
|
99
132
|
|
|
100
133
|
def should_dynamically_generate_block(self, block) -> bool:
|
|
134
|
+
"""
|
|
135
|
+
Determines whether a block should be dynamically generated based on the parent
|
|
136
|
+
configuration.
|
|
137
|
+
|
|
138
|
+
This method checks the `parent` attribute from the dynamic configuration.
|
|
139
|
+
- If `parent` is `True`, the block should be dynamically generated.
|
|
140
|
+
- If `parent` is a list, the block should be dynamically generated only if its UUID is
|
|
141
|
+
present in the `parent` list.
|
|
142
|
+
- If `parent` is neither `True` nor a list, the block should not be dynamically generated.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
block: The block object that contains the `uuid` attribute to be checked.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
bool: `True` if the block should be dynamically generated, `False` otherwise.
|
|
149
|
+
"""
|
|
101
150
|
parent = self.__dynamic_configuration().parent
|
|
102
151
|
|
|
103
152
|
if parent is True:
|
|
@@ -111,8 +160,18 @@ class DynamicMixin:
|
|
|
111
160
|
def __dynamic_configuration(self) -> DynamicConfiguration:
|
|
112
161
|
if self.configuration:
|
|
113
162
|
config = self.configuration.get('dynamic', {})
|
|
163
|
+
reduce_output = self.configuration.get('reduce_output')
|
|
114
164
|
if config and isinstance(config, dict):
|
|
115
|
-
|
|
165
|
+
dynamic_configuration = DynamicConfiguration.load(**config)
|
|
116
166
|
elif isinstance(config, DynamicConfiguration):
|
|
117
|
-
|
|
167
|
+
dynamic_configuration = config
|
|
168
|
+
else:
|
|
169
|
+
dynamic_configuration = DynamicConfiguration()
|
|
170
|
+
|
|
171
|
+
# For backward compatible with dynamic block v1
|
|
172
|
+
if isinstance(config, bool):
|
|
173
|
+
dynamic_configuration.parent = config
|
|
174
|
+
if isinstance(reduce_output, bool):
|
|
175
|
+
dynamic_configuration.reduce_output = reduce_output
|
|
176
|
+
return dynamic_configuration
|
|
118
177
|
return DynamicConfiguration()
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
|
-
from enum import Enum
|
|
3
2
|
from typing import Optional
|
|
4
3
|
|
|
4
|
+
from mage_ai.shared.enum import StrEnum
|
|
5
5
|
from mage_ai.shared.models import BaseDataClass
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
class GlobalDataProductObjectType(
|
|
8
|
+
class GlobalDataProductObjectType(StrEnum):
|
|
9
9
|
BLOCK = 'block'
|
|
10
10
|
PIPELINE = 'pipeline'
|
|
11
11
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import os
|
|
2
|
-
from enum import Enum
|
|
3
2
|
|
|
4
3
|
from mage_ai.data_preparation.models.variables.constants import (
|
|
5
4
|
VariableAggregateDataType,
|
|
6
5
|
)
|
|
6
|
+
from mage_ai.shared.enum import StrEnum
|
|
7
7
|
|
|
8
8
|
DATAFRAME_ANALYSIS_KEYS = frozenset([
|
|
9
9
|
VariableAggregateDataType.INSIGHTS.value,
|
|
@@ -31,12 +31,12 @@ VARIABLE_DIR = '.variables'
|
|
|
31
31
|
PIPELINE_RUN_STATUS_LAST_RUN_FAILED = 'last_run_failed'
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
class AIMode(
|
|
34
|
+
class AIMode(StrEnum):
|
|
35
35
|
OPEN_AI = 'open_ai'
|
|
36
36
|
HUGGING_FACE = 'hugging_face'
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
class BlockLanguage(
|
|
39
|
+
class BlockLanguage(StrEnum):
|
|
40
40
|
MARKDOWN = 'markdown'
|
|
41
41
|
PYTHON = 'python'
|
|
42
42
|
R = 'r'
|
|
@@ -44,14 +44,14 @@ class BlockLanguage(str, Enum):
|
|
|
44
44
|
YAML = 'yaml'
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
class BlockStatus(
|
|
47
|
+
class BlockStatus(StrEnum):
|
|
48
48
|
EXECUTED = 'executed'
|
|
49
49
|
FAILED = 'failed'
|
|
50
50
|
NOT_EXECUTED = 'not_executed'
|
|
51
51
|
UPDATED = 'updated'
|
|
52
52
|
|
|
53
53
|
|
|
54
|
-
class BlockType(
|
|
54
|
+
class BlockType(StrEnum):
|
|
55
55
|
CALLBACK = 'callback'
|
|
56
56
|
CONDITIONAL = 'conditional'
|
|
57
57
|
CHART = 'chart'
|
|
@@ -71,7 +71,7 @@ class BlockType(str, Enum):
|
|
|
71
71
|
PIPELINE = 'pipeline'
|
|
72
72
|
|
|
73
73
|
|
|
74
|
-
class BlockColor(
|
|
74
|
+
class BlockColor(StrEnum):
|
|
75
75
|
BLUE = 'blue'
|
|
76
76
|
GREY = 'grey'
|
|
77
77
|
PINK = 'pink'
|
|
@@ -80,12 +80,12 @@ class BlockColor(str, Enum):
|
|
|
80
80
|
YELLOW = 'yellow'
|
|
81
81
|
|
|
82
82
|
|
|
83
|
-
class CallbackStatus(
|
|
83
|
+
class CallbackStatus(StrEnum):
|
|
84
84
|
FAILURE = 'failure'
|
|
85
85
|
SUCCESS = 'success'
|
|
86
86
|
|
|
87
87
|
|
|
88
|
-
class ExecutorType(
|
|
88
|
+
class ExecutorType(StrEnum):
|
|
89
89
|
AZURE_CONTAINER_INSTANCE = 'azure_container_instance'
|
|
90
90
|
ECS = 'ecs'
|
|
91
91
|
GCP_CLOUD_RUN = 'gcp_cloud_run'
|
|
@@ -100,7 +100,7 @@ class ExecutorType(str, Enum):
|
|
|
100
100
|
return executor_type.upper() in cls.__members__
|
|
101
101
|
|
|
102
102
|
|
|
103
|
-
class PipelineType(
|
|
103
|
+
class PipelineType(StrEnum):
|
|
104
104
|
INTEGRATION = 'integration'
|
|
105
105
|
DATABRICKS = 'databricks'
|
|
106
106
|
PYTHON = 'python'
|
|
@@ -109,7 +109,7 @@ class PipelineType(str, Enum):
|
|
|
109
109
|
EXECUTION_FRAMEWORK = 'execution_framework'
|
|
110
110
|
|
|
111
111
|
|
|
112
|
-
class PipelineStatus(
|
|
112
|
+
class PipelineStatus(StrEnum):
|
|
113
113
|
ACTIVE = ('active',) # At least one active trigger
|
|
114
114
|
INACTIVE = ('inactive',) # All inactive triggers
|
|
115
115
|
NO_SCHEDULES = ('no_schedules',) # No triggers
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
|
-
|
|
3
1
|
from mage_ai.authentication.permissions.constants import EntityName
|
|
2
|
+
from mage_ai.shared.enum import StrEnum
|
|
4
3
|
|
|
5
4
|
GLOBAL_HOOKS_FILENAME = 'global_hooks.yaml'
|
|
6
5
|
|
|
@@ -30,7 +29,7 @@ RESTRICTED_RESOURCE_TYPES = [
|
|
|
30
29
|
RESOURCE_TYPES = [en for en in EntityName if en not in DISABLED_RESOURCE_TYPES]
|
|
31
30
|
|
|
32
31
|
|
|
33
|
-
class HookOutputKey(
|
|
32
|
+
class HookOutputKey(StrEnum):
|
|
34
33
|
ERROR = 'error'
|
|
35
34
|
META = 'meta'
|
|
36
35
|
METADATA = 'metadata'
|
|
@@ -40,7 +39,7 @@ class HookOutputKey(str, Enum):
|
|
|
40
39
|
RESOURCES = 'resources'
|
|
41
40
|
|
|
42
41
|
|
|
43
|
-
class HookInputKey(
|
|
42
|
+
class HookInputKey(StrEnum):
|
|
44
43
|
HOOK = 'hook'
|
|
45
44
|
PROJECT = 'project'
|
|
46
45
|
RESOURCE_ID = 'resource_id'
|
|
@@ -67,12 +66,12 @@ VALID_KEYS_FOR_INPUT_OUTPUT_DATA_ALL = \
|
|
|
67
66
|
INTERNAL_DEFAULT_PREDICATE_VALUE = '__INTERNAL_DEFAULT_PREDICATE_VALUE__'
|
|
68
67
|
|
|
69
68
|
|
|
70
|
-
class PredicateAndOrOperator(
|
|
69
|
+
class PredicateAndOrOperator(StrEnum):
|
|
71
70
|
AND = 'and'
|
|
72
71
|
OR = 'or'
|
|
73
72
|
|
|
74
73
|
|
|
75
|
-
class PredicateObjectType(
|
|
74
|
+
class PredicateObjectType(StrEnum):
|
|
76
75
|
ERROR = HookOutputKey.ERROR.value
|
|
77
76
|
HOOK = HookInputKey.HOOK.value
|
|
78
77
|
META = HookOutputKey.META.value
|
|
@@ -89,7 +88,7 @@ class PredicateObjectType(str, Enum):
|
|
|
89
88
|
USER = HookInputKey.USER.value
|
|
90
89
|
|
|
91
90
|
|
|
92
|
-
class PredicateOperator(
|
|
91
|
+
class PredicateOperator(StrEnum):
|
|
93
92
|
EQUALS = 'EQUALS'
|
|
94
93
|
GREATER_THAN = 'GREATER_THAN'
|
|
95
94
|
GREATER_THAN_OR_EQUALS = 'GREATER_THAN_OR_EQUALS'
|
|
@@ -102,7 +101,7 @@ class PredicateOperator(str, Enum):
|
|
|
102
101
|
PRESENT = 'PRESENT'
|
|
103
102
|
|
|
104
103
|
|
|
105
|
-
class PredicateValueDataType(
|
|
104
|
+
class PredicateValueDataType(StrEnum):
|
|
106
105
|
BOOLEAN = 'BOOLEAN'
|
|
107
106
|
DICTIONARY = 'DICTIONARY'
|
|
108
107
|
FLOAT = 'FLOAT'
|
|
@@ -2,7 +2,6 @@ import hashlib
|
|
|
2
2
|
import os
|
|
3
3
|
from dataclasses import dataclass, field, make_dataclass
|
|
4
4
|
from datetime import datetime
|
|
5
|
-
from enum import Enum
|
|
6
5
|
from typing import Dict, List, Tuple, Union
|
|
7
6
|
|
|
8
7
|
import yaml
|
|
@@ -30,6 +29,7 @@ from mage_ai.settings.platform import (
|
|
|
30
29
|
)
|
|
31
30
|
from mage_ai.settings.repo import get_repo_path
|
|
32
31
|
from mage_ai.shared.array import find, find_index, flatten
|
|
32
|
+
from mage_ai.shared.enum import StrEnum
|
|
33
33
|
from mage_ai.shared.environments import is_debug, is_test
|
|
34
34
|
from mage_ai.shared.hash import (
|
|
35
35
|
dig,
|
|
@@ -44,7 +44,7 @@ from mage_ai.shared.models import BaseDataClass
|
|
|
44
44
|
from mage_ai.shared.multi import run_parallel_multiple_args
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
class HookOperation(
|
|
47
|
+
class HookOperation(StrEnum):
|
|
48
48
|
CREATE = OperationType.CREATE.value
|
|
49
49
|
DELETE = OperationType.DELETE.value
|
|
50
50
|
DETAIL = OperationType.DETAIL.value
|
|
@@ -54,18 +54,18 @@ class HookOperation(str, Enum):
|
|
|
54
54
|
UPDATE_ANYWHERE = 'update_anywhere'
|
|
55
55
|
|
|
56
56
|
|
|
57
|
-
class HookCondition(
|
|
57
|
+
class HookCondition(StrEnum):
|
|
58
58
|
FAILURE = 'failure'
|
|
59
59
|
SUCCESS = 'success'
|
|
60
60
|
|
|
61
61
|
|
|
62
|
-
class HookStrategy(
|
|
62
|
+
class HookStrategy(StrEnum):
|
|
63
63
|
BREAK = 'break'
|
|
64
64
|
CONTINUE = 'continue'
|
|
65
65
|
RAISE = 'raise'
|
|
66
66
|
|
|
67
67
|
|
|
68
|
-
class HookStage(
|
|
68
|
+
class HookStage(StrEnum):
|
|
69
69
|
AFTER = 'after'
|
|
70
70
|
BEFORE = 'before'
|
|
71
71
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import enum
|
|
2
1
|
import os
|
|
3
2
|
import traceback
|
|
4
3
|
from dataclasses import dataclass, field
|
|
@@ -12,6 +11,7 @@ from mage_ai.data_preparation.models.constants import PIPELINES_FOLDER
|
|
|
12
11
|
from mage_ai.settings.repo import get_repo_path
|
|
13
12
|
from mage_ai.shared.config import BaseConfig
|
|
14
13
|
from mage_ai.shared.constants import VALID_ENVS
|
|
14
|
+
from mage_ai.shared.enum import StrEnum
|
|
15
15
|
from mage_ai.shared.hash import index_by
|
|
16
16
|
from mage_ai.shared.io import safe_write
|
|
17
17
|
from mage_ai.shared.yaml import load_yaml
|
|
@@ -19,12 +19,12 @@ from mage_ai.shared.yaml import load_yaml
|
|
|
19
19
|
TRIGGER_FILE_NAME = 'triggers.yaml'
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
class ScheduleStatus(
|
|
22
|
+
class ScheduleStatus(StrEnum):
|
|
23
23
|
ACTIVE = 'active'
|
|
24
24
|
INACTIVE = 'inactive'
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
class ScheduleType(
|
|
27
|
+
class ScheduleType(StrEnum):
|
|
28
28
|
API = 'api'
|
|
29
29
|
EVENT = 'event'
|
|
30
30
|
TIME = 'time'
|
|
@@ -37,7 +37,7 @@ SCHEDULE_TYPE_TO_LABEL = {
|
|
|
37
37
|
}
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
class ScheduleInterval(
|
|
40
|
+
class ScheduleInterval(StrEnum):
|
|
41
41
|
ONCE = '@once'
|
|
42
42
|
HOURLY = '@hourly'
|
|
43
43
|
DAILY = '@daily'
|
|
@@ -266,6 +266,8 @@ def build_triggers(
|
|
|
266
266
|
|
|
267
267
|
# Add flag to settings so frontend can detect triggers with invalid cron expressions
|
|
268
268
|
if not trigger.has_valid_schedule_interval:
|
|
269
|
+
if not trigger.settings:
|
|
270
|
+
trigger.settings = dict()
|
|
269
271
|
trigger.settings['invalid_schedule_interval'] = True
|
|
270
272
|
|
|
271
273
|
triggers.append(trigger)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
CONFIG_JSON_FILE = 'config.json'
|
|
4
4
|
DATAFRAME_COLUMN_TYPES_FILE = 'data_column_types.json'
|
|
@@ -14,7 +14,7 @@ MEDIA_IMAGE_VISUALIZATION_FILE = 'visualization.png'
|
|
|
14
14
|
UBJSON_MODEL_FILENAME = 'model.ubj'
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
class VariableType(
|
|
17
|
+
class VariableType(StrEnum):
|
|
18
18
|
CUSTOM_OBJECT = 'custom_object'
|
|
19
19
|
DATAFRAME = 'dataframe'
|
|
20
20
|
DATAFRAME_ANALYSIS = 'dataframe_analysis'
|
|
@@ -31,7 +31,7 @@ class VariableType(str, Enum):
|
|
|
31
31
|
SPARK_DATAFRAME = 'spark_dataframe'
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
class VariableAggregateDataType(
|
|
34
|
+
class VariableAggregateDataType(StrEnum):
|
|
35
35
|
INSIGHTS = 'insights'
|
|
36
36
|
METADATA = 'metadata'
|
|
37
37
|
RESOURCE_USAGE = 'resource_usage'
|
|
@@ -41,7 +41,7 @@ class VariableAggregateDataType(str, Enum):
|
|
|
41
41
|
TYPE = 'type'
|
|
42
42
|
|
|
43
43
|
|
|
44
|
-
class VariableAggregateDataTypeFilename(
|
|
44
|
+
class VariableAggregateDataTypeFilename(StrEnum):
|
|
45
45
|
INSIGHTS = f'{VariableAggregateDataType.INSIGHTS}.json'
|
|
46
46
|
METADATA = f'{VariableAggregateDataType.METADATA}.json'
|
|
47
47
|
RESOURCE_USAGE = f'{VariableAggregateDataType.RESOURCE_USAGE}.json'
|
|
@@ -67,6 +67,6 @@ JSON_SAMPLE_FILE = VariableAggregateDataTypeFilename.SAMPLE_DATA.value
|
|
|
67
67
|
METADATA_FILE = VariableAggregateDataTypeFilename.TYPE.value
|
|
68
68
|
|
|
69
69
|
|
|
70
|
-
class VariableAggregateSummaryGroupType(
|
|
70
|
+
class VariableAggregateSummaryGroupType(StrEnum):
|
|
71
71
|
DYNAMIC = 'dynamic'
|
|
72
72
|
PARTS = 'parts'
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
|
-
|
|
3
1
|
from dateutil.relativedelta import relativedelta
|
|
4
2
|
|
|
3
|
+
from mage_ai.shared.enum import StrEnum
|
|
4
|
+
|
|
5
5
|
VARIABLE_NAME_BUCKETS = 'buckets'
|
|
6
6
|
VARIABLE_NAME_GROUP_BY = 'group_by'
|
|
7
7
|
VARIABLE_NAME_ORDER_BY = 'order_by'
|
|
@@ -14,7 +14,7 @@ VARIABLE_NAME_X = 'x'
|
|
|
14
14
|
VARIABLE_NAME_Y = 'y'
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
class AggregationFunction(
|
|
17
|
+
class AggregationFunction(StrEnum):
|
|
18
18
|
AVERAGE = 'average'
|
|
19
19
|
COUNT = 'count'
|
|
20
20
|
COUNT_DISTINCT = 'count_distinct'
|
|
@@ -25,7 +25,7 @@ class AggregationFunction(str, Enum):
|
|
|
25
25
|
SUM = 'sum'
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
class TimeInterval(
|
|
28
|
+
class TimeInterval(StrEnum):
|
|
29
29
|
DAY = 'day'
|
|
30
30
|
HOUR = 'hour'
|
|
31
31
|
MINUTE = 'minute'
|
|
@@ -36,7 +36,7 @@ class TimeInterval(str, Enum):
|
|
|
36
36
|
YEAR = 'year'
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
class ChartType(
|
|
39
|
+
class ChartType(StrEnum):
|
|
40
40
|
BAR_CHART = 'bar chart'
|
|
41
41
|
CUSTOM = 'custom'
|
|
42
42
|
HISTOGRAM = 'histogram'
|
|
@@ -37,15 +37,13 @@ ENV_VAR_TO_CONFIG_KEY = {
|
|
|
37
37
|
GIT_SYNC_SUBMODULES: 'sync_submodules',
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
BOOLEAN_ENV_VARS = set(
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
]
|
|
48
|
-
)
|
|
40
|
+
BOOLEAN_ENV_VARS = set([
|
|
41
|
+
GIT_SYNC_ON_PIPELINE_RUN,
|
|
42
|
+
GIT_SYNC_ON_START,
|
|
43
|
+
GIT_SYNC_ON_EXECUTOR_START,
|
|
44
|
+
GIT_SYNC_SUBMODULES,
|
|
45
|
+
GIT_ENABLE_GIT_INTEGRATION,
|
|
46
|
+
])
|
|
49
47
|
|
|
50
48
|
|
|
51
49
|
def get_value_for_sync_config(env_var, value) -> bool:
|
|
@@ -67,10 +65,7 @@ def build_sync_config(project_sync_config: Dict) -> Dict:
|
|
|
67
65
|
}
|
|
68
66
|
|
|
69
67
|
# Read settings from settings backend
|
|
70
|
-
config_from_settings = {
|
|
71
|
-
k: get_settings_value(k)
|
|
72
|
-
for k in ENV_VAR_TO_CONFIG_KEY.keys()
|
|
73
|
-
}
|
|
68
|
+
config_from_settings = {k: get_settings_value(k) for k in ENV_VAR_TO_CONFIG_KEY.keys()}
|
|
74
69
|
|
|
75
70
|
if any([v is not None for v in config_from_settings.values()]):
|
|
76
71
|
# Map sync config keys to settings values
|
|
@@ -153,9 +148,7 @@ class Preferences:
|
|
|
153
148
|
def get_preferences(repo_path=None, user: User = None) -> Preferences:
|
|
154
149
|
default_preferences = Preferences(repo_path=repo_path)
|
|
155
150
|
if user:
|
|
156
|
-
if user.preferences is None and os.path.exists(
|
|
157
|
-
default_preferences.preferences_file_path
|
|
158
|
-
):
|
|
151
|
+
if user.preferences is None and os.path.exists(default_preferences.preferences_file_path):
|
|
159
152
|
return default_preferences
|
|
160
153
|
else:
|
|
161
154
|
return Preferences(user=user, repo_path=repo_path)
|