mage-ai 0.9.73__py3-none-any.whl → 0.9.74__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mage_ai/ai/constants.py +2 -2
- mage_ai/ai/llm_pipeline_wizard.py +5 -3
- mage_ai/ai/openai_client.py +11 -0
- mage_ai/api/constants.py +4 -5
- mage_ai/api/oauth_scope.py +2 -2
- mage_ai/api/operations/constants.py +2 -2
- mage_ai/api/presenters/SyncPresenter.py +4 -1
- mage_ai/authentication/oauth/constants.py +2 -2
- mage_ai/authentication/operation_history/constants.py +2 -2
- mage_ai/authentication/permissions/constants.py +5 -6
- mage_ai/cache/constants.py +2 -2
- mage_ai/cache/dbt/constants.py +2 -3
- mage_ai/cluster_manager/constants.py +2 -2
- mage_ai/cluster_manager/kubernetes/workload_manager.py +3 -2
- mage_ai/command_center/constants.py +13 -13
- mage_ai/data_cleaner/column_types/constants.py +2 -2
- mage_ai/data_cleaner/transformer_actions/column.py +19 -7
- mage_ai/data_cleaner/transformer_actions/constants.py +10 -9
- mage_ai/data_integrations/sources/constants.py +2 -0
- mage_ai/data_preparation/git/utils.py +4 -1
- mage_ai/data_preparation/logging/__init__.py +2 -2
- mage_ai/data_preparation/logging/logger.py +3 -3
- mage_ai/data_preparation/models/block/__init__.py +9 -3
- mage_ai/data_preparation/models/block/data_integration/constants.py +2 -2
- mage_ai/data_preparation/models/block/dbt/constants.py +3 -3
- mage_ai/data_preparation/models/block/dynamic/factory.py +40 -3
- mage_ai/data_preparation/models/block/dynamic/utils.py +31 -33
- mage_ai/data_preparation/models/block/dynamic/variables.py +1 -1
- mage_ai/data_preparation/models/block/settings/dynamic/constants.py +3 -3
- mage_ai/data_preparation/models/block/settings/dynamic/mixins.py +63 -4
- mage_ai/data_preparation/models/block/settings/global_data_products/models.py +2 -2
- mage_ai/data_preparation/models/constants.py +10 -10
- mage_ai/data_preparation/models/global_hooks/constants.py +7 -8
- mage_ai/data_preparation/models/global_hooks/models.py +5 -5
- mage_ai/data_preparation/models/project/constants.py +2 -2
- mage_ai/data_preparation/models/triggers/__init__.py +6 -4
- mage_ai/data_preparation/models/variables/constants.py +5 -5
- mage_ai/data_preparation/models/widget/constants.py +5 -5
- mage_ai/data_preparation/preferences.py +9 -16
- mage_ai/data_preparation/repo_manager.py +2 -2
- mage_ai/data_preparation/sync/__init__.py +2 -2
- mage_ai/data_preparation/templates/data_exporters/streaming/generic_python.py +1 -1
- mage_ai/data_preparation/templates/data_loaders/streaming/nats.yaml +3 -0
- mage_ai/errors/constants.py +2 -2
- mage_ai/io/base.py +28 -15
- mage_ai/io/config.py +3 -3
- mage_ai/io/export_utils.py +2 -2
- mage_ai/io/google_cloud_storage.py +3 -2
- mage_ai/io/io_config.py +3 -2
- mage_ai/io/postgres.py +2 -1
- mage_ai/kernels/magic/constants.py +4 -4
- mage_ai/orchestration/constants.py +2 -2
- mage_ai/orchestration/db/constants.py +2 -2
- mage_ai/orchestration/db/models/oauth.py +5 -5
- mage_ai/orchestration/db/models/schedules.py +28 -8
- mage_ai/orchestration/db/models/schedules_project_platform.py +10 -8
- mage_ai/orchestration/job_manager.py +2 -2
- mage_ai/orchestration/monitor/monitor_stats.py +2 -2
- mage_ai/orchestration/notification/config.py +2 -2
- mage_ai/orchestration/pipeline_scheduler_original.py +4 -12
- mage_ai/orchestration/queue/config.py +2 -2
- mage_ai/orchestration/queue/process_queue.py +3 -3
- mage_ai/presenters/charts/data_sources/constants.py +2 -2
- mage_ai/presenters/interactions/constants.py +4 -4
- mage_ai/presenters/pages/models/constants.py +4 -4
- mage_ai/server/constants.py +1 -1
- mage_ai/server/frontend_dist/404.html +2 -2
- mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-663c909dcda4c23a.js → _app-5bdff745074fb350.js} +2 -2
- mage_ai/server/frontend_dist/_next/static/chunks/{webpack-43534cc51fce8644.js → webpack-b9a067f3bd0a3a05.js} +1 -1
- mage_ai/server/frontend_dist/block-layout.html +2 -2
- mage_ai/server/frontend_dist/compute.html +2 -2
- mage_ai/server/frontend_dist/files.html +2 -2
- mage_ai/server/frontend_dist/global-data-products/[...slug].html +2 -2
- mage_ai/server/frontend_dist/global-data-products.html +2 -2
- mage_ai/server/frontend_dist/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist/global-hooks.html +2 -2
- mage_ai/server/frontend_dist/index.html +2 -2
- mage_ai/server/frontend_dist/manage/files.html +2 -2
- mage_ai/server/frontend_dist/manage/overview.html +2 -2
- mage_ai/server/frontend_dist/manage/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist/manage/settings.html +2 -2
- mage_ai/server/frontend_dist/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist/manage/users.html +2 -2
- mage_ai/server/frontend_dist/manage.html +2 -2
- mage_ai/server/frontend_dist/oauth.html +3 -3
- mage_ai/server/frontend_dist/overview.html +2 -2
- mage_ai/server/frontend_dist/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/dashboard.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist/pipelines.html +2 -2
- mage_ai/server/frontend_dist/platform/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist/platform/global-hooks.html +2 -2
- mage_ai/server/frontend_dist/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist/settings/platform/preferences.html +2 -2
- mage_ai/server/frontend_dist/settings/platform/settings.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/permissions/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/permissions.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/roles/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/roles.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/users/[...slug].html +2 -2
- mage_ai/server/frontend_dist/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist/settings.html +2 -2
- mage_ai/server/frontend_dist/sign-in.html +5 -5
- mage_ai/server/frontend_dist/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist/templates.html +2 -2
- mage_ai/server/frontend_dist/terminal.html +2 -2
- mage_ai/server/frontend_dist/test.html +2 -2
- mage_ai/server/frontend_dist/triggers.html +2 -2
- mage_ai/server/frontend_dist/v2/canvas.html +2 -2
- mage_ai/server/frontend_dist/v2.html +2 -2
- mage_ai/server/frontend_dist/version-control.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/404.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{_app-bb4a0e0d783622a8.js → _app-90de19bc03f1484b.js} +2 -2
- mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/{webpack-9eb1dd2ee735aaac.js → webpack-12ad70eb5c31aa92.js} +1 -1
- mage_ai/server/frontend_dist_base_path_template/block-layout.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/compute.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/files.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-data-products/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-data-products.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/global-hooks.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/index.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/files.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/overview.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users/[user].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users/new.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage/users.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/manage.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/oauth.html +3 -3
- mage_ai/server/frontend_dist_base_path_template/overview.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipeline-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/backfills.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/dashboard.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/edit.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/logs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors/block-runtime.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/monitors.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs/[run].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/runs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/syncs.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline]/triggers.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines/[pipeline].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/pipelines.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/platform/global-hooks.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/account/profile.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/platform/preferences.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/platform/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/permissions.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/preferences.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/roles.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/sync-data.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings/workspace/users.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/settings.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/sign-in.html +5 -5
- mage_ai/server/frontend_dist_base_path_template/templates/[...slug].html +2 -2
- mage_ai/server/frontend_dist_base_path_template/templates.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/terminal.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/test.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/triggers.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/v2/canvas.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/v2.html +2 -2
- mage_ai/server/frontend_dist_base_path_template/version-control.html +2 -2
- mage_ai/server/kernel_output_parser.py +2 -3
- mage_ai/server/kernels.py +2 -3
- mage_ai/server/scheduler_manager.py +2 -2
- mage_ai/server/server.py +12 -0
- mage_ai/server/websockets/constants.py +4 -4
- mage_ai/services/compute/aws/constants.py +2 -2
- mage_ai/services/compute/aws/steps.py +16 -10
- mage_ai/services/compute/constants.py +4 -4
- mage_ai/services/compute/models.py +4 -4
- mage_ai/services/k8s/config.py +4 -0
- mage_ai/services/k8s/job_manager.py +2 -0
- mage_ai/services/spark/constants.py +3 -3
- mage_ai/services/spark/models/jobs.py +2 -2
- mage_ai/services/spark/models/sqls.py +2 -2
- mage_ai/services/spark/models/stages.py +4 -4
- mage_ai/services/spark/models/threads.py +2 -2
- mage_ai/settings/backends.py +3 -2
- mage_ai/settings/models/configuration_option.py +3 -3
- mage_ai/settings/server.py +5 -0
- mage_ai/shared/constants.py +3 -3
- mage_ai/shared/custom_logger.py +13 -13
- mage_ai/shared/enum.py +11 -0
- mage_ai/shared/environments.py +1 -1
- mage_ai/shared/logger.py +2 -2
- mage_ai/shared/models.py +2 -1
- mage_ai/streaming/constants.py +3 -3
- mage_ai/streaming/sinks/kafka.py +2 -2
- mage_ai/streaming/sinks/postgres.py +6 -0
- mage_ai/streaming/sources/amazon_sqs.py +2 -2
- mage_ai/streaming/sources/base.py +2 -2
- mage_ai/streaming/sources/kafka.py +20 -4
- mage_ai/streaming/sources/nats_js.py +10 -3
- mage_ai/streaming/sources/shared.py +3 -2
- mage_ai/system/constants.py +2 -2
- mage_ai/tests/api/policies/permissions/test_base_policy_with_permissions.py +2 -2
- mage_ai/tests/data_cleaner/transformer_actions/test_trim_transformer.py +161 -0
- mage_ai/tests/data_preparation/models/block/dbt/test_profiles.py +1 -1
- mage_ai/tests/data_preparation/models/block/dynamic/test_combos.py +1 -1
- mage_ai/tests/data_preparation/models/block/hook/test_hook_block.py +3 -2
- mage_ai/tests/data_preparation/models/test_blocks_helper.py +1 -1
- mage_ai/tests/data_preparation/models/variables/test_summarizer.py +1 -1
- mage_ai/tests/data_preparation/sync/test_git_sync.py +6 -6
- mage_ai/tests/orchestration/queue/test_process_queue.py +3 -2
- mage_ai/tests/orchestration/test_pipeline_scheduler.py +1 -0
- mage_ai/tests/settings/test_platform.py +2 -2
- mage_ai/tests/streaming/sinks/test_generic_io.py +25 -21
- mage_ai/usage_statistics/constants.py +4 -4
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/METADATA +172 -171
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/RECORD +247 -245
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/WHEEL +1 -1
- /mage_ai/server/frontend_dist/_next/static/chunks/pages/{_app-663c909dcda4c23a.js.LICENSE.txt → _app-5bdff745074fb350.js.LICENSE.txt} +0 -0
- /mage_ai/server/frontend_dist/_next/static/{kxGpiudO3f9aX6FAiqydf → pLWT6Sqd09xYpufCVIqnz}/_buildManifest.js +0 -0
- /mage_ai/server/frontend_dist/_next/static/{kxGpiudO3f9aX6FAiqydf → pLWT6Sqd09xYpufCVIqnz}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{H5pcGxWf1BkhXDRs2BqiI → JQewSAObpbhO0wrdAM6Ng}/_buildManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/{H5pcGxWf1BkhXDRs2BqiI → JQewSAObpbhO0wrdAM6Ng}/_ssgManifest.js +0 -0
- /mage_ai/server/frontend_dist_base_path_template/_next/static/chunks/pages/{_app-bb4a0e0d783622a8.js.LICENSE.txt → _app-90de19bc03f1484b.js.LICENSE.txt} +0 -0
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/LICENSE +0 -0
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/entry_points.txt +0 -0
- {mage_ai-0.9.73.dist-info → mage_ai-0.9.74.dist-info}/top_level.txt +0 -0
mage_ai/ai/constants.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
class LLMUseCase(
|
|
4
|
+
class LLMUseCase(StrEnum):
|
|
5
5
|
GENERATE_DOC_FOR_BLOCK = 'generate_doc_for_block'
|
|
6
6
|
GENERATE_DOC_FOR_PIPELINE = 'generate_doc_for_pipeline'
|
|
7
7
|
GENERATE_BLOCK_WITH_DESCRIPTION = 'generate_block_with_description'
|
|
@@ -30,6 +30,7 @@ from mage_ai.data_preparation.templates.template import (
|
|
|
30
30
|
from mage_ai.io.base import DataSource
|
|
31
31
|
from mage_ai.orchestration.ai.config import AIConfig
|
|
32
32
|
from mage_ai.server.logger import Logger
|
|
33
|
+
from mage_ai.settings import ENABLE_HUGGING_FACE, ENABLE_OPEN_AI
|
|
33
34
|
|
|
34
35
|
logger = Logger().new_server_logger(__name__)
|
|
35
36
|
|
|
@@ -190,12 +191,13 @@ TEMPLATE_CLASSIFICATION_FUNCTION = [
|
|
|
190
191
|
class LLMPipelineWizard:
|
|
191
192
|
def __init__(self):
|
|
192
193
|
ai_config = AIConfig.load(config=get_repo_config().ai_config)
|
|
193
|
-
if ai_config.mode == AIMode.OPEN_AI:
|
|
194
|
+
if ENABLE_OPEN_AI and ai_config.mode == AIMode.OPEN_AI:
|
|
194
195
|
self.client = OpenAIClient(ai_config.open_ai_config)
|
|
195
|
-
elif ai_config.mode == AIMode.HUGGING_FACE:
|
|
196
|
+
elif ENABLE_HUGGING_FACE and ai_config.mode == AIMode.HUGGING_FACE:
|
|
196
197
|
self.client = HuggingFaceClient(ai_config.hugging_face_config)
|
|
197
198
|
else:
|
|
198
|
-
raise Exception('AI Mode is not available.'
|
|
199
|
+
raise Exception('AI Mode is not available. ENABLE_OPEN_AI: '
|
|
200
|
+
f'{ENABLE_OPEN_AI}, ENABLE_HUGGING_FACE: {ENABLE_HUGGING_FACE}')
|
|
199
201
|
|
|
200
202
|
async def __async_llm_call(
|
|
201
203
|
self,
|
mage_ai/ai/openai_client.py
CHANGED
|
@@ -187,7 +187,18 @@ class OpenAIClient(AIClient):
|
|
|
187
187
|
self,
|
|
188
188
|
block_description: str):
|
|
189
189
|
messages = [{'role': 'user', 'content': block_description}]
|
|
190
|
+
# Fetch response form API call with retries
|
|
191
|
+
# retry __chat_completion_request twice.
|
|
192
|
+
# If it still returns error, raise error in find_block_params.
|
|
193
|
+
# If not error anymore, proceed with rest of the code change.
|
|
194
|
+
max_retries = 2
|
|
195
|
+
attempt = 0
|
|
190
196
|
response = self.__chat_completion_request(messages)
|
|
197
|
+
while attempt <= max_retries and isinstance(response, Exception):
|
|
198
|
+
response = self.__chat_completion_request(messages)
|
|
199
|
+
attempt += 1
|
|
200
|
+
if isinstance(response, Exception):
|
|
201
|
+
raise Exception("Error in __chat_completion_request after retries: " + str(response))
|
|
191
202
|
arguments = response.choices[0].message.tool_calls[0].function.arguments
|
|
192
203
|
if arguments:
|
|
193
204
|
function_args = json.loads(arguments)
|
mage_ai/api/constants.py
CHANGED
|
@@ -1,20 +1,19 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
|
-
|
|
3
1
|
from mage_ai.api.operations.constants import OperationType
|
|
4
2
|
from mage_ai.orchestration.db.models.oauth import Permission
|
|
3
|
+
from mage_ai.shared.enum import StrEnum
|
|
5
4
|
|
|
6
5
|
|
|
7
|
-
class AttributeOperationType(
|
|
6
|
+
class AttributeOperationType(StrEnum):
|
|
8
7
|
QUERY = 'query'
|
|
9
8
|
READ = 'read'
|
|
10
9
|
WRITE = 'write'
|
|
11
10
|
|
|
12
11
|
|
|
13
|
-
class AttributeType(
|
|
12
|
+
class AttributeType(StrEnum):
|
|
14
13
|
ALL = '__*MAGE*__'
|
|
15
14
|
|
|
16
15
|
|
|
17
|
-
class AuthorizeStatusType(
|
|
16
|
+
class AuthorizeStatusType(StrEnum):
|
|
18
17
|
ALL = 'all'
|
|
19
18
|
FAILED = 'failed'
|
|
20
19
|
SUCCEEDED = 'succeeded'
|
mage_ai/api/oauth_scope.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class OauthScope():
|
|
@@ -18,7 +18,7 @@ class OauthScope():
|
|
|
18
18
|
TOKEN_SCOPES = []
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
class OauthScopeType(
|
|
21
|
+
class OauthScopeType(StrEnum):
|
|
22
22
|
CLIENT_ALL = 'all'
|
|
23
23
|
CLIENT_INTERNAL = 'internal'
|
|
24
24
|
CLIENT_PRIVATE = 'private'
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
ALL = 'all'
|
|
4
4
|
CREATE = 'create'
|
|
@@ -20,7 +20,7 @@ META_KEY_ORDER_BY = '_order_by[]'
|
|
|
20
20
|
COOKIE_PREFIX = '__COOKIE__'
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
class OperationType(
|
|
23
|
+
class OperationType(StrEnum):
|
|
24
24
|
ALL = ALL
|
|
25
25
|
CREATE = CREATE
|
|
26
26
|
DELETE = DELETE
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from typing import Any, Union
|
|
2
|
+
|
|
1
3
|
from mage_ai.api.presenters.BasePresenter import BasePresenter
|
|
2
4
|
from mage_ai.data_preparation.shared.secrets import get_secret_value
|
|
3
5
|
from mage_ai.settings.repo import get_repo_path
|
|
@@ -16,6 +18,7 @@ class SyncPresenter(BasePresenter):
|
|
|
16
18
|
'ssh_private_key_secret_name',
|
|
17
19
|
'ssh_public_key',
|
|
18
20
|
'ssh_public_key_secret_name',
|
|
21
|
+
'sync_on_executor_start',
|
|
19
22
|
'sync_on_pipeline_run',
|
|
20
23
|
'sync_on_start',
|
|
21
24
|
'sync_submodules',
|
|
@@ -24,7 +27,7 @@ class SyncPresenter(BasePresenter):
|
|
|
24
27
|
'username',
|
|
25
28
|
]
|
|
26
29
|
|
|
27
|
-
def
|
|
30
|
+
async def prepare_present(self, **kwargs) -> Union[Any, None]:
|
|
28
31
|
data = self.model
|
|
29
32
|
|
|
30
33
|
def filter_invalid_secret_values(data):
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
1
|
from typing import Optional
|
|
3
2
|
|
|
4
3
|
from mage_ai.settings import get_settings_value
|
|
5
4
|
from mage_ai.settings.keys import GHE_HOSTNAME
|
|
5
|
+
from mage_ai.shared.enum import StrEnum
|
|
6
6
|
|
|
7
7
|
ACTIVE_DIRECTORY_CLIENT_ID = '51aec820-9d49-40a9-b046-17c1f28f620d'
|
|
8
8
|
|
|
@@ -10,7 +10,7 @@ GITHUB_CLIENT_ID = '8577f13ddc81e2848b07'
|
|
|
10
10
|
GITHUB_STATE = '1337'
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class ProviderName(
|
|
13
|
+
class ProviderName(StrEnum):
|
|
14
14
|
ACTIVE_DIRECTORY = 'active_directory'
|
|
15
15
|
AZURE_DEVOPS = 'azure_devops'
|
|
16
16
|
BITBUCKET = 'bitbucket'
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
MAGE_OPERATION_HISTORY_DIRECTORY_DEFAULT = '.operation_history'
|
|
4
4
|
MAGE_OPERATION_HISTORY_DIRECTORY_ENVIRONMENT_VARIABLE_NAME = 'MAGE_OPERATION_HISTORY_DIRECTORY'
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
class ResourceType(
|
|
7
|
+
class ResourceType(StrEnum):
|
|
8
8
|
PIPELINE = 'pipeline'
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
|
-
|
|
3
1
|
from mage_ai.api.operations.constants import OperationType
|
|
4
2
|
from mage_ai.data_preparation.models.constants import BlockType, PipelineType
|
|
3
|
+
from mage_ai.shared.enum import IntEnum, StrEnum
|
|
5
4
|
|
|
6
5
|
|
|
7
|
-
class EntityName(
|
|
6
|
+
class EntityName(StrEnum):
|
|
8
7
|
ALL = 'ALL'
|
|
9
8
|
ALL_EXCEPT_RESERVED = 'ALL_EXCEPT_RESERVED'
|
|
10
9
|
AutocompleteItem = 'AutocompleteItem'
|
|
@@ -109,7 +108,7 @@ RESERVED_ENTITY_NAMES = [
|
|
|
109
108
|
]
|
|
110
109
|
|
|
111
110
|
|
|
112
|
-
class BaseEntityType(
|
|
111
|
+
class BaseEntityType(StrEnum):
|
|
113
112
|
pass
|
|
114
113
|
|
|
115
114
|
|
|
@@ -137,7 +136,7 @@ class PipelineEntityType(BaseEntityType):
|
|
|
137
136
|
STREAMING = PipelineType.STREAMING.value
|
|
138
137
|
|
|
139
138
|
|
|
140
|
-
class PermissionAccess(
|
|
139
|
+
class PermissionAccess(IntEnum):
|
|
141
140
|
OWNER = 1
|
|
142
141
|
ADMIN = 2
|
|
143
142
|
# Editor: list, detail, create, update, delete
|
|
@@ -177,7 +176,7 @@ class PermissionAccess(int, Enum):
|
|
|
177
176
|
DISABLE_UNLESS_CONDITIONS = 1073741824
|
|
178
177
|
|
|
179
178
|
|
|
180
|
-
class PermissionCondition(
|
|
179
|
+
class PermissionCondition(StrEnum):
|
|
181
180
|
HAS_NOTEBOOK_EDIT_ACCESS = 'HAS_NOTEBOOK_EDIT_ACCESS'
|
|
182
181
|
HAS_PIPELINE_EDIT_ACCESS = 'HAS_PIPELINE_EDIT_ACCESS'
|
|
183
182
|
USER_OWNS_ENTITY = 'USER_OWNS_ENTITY'
|
mage_ai/cache/constants.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
CACHE_KEY_BLOCKS_TO_PIPELINE_MAPPING = 'blocks_to_pipeline_mapping'
|
|
4
4
|
CACHE_KEY_BLOCK_ACTION_OBJECTS_MAPPING = 'block_action_objects_mapping'
|
|
@@ -11,5 +11,5 @@ MAGE_CACHE_DIRECTORY_DEFAULT = '.cache'
|
|
|
11
11
|
MAGE_CACHE_DIRECTORY_ENVIRONMENT_VARIABLE_NAME = 'MAGE_CACHE_DIRECTORY'
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
class CacheItemType(
|
|
14
|
+
class CacheItemType(StrEnum):
|
|
15
15
|
DBT = 'dbt'
|
mage_ai/cache/dbt/constants.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
|
-
|
|
3
1
|
from mage_ai.data_preparation.models.constants import (
|
|
4
2
|
BLOCK_TYPE_DIRECTORY_NAME,
|
|
5
3
|
PIPELINES_FOLDER,
|
|
6
4
|
BlockType,
|
|
7
5
|
)
|
|
6
|
+
from mage_ai.shared.enum import StrEnum
|
|
8
7
|
from mage_ai.shared.hash import merge_dict
|
|
9
8
|
|
|
10
9
|
IGNORE_DIRECTORY_NAMES = merge_dict(
|
|
@@ -19,7 +18,7 @@ PROJECT_FILENAME = 'dbt_project.yml'
|
|
|
19
18
|
PROJECT_FILENAMES = [PROJECT_FILENAME, 'dbt_project.yaml']
|
|
20
19
|
|
|
21
20
|
|
|
22
|
-
class FileType(
|
|
21
|
+
class FileType(StrEnum):
|
|
23
22
|
MODEL = 'model'
|
|
24
23
|
PROFILES = 'profiles'
|
|
25
24
|
PROJECT = 'project'
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
# ECS environment variables
|
|
4
4
|
ECS_CLUSTER_NAME = 'ECS_CLUSTER_NAME'
|
|
@@ -29,7 +29,7 @@ GCP_BACKEND_CONFIG_ANNOTATION = 'cloud.google.com/backend-config'
|
|
|
29
29
|
NODE_PORT_SERVICE_TYPE = 'NodePort'
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
class ClusterType(
|
|
32
|
+
class ClusterType(StrEnum):
|
|
33
33
|
EMR = 'emr'
|
|
34
34
|
ECS = 'ecs'
|
|
35
35
|
CLOUD_RUN = 'cloud_run'
|
|
@@ -43,7 +43,7 @@ from mage_ai.services.k8s.constants import (
|
|
|
43
43
|
)
|
|
44
44
|
from mage_ai.settings import MAGE_SETTINGS_ENVIRONMENT_VARIABLES
|
|
45
45
|
from mage_ai.shared.array import find
|
|
46
|
-
from mage_ai.shared.hash import safe_dig
|
|
46
|
+
from mage_ai.shared.hash import merge_dict, safe_dig
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
class WorkloadManager:
|
|
@@ -391,6 +391,7 @@ class WorkloadManager:
|
|
|
391
391
|
)
|
|
392
392
|
|
|
393
393
|
pod_spec = self.pod_config.spec.to_dict() if self.pod_config else dict()
|
|
394
|
+
pod_labels = self.pod_config.metadata.labels or dict()
|
|
394
395
|
stateful_set_template_spec = dict(
|
|
395
396
|
imagePullSecrets=pod_spec.get('image_pull_secrets'),
|
|
396
397
|
initContainers=init_containers,
|
|
@@ -410,7 +411,7 @@ class WorkloadManager:
|
|
|
410
411
|
'replicas': 1,
|
|
411
412
|
'minReadySeconds': 10,
|
|
412
413
|
'template': {
|
|
413
|
-
'metadata': {'labels': {'app': name}},
|
|
414
|
+
'metadata': merge_dict(pod_labels, {'labels': {'app': name}}),
|
|
414
415
|
'spec': stateful_set_template_spec,
|
|
415
416
|
},
|
|
416
417
|
'volumeClaimTemplates': [
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
SETTINGS_FILENAME = '.command_center.yaml'
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
class ItemTagEnum(
|
|
6
|
+
class ItemTagEnum(StrEnum):
|
|
7
7
|
PINNED = 'pinned'
|
|
8
8
|
RECENT = 'recent'
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
class ItemType(
|
|
11
|
+
class ItemType(StrEnum):
|
|
12
12
|
ACTION = 'action' # Cast spell
|
|
13
13
|
CREATE = 'create' # Conjure
|
|
14
14
|
DELETE = 'delete'
|
|
@@ -22,7 +22,7 @@ class ItemType(str, Enum):
|
|
|
22
22
|
UPDATE = 'update'
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
class ObjectType(
|
|
25
|
+
class ObjectType(StrEnum):
|
|
26
26
|
APPLICATION = 'application'
|
|
27
27
|
APPLICATION_EXPANSION = 'application_expansion'
|
|
28
28
|
AUTHENTICATION = 'authentication'
|
|
@@ -42,11 +42,11 @@ class ObjectType(str, Enum):
|
|
|
42
42
|
VERSION_CONTROL_FILE = 'version_control_file'
|
|
43
43
|
|
|
44
44
|
|
|
45
|
-
class ModeType(
|
|
45
|
+
class ModeType(StrEnum):
|
|
46
46
|
VERSION_CONTROL = 'version_control'
|
|
47
47
|
|
|
48
48
|
|
|
49
|
-
class FileExtension(
|
|
49
|
+
class FileExtension(StrEnum):
|
|
50
50
|
CSV = 'csv'
|
|
51
51
|
JSON = 'json'
|
|
52
52
|
MD = 'md'
|
|
@@ -59,7 +59,7 @@ class FileExtension(str, Enum):
|
|
|
59
59
|
YML = 'yml'
|
|
60
60
|
|
|
61
61
|
|
|
62
|
-
class ButtonActionType(
|
|
62
|
+
class ButtonActionType(StrEnum):
|
|
63
63
|
ADD_APPLICATION = 'add_application'
|
|
64
64
|
CLOSE_APPLICATION = 'close_application' # Go back out of the current application.
|
|
65
65
|
CLOSE_COMMAND_CENTER = 'close_command_center'
|
|
@@ -70,7 +70,7 @@ class ButtonActionType(str, Enum):
|
|
|
70
70
|
SELECT_ITEM_FROM_REQUEST = 'select_item_from_request'
|
|
71
71
|
|
|
72
72
|
|
|
73
|
-
class InteractionType(
|
|
73
|
+
class InteractionType(StrEnum):
|
|
74
74
|
CLICK = 'click'
|
|
75
75
|
CLOSE_APPLICATION = ButtonActionType.CLOSE_APPLICATION.value
|
|
76
76
|
CLOSE_COMMAND_CENTER = ButtonActionType.CLOSE_COMMAND_CENTER.value
|
|
@@ -81,7 +81,7 @@ class InteractionType(str, Enum):
|
|
|
81
81
|
SELECT_ITEM = 'select_item'
|
|
82
82
|
|
|
83
83
|
|
|
84
|
-
class ApplicationType(
|
|
84
|
+
class ApplicationType(StrEnum):
|
|
85
85
|
DETAIL = 'detail'
|
|
86
86
|
DETAIL_LIST = 'detail_list'
|
|
87
87
|
EXPANSION = 'expansion'
|
|
@@ -89,22 +89,22 @@ class ApplicationType(str, Enum):
|
|
|
89
89
|
LIST = 'list'
|
|
90
90
|
|
|
91
91
|
|
|
92
|
-
class ValidationType(
|
|
92
|
+
class ValidationType(StrEnum):
|
|
93
93
|
CONFIRMATION = 'confirmation'
|
|
94
94
|
CUSTOM_VALIDATION_PARSERS = 'custom_validation_parsers'
|
|
95
95
|
|
|
96
96
|
|
|
97
|
-
class RenderLocationType(
|
|
97
|
+
class RenderLocationType(StrEnum):
|
|
98
98
|
ITEMS_CONTAINER_AFTER = 'items_container_after'
|
|
99
99
|
|
|
100
100
|
|
|
101
|
-
class ApplicationExpansionUUID(
|
|
101
|
+
class ApplicationExpansionUUID(StrEnum):
|
|
102
102
|
ArcaneLibrary = 'ArcaneLibrary'
|
|
103
103
|
PortalTerminal = 'PortalTerminal'
|
|
104
104
|
VersionControlFileDiffs = 'VersionControlFileDiffs'
|
|
105
105
|
|
|
106
106
|
|
|
107
|
-
class ApplicationExpansionStatus(
|
|
107
|
+
class ApplicationExpansionStatus(StrEnum):
|
|
108
108
|
ACTIVE = 'ACTIVE'
|
|
109
109
|
CLOSED = 'CLOSED'
|
|
110
110
|
INACTIVE = 'INACTIVE'
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
5
|
+
|
|
1
6
|
from mage_ai.data_cleaner.column_types.column_type_detector import find_syntax_errors
|
|
2
7
|
from mage_ai.data_cleaner.column_types.constants import NUMBER_TYPES, ColumnType
|
|
3
8
|
from mage_ai.data_cleaner.estimators.outlier_removal import OutlierRemover
|
|
@@ -15,11 +20,10 @@ from mage_ai.data_cleaner.transformer_actions.helpers import (
|
|
|
15
20
|
get_time_window_str,
|
|
16
21
|
)
|
|
17
22
|
from mage_ai.data_cleaner.transformer_actions.udf.base import execute_udf
|
|
18
|
-
from mage_ai.data_cleaner.transformer_actions.utils import
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
+
from mage_ai.data_cleaner.transformer_actions.utils import (
|
|
24
|
+
clean_column_name,
|
|
25
|
+
generate_string_cols,
|
|
26
|
+
)
|
|
23
27
|
|
|
24
28
|
logger = logging.getLogger(__name__)
|
|
25
29
|
|
|
@@ -115,7 +119,8 @@ def impute(df, action, **kwargs):
|
|
|
115
119
|
mode = df[column].mode().iloc[0]
|
|
116
120
|
if dtype == ColumnType.LIST:
|
|
117
121
|
df[column] = df[column].apply(
|
|
118
|
-
lambda element: element if element not in [
|
|
122
|
+
lambda element, mode=mode: element if element not in [
|
|
123
|
+
None, np.nan] else mode
|
|
119
124
|
)
|
|
120
125
|
else:
|
|
121
126
|
df[columns] = df[columns].fillna(mode)
|
|
@@ -180,7 +185,6 @@ def reformat(df, action, **kwargs):
|
|
|
180
185
|
columns = action['action_arguments']
|
|
181
186
|
options = action['action_options']
|
|
182
187
|
reformat_action = options['reformat']
|
|
183
|
-
df.loc[:, columns] = df[columns].replace(r'^\s*$', np.nan, regex=True)
|
|
184
188
|
|
|
185
189
|
if reformat_action == 'caps_standardization':
|
|
186
190
|
capitalization = options['capitalization']
|
|
@@ -189,6 +193,8 @@ def reformat(df, action, **kwargs):
|
|
|
189
193
|
df.loc[:, column] = df[columns][column].str.upper()
|
|
190
194
|
else:
|
|
191
195
|
df.loc[:, column] = df[columns][column].str.lower()
|
|
196
|
+
# Convert empty strings to NaN for this action
|
|
197
|
+
df.loc[:, columns] = df[columns].replace(r'^\s*$', np.nan, regex=True)
|
|
192
198
|
elif reformat_action == 'currency_to_num':
|
|
193
199
|
for column in generate_string_cols(df, columns):
|
|
194
200
|
clean_col = df[column].replace(CURRENCY_SYMBOLS, '', regex=True)
|
|
@@ -215,6 +221,12 @@ def reformat(df, action, **kwargs):
|
|
|
215
221
|
df.loc[:, column] = pd.to_datetime(
|
|
216
222
|
clean_col, infer_datetime_format=True, errors='coerce'
|
|
217
223
|
)
|
|
224
|
+
elif reformat_action == 'trim':
|
|
225
|
+
for column in columns:
|
|
226
|
+
df[column] = df[column].str.strip()
|
|
227
|
+
else:
|
|
228
|
+
# Apply NaN replacement only for other actions
|
|
229
|
+
df.loc[:, columns] = df[columns].replace(r'^\s*$', np.nan, regex=True)
|
|
218
230
|
|
|
219
231
|
return df
|
|
220
232
|
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
from mage_ai.data_cleaner.column_types.constants import ColumnType
|
|
2
|
-
from enum import Enum
|
|
3
|
-
import pandas as pd
|
|
4
|
-
import numpy as np
|
|
5
1
|
import re
|
|
6
2
|
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
5
|
+
|
|
6
|
+
from mage_ai.data_cleaner.column_types.constants import ColumnType
|
|
7
|
+
from mage_ai.shared.enum import StrEnum
|
|
7
8
|
|
|
8
9
|
CONSTANT_IMPUTATION_DEFAULTS = {
|
|
9
10
|
ColumnType.CATEGORY: 'missing',
|
|
@@ -34,7 +35,7 @@ INVALID_VALUE_PLACEHOLDERS = {
|
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
|
|
37
|
-
class ActionType(
|
|
38
|
+
class ActionType(StrEnum):
|
|
38
39
|
ADD = 'add'
|
|
39
40
|
AVERAGE = 'average'
|
|
40
41
|
CLEAN_COLUMN_NAME = 'clean_column_name'
|
|
@@ -73,18 +74,18 @@ class ActionType(str, Enum):
|
|
|
73
74
|
STANDARDIZE = 'standardize'
|
|
74
75
|
|
|
75
76
|
|
|
76
|
-
class Axis(
|
|
77
|
+
class Axis(StrEnum):
|
|
77
78
|
COLUMN = 'column'
|
|
78
79
|
ROW = 'row'
|
|
79
80
|
|
|
80
81
|
|
|
81
|
-
class VariableType(
|
|
82
|
+
class VariableType(StrEnum):
|
|
82
83
|
FEATURE = 'feature'
|
|
83
84
|
FEATURE_SET = 'feature_set'
|
|
84
85
|
FEATURE_SET_VERSION = 'feature_set_version'
|
|
85
86
|
|
|
86
87
|
|
|
87
|
-
class Operator(
|
|
88
|
+
class Operator(StrEnum):
|
|
88
89
|
CONTAINS = 'contains'
|
|
89
90
|
NOT_CONTAINS = 'not contains'
|
|
90
91
|
EQUALS = '=='
|
|
@@ -95,7 +96,7 @@ class Operator(str, Enum):
|
|
|
95
96
|
LESS_THAN_OR_EQUALS = '<='
|
|
96
97
|
|
|
97
98
|
|
|
98
|
-
class ImputationStrategy(
|
|
99
|
+
class ImputationStrategy(StrEnum):
|
|
99
100
|
AVERAGE = 'average'
|
|
100
101
|
COLUMN = 'column'
|
|
101
102
|
CONSTANT = 'constant'
|
|
@@ -17,6 +17,7 @@ SQL_SOURCES = [
|
|
|
17
17
|
SQL_SOURCES_MAPPING = index_by(get_uuid, SQL_SOURCES)
|
|
18
18
|
|
|
19
19
|
SOURCES = sorted([
|
|
20
|
+
dict(name='Airtable'),
|
|
20
21
|
dict(name='Amazon S3'),
|
|
21
22
|
dict(name='Amplitude'),
|
|
22
23
|
dict(name='Api'),
|
|
@@ -33,6 +34,7 @@ SOURCES = sorted([
|
|
|
33
34
|
dict(name='GitHub'),
|
|
34
35
|
dict(name='Google Ads'),
|
|
35
36
|
dict(name='Google Analytics'),
|
|
37
|
+
dict(name='Google Cloud Storage'),
|
|
36
38
|
dict(name='Google Search Console'),
|
|
37
39
|
dict(name='Google Sheets'),
|
|
38
40
|
dict(name='HubSpot'),
|
|
@@ -172,7 +172,10 @@ def get_access_token(git_config, repo_path: str) -> str:
|
|
|
172
172
|
def build_authenticated_remote_url(remote_url: str, username: str, token: str) -> str:
|
|
173
173
|
# https://[username]:[token]@github.com/[remote_url]
|
|
174
174
|
url = urlsplit(remote_url)
|
|
175
|
-
|
|
175
|
+
if remote_url.startswith('https://bitbucket.org/'):
|
|
176
|
+
url = url._replace(netloc=f'x-token-auth:{token}@{url.netloc}')
|
|
177
|
+
else:
|
|
178
|
+
url = url._replace(netloc=f'{username}:{token}@{url.netloc}')
|
|
176
179
|
return urlunsplit(url)
|
|
177
180
|
|
|
178
181
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
from dataclasses import dataclass, field
|
|
2
|
-
from enum import Enum
|
|
3
2
|
from typing import Dict
|
|
4
3
|
|
|
5
4
|
from mage_ai.shared.config import BaseConfig
|
|
5
|
+
from mage_ai.shared.enum import StrEnum
|
|
6
6
|
from mage_ai.shared.logger import LoggingLevel
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
class LoggerType(
|
|
9
|
+
class LoggerType(StrEnum):
|
|
10
10
|
DEFAULT = 'file'
|
|
11
11
|
S3 = 's3'
|
|
12
12
|
GCS = 'gcs'
|
|
@@ -61,9 +61,9 @@ class DictLogger():
|
|
|
61
61
|
)
|
|
62
62
|
|
|
63
63
|
if error:
|
|
64
|
-
data['error'] = traceback.format_exc()
|
|
65
|
-
data['error_stack'] = traceback.format_stack()
|
|
66
|
-
data['error_stacktrace'] = str(error)
|
|
64
|
+
data['error'] = traceback.format_exc()
|
|
65
|
+
data['error_stack'] = traceback.format_stack()
|
|
66
|
+
data['error_stacktrace'] = str(error)
|
|
67
67
|
|
|
68
68
|
msg = simplejson.dumps(
|
|
69
69
|
merge_dict(self.logging_tags or dict(), merge_dict(kwargs, data)),
|
|
@@ -878,7 +878,7 @@ class Block(
|
|
|
878
878
|
relative_path: bool = False,
|
|
879
879
|
) -> str:
|
|
880
880
|
file_extension = BLOCK_LANGUAGE_TO_FILE_EXTENSION[language]
|
|
881
|
-
block_directory =
|
|
881
|
+
block_directory = block_type + 's' if block_type != BlockType.CUSTOM else block_type
|
|
882
882
|
|
|
883
883
|
parts = []
|
|
884
884
|
if not relative_path:
|
|
@@ -1006,6 +1006,12 @@ class Block(
|
|
|
1006
1006
|
|
|
1007
1007
|
return extract_full_table_name(self.content)
|
|
1008
1008
|
|
|
1009
|
+
def get_typed_content(
|
|
1010
|
+
self,
|
|
1011
|
+
content: str,
|
|
1012
|
+
) -> str:
|
|
1013
|
+
return '@' + self.type + '\n' + content
|
|
1014
|
+
|
|
1009
1015
|
@classmethod
|
|
1010
1016
|
def after_create(cls, block: 'Block', **kwargs) -> None:
|
|
1011
1017
|
widget = kwargs.get('widget')
|
|
@@ -1193,7 +1199,7 @@ class Block(
|
|
|
1193
1199
|
|
|
1194
1200
|
@classmethod
|
|
1195
1201
|
def file_directory_name(self, block_type: BlockType) -> str:
|
|
1196
|
-
return
|
|
1202
|
+
return block_type + 's' if block_type != BlockType.CUSTOM else block_type
|
|
1197
1203
|
|
|
1198
1204
|
@classmethod
|
|
1199
1205
|
def block_type_from_path(
|
|
@@ -1212,7 +1218,7 @@ class Block(
|
|
|
1212
1218
|
for block_type in BlockType:
|
|
1213
1219
|
if BlockType.CUSTOM == block_type and dir_name == block_type:
|
|
1214
1220
|
return BlockType.CUSTOM
|
|
1215
|
-
elif dir_name ==
|
|
1221
|
+
elif dir_name == block_type + 's':
|
|
1216
1222
|
return block_type
|
|
1217
1223
|
|
|
1218
1224
|
@classmethod
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
BLOCK_CATALOG_FILENAME = 'catalog.json'
|
|
4
4
|
REPLICATION_METHOD_INCREMENTAL = 'INCREMENTAL'
|
|
@@ -81,6 +81,6 @@ MAX_QUERY_STRING_SIZE = 10 * MB_1
|
|
|
81
81
|
VARIABLE_BOOKMARK_VALUES_KEY = '__bookmark_values__'
|
|
82
82
|
|
|
83
83
|
|
|
84
|
-
class IngestMode(
|
|
84
|
+
class IngestMode(StrEnum):
|
|
85
85
|
DISK = 'disk'
|
|
86
86
|
MEMORY = 'memory'
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from mage_ai.shared.enum import StrEnum
|
|
2
2
|
|
|
3
3
|
DBT_DIRECTORY_NAME = 'dbt'
|
|
4
4
|
|
|
@@ -13,12 +13,12 @@ SKIP_LIMIT_ADAPTER_NAMES = [
|
|
|
13
13
|
]
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
class Flag(
|
|
16
|
+
class Flag(StrEnum):
|
|
17
17
|
PROFILES_DIR = 'profiles-dir'
|
|
18
18
|
PROJECT_DIR = 'project-dir'
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
class LogLevel(
|
|
21
|
+
class LogLevel(StrEnum):
|
|
22
22
|
DEBUG = 'debug'
|
|
23
23
|
INFO = 'info'
|
|
24
24
|
WARN = 'warn'
|