mage-ai 0.9.70__py3-none-any.whl → 0.9.71__py3-none-any.whl

mage_ai/data_preparation/executors/streaming_pipeline_executor.py

@@ -96,7 +96,7 @@ class StreamingPipelineExecutor(PipelineExecutor):
         try:
             if retry_config is None:
                 retry_config = self.pipeline.retry_config or dict()
-            infinite_retries = False if retry_config else False
+            infinite_retries = False if retry_config else True
 
             if type(retry_config) is not RetryConfig:
                 retry_config = RetryConfig.load(config=retry_config)

mage_ai/data_preparation/git/__init__.py

@@ -42,22 +42,32 @@ class Git:
     def __init__(
         self,
         auth_type: AuthType = None,
+        context_data: Dict = None,
         git_config: GitConfig = None,
+        repo_path: str = None,
         setup_repo: bool = False,
+        user: User = None,
     ) -> None:
         import git
         import git.exc
 
         self.auth_type = auth_type if auth_type else AuthType.SSH
+        self.context_data = context_data if context_data is not None else dict()
         self.git_config = git_config
+        self.user = user
         self.origin = None
         self.remote_repo_link = None
         self.repo = None
 
-        self.repo_path = os.getcwd()
+        self.project_repo_path = repo_path or get_repo_path(user=user)
 
+        self.repo_path = os.getcwd()
         if project_platform_activated():
-            git_dict = git_settings()
+            git_dict = git_settings(
+                context_data=self.context_data,
+                repo_path=self.project_repo_path,
+                user=user,
+            )
             if git_dict and git_dict.get('path'):
                 self.repo_path = git_dict.get('path')
 
@@ -149,12 +159,19 @@ class Git:
     @classmethod
     def get_manager(
         self,
-        user: User = None,
-        setup_repo: bool = False,
         auth_type: str = None,
         config_overwrite: Dict = None,
+        context_data: Dict = None,
+        preferences=None,
+        repo_path: str = None,
+        setup_repo: bool = False,
+        user: User = None,
     ) -> 'Git':
-        preferences = get_preferences(user=user)
+        if preferences is None:
+            preferences = get_preferences(
+                repo_path=repo_path,
+                user=user,
+            )
         git_config = None
         config = preferences.sync_config if preferences and preferences.sync_config else {}
 
@@ -168,8 +185,11 @@ class Git:
 
         return Git(
             auth_type=auth_type,
+            context_data=context_data,
             git_config=git_config,
+            repo_path=repo_path,
             setup_repo=setup_repo,
+            user=user,
         )
 
     @property
@@ -784,10 +804,10 @@ class Git:
                 pass
 
     def __create_ssh_keys(self, overwrite: bool = False) -> str:
-        return create_ssh_keys(self.git_config, get_repo_path(), overwrite=overwrite)
+        return create_ssh_keys(self.git_config, self.project_repo_path, overwrite=overwrite)
 
     def __add_host_to_known_hosts(self):
         return add_host_to_known_hosts_util(self.git_config.remote_repo_link)
 
     def get_access_token(self) -> str:
-        return get_access_token(self.git_config)
+        return get_access_token(self.git_config, self.project_repo_path)

mage_ai/data_preparation/git/api.py

@@ -37,6 +37,7 @@ def switch_branch(
     token: str,
     config_overwrite: Dict = None,
     user: User = None,
+    **kwargs,
 ):
     from mage_ai.data_preparation.git import Git
     provider = get_provider_from_remote_url(remote_url)
@@ -67,6 +68,7 @@ def fetch(
     token: str,
     user: User = None,
     config_overwrite: Dict = None,
+    **kwargs,
 ):
     """
     Returns:
@@ -89,7 +91,7 @@ def fetch(
     remote.set_url(url)
 
     try:
-        remote.fetch(progress=custom_progress)
+        remote.fetch(progress=custom_progress, **kwargs)
     except Exception as err:
         raise err
     finally:
@@ -109,6 +111,7 @@ def pull(
     token: str,
     user: User = None,
     config_overwrite: Dict = None,
+    **kwargs,
 ):
     """
     Returns:
@@ -190,6 +193,7 @@ def push(
     token: str,
     user: User = None,
     config_overwrite: Dict = None,
+    **kwargs,
 ):
     """
     Returns:
@@ -218,6 +222,7 @@ def reset_hard(
     token: str,
     user: User = None,
     config_overwrite: Dict = None,
+    **kwargs,
 ) -> None:
     from mage_ai.data_preparation.git import Git
 
@@ -248,6 +253,7 @@ def clone(
     token: str,
     user: User = None,
     config_overwrite: Dict = None,
+    **kwargs,
 ) -> None:
     from mage_ai.data_preparation.git import Git
 

mage_ai/data_preparation/git/utils.py

@@ -24,7 +24,6 @@ from mage_ai.settings.keys import (
     GIT_SSH_PUBLIC_KEY,
     GITLAB_HOST,
 )
-from mage_ai.settings.repo import get_repo_path
 
 
 def get_auth_type_from_url(remote_url: str) -> AuthType:
@@ -72,12 +71,15 @@ def create_ssh_keys(
         if not os.path.exists(public_key_file) or overwrite:
             try:
                 public_key = get_settings_value(GIT_SSH_PUBLIC_KEY)
-                if not public_key or overwrite_with_project_settings:
-                    public_key = get_secret_value(
-                        pubk_secret_name,
-                        repo_name=repo_path,
-                        suppress_warning=True,
-                    )
+                public_key_from_secrets = get_secret_value(
+                    pubk_secret_name,
+                    repo_name=repo_path,
+                    suppress_warning=True,
+                )
+                if public_key_from_secrets and (
+                    not public_key or overwrite_with_project_settings
+                ):
+                    public_key = public_key_from_secrets
                 if public_key:
                     with open(public_key_file, 'w') as f:
                         f.write(base64.b64decode(public_key).decode('utf-8'))
@@ -93,12 +95,15 @@ def create_ssh_keys(
         if not os.path.exists(custom_private_key_file) or overwrite:
             try:
                 private_key = get_settings_value(GIT_SSH_PRIVATE_KEY)
-                if not private_key or overwrite_with_project_settings:
-                    private_key = get_secret_value(
-                        pk_secret_name,
-                        repo_name=repo_path,
-                        suppress_warning=True,
-                    )
+                private_key_from_secrets = get_secret_value(
+                    pk_secret_name,
+                    repo_name=repo_path,
+                    suppress_warning=True,
+                )
+                if private_key_from_secrets and (
+                    not private_key or overwrite_with_project_settings
+                ):
+                    private_key = private_key_from_secrets
                 if private_key:
                     with open(custom_private_key_file, 'w') as f:
                         f.write(base64.b64decode(private_key).decode('utf-8'))
@@ -147,7 +152,7 @@ def add_host_to_known_hosts(remote_repo_link: str):
     return False
 
 
-def get_access_token(git_config, repo_path: str = None) -> str:
+def get_access_token(git_config, repo_path: str) -> str:
     token = get_settings_value(GIT_ACCESS_TOKEN)
     overwrite_with_project_settings = get_bool_value(
         get_settings_value(GIT_OVERWRITE_WITH_PROJECT_SETTINGS)
@@ -155,9 +160,10 @@ def get_access_token(git_config, repo_path: str = None) -> str:
     if git_config and git_config.access_token_secret_name:
         token_from_secrets = get_secret_value(
             git_config.access_token_secret_name,
-            repo_name=repo_path or get_repo_path(),
+            repo_name=repo_path,
+            suppress_warning=True,
         )
-        if not token or overwrite_with_project_settings:
+        if token_from_secrets and (not token or overwrite_with_project_settings):
             token = token_from_secrets
 
     return token

mage_ai/data_preparation/logging/logger_manager.py

@@ -7,11 +7,12 @@ from datetime import datetime
 from typing import Callable, Dict, List
 
 from mage_ai.data_preparation.logging import LoggingConfig
-from mage_ai.data_preparation.models.constants import LOGS_DIR
+from mage_ai.data_preparation.models.constants import LOGS_SUBDIR
 from mage_ai.data_preparation.models.file import File
 from mage_ai.data_preparation.repo_manager import RepoConfig, get_repo_config
 from mage_ai.data_preparation.storage.local_storage import LocalStorage
 from mage_ai.settings.repo import get_repo_path
+from mage_ai.settings.server import LOGS_DIR_PATH
 from mage_ai.shared.array import find
 from mage_ai.shared.dates import str_to_timedelta
 
@@ -175,13 +176,13 @@ class LoggerManager:
             str: The log file path prefix based on pipeline_uuid, logs_dir, partition, and
                  subpartition.
         """
-        logs_dir = self.logs_dir or self.repo_config.variables_dir
+        logs_dir = self.logs_dir or LOGS_DIR_PATH or self.repo_config.variables_dir
 
         return os.path.join(
             logs_dir,
             'pipelines',
             pipeline_uuid or self.pipeline_uuid or 'all_pipelines',
-            LOGS_DIR,
+            LOGS_SUBDIR,
             self.partition or '',
             self.subpartition or '',
         )