mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/utils/async_http.py

@@ -24,9 +24,15 @@ from aiohttp_retry import ExponentialRetry, RequestParams, RetryClient, RetryOpt
 from aiohttp_retry.client import _RequestContext
 
 from mlrun.config import config
-from mlrun.errors import err_to_str, raise_for_status
+from mlrun.errors import err_to_str
+from mlrun.errors import raise_for_status as ml_raise_for_status
+from mlrun.utils.helpers import logger as mlrun_logger
 
-from .helpers import logger as mlrun_logger
+DEFAULT_BLACKLISTED_METHODS = [
+    "POST",
+    "PUT",
+    "PATCH",
+]
 
 
 class AsyncClientWithRetry(RetryClient):
@@ -42,20 +48,10 @@ class AsyncClientWithRetry(RetryClient):
         retry_on_exception: bool = True,
         raise_for_status: bool = True,
         blacklisted_methods: typing.Optional[list[str]] = None,
-        logger: logging.Logger = None,
+        logger: Optional[logging.Logger] = None,
         *args,
         **kwargs,
     ):
-        # do not retry on PUT / PATCH as they might have side effects (not truly idempotent)
-        blacklisted_methods = (
-            blacklisted_methods
-            if blacklisted_methods is not None
-            else [
-                "POST",
-                "PUT",
-                "PATCH",
-            ]
-        )
         super().__init__(
             *args,
             retry_options=ExponentialRetryOverride(
@@ -72,9 +68,13 @@ class AsyncClientWithRetry(RetryClient):
             **kwargs,
         )
 
+    @property
+    def retry_options(self) -> "ExponentialRetryOverride":
+        return typing.cast(ExponentialRetryOverride, self._retry_options)
+
     def methods_blacklist_update_required(self, new_blacklist: str):
         self._retry_options: ExponentialRetryOverride
-        return set(self._retry_options.blacklisted_methods).difference(
+        return set(self.retry_options.blacklisted_methods).difference(
             set(new_blacklist)
         )
 
@@ -110,12 +110,13 @@ class ExponentialRetryOverride(ExponentialRetry):
         # aiohttp exceptions that can be raised during connection establishment
         aiohttp.ClientConnectionError,
         aiohttp.ServerDisconnectedError,
+        asyncio.exceptions.TimeoutError,
     ]
 
     def __init__(
         self,
-        retry_on_exception: bool,
-        blacklisted_methods: list[str],
+        retry_on_exception: typing.Optional[bool] = True,
+        blacklisted_methods: typing.Optional[list[str]] = None,
         *args,
         **kwargs,
     ):
@@ -123,7 +124,12 @@ class ExponentialRetryOverride(ExponentialRetry):
         self.retry_on_exception = retry_on_exception
 
         # methods that should not be retried
-        self.blacklisted_methods = blacklisted_methods
+        # do not retry on PUT / PATCH as they might have side effects (not truly idempotent)
+        self.blacklisted_methods = (
+            blacklisted_methods
+            if blacklisted_methods is not None
+            else DEFAULT_BLACKLISTED_METHODS
+        )
 
         # default exceptions that should be retried on (when retry_on_exception is True)
         if "exceptions" not in kwargs:
@@ -136,6 +142,10 @@ class _CustomRequestContext(_RequestContext):
     Extends by adding retry logic on both error statuses and certain exceptions.
     """
 
+    @property
+    def retry_options(self) -> ExponentialRetryOverride:
+        return typing.cast(ExponentialRetryOverride, self._retry_options)
+
     async def _do_request(self) -> aiohttp.ClientResponse:
         current_attempt = 0
         while True:
@@ -186,9 +196,9 @@ class _CustomRequestContext(_RequestContext):
                     return response
 
                 last_attempt = current_attempt == self._retry_options.attempts
-                if self._is_status_code_ok(response.status) or last_attempt:
+                if not self._is_status_retryable(response.status) or last_attempt:
                     if self._raise_for_status:
-                        raise_for_status(response)
+                        ml_raise_for_status(response)
 
                     self._response = response
                     return response
@@ -204,7 +214,7 @@ class _CustomRequestContext(_RequestContext):
                     )
 
             except Exception as exc:
-                if not self._retry_options.retry_on_exception:
+                if not self.retry_options.retry_on_exception:
                     self._logger.warning(
                         "Request failed, retry on exception is disabled",
                         method=params.method,
@@ -250,7 +260,10 @@ class _CustomRequestContext(_RequestContext):
             await asyncio.sleep(retry_wait)
 
     def _is_method_retryable(self, method: str):
-        return method not in self._retry_options.blacklisted_methods
+        return method not in self.retry_options.blacklisted_methods
+
+    def _is_status_retryable(self, status: int):
+        return status in self._retry_options.statuses
 
     async def _check_response_callback(
         self,
@@ -290,7 +303,7 @@ class _CustomRequestContext(_RequestContext):
                 if isinstance(exc.os_error, exc_type):
                     return
         if exc.__cause__:
-            # If the cause exception is retriable, return, otherwise, raise the original exception
+            # If the cause exception is retryable, return, otherwise, raise the original exception
             try:
                 self.verify_exception_type(exc.__cause__)
             except Exception:

mlrun/utils/clones.py

@@ -122,7 +122,7 @@ def add_credentials_git_remote_url(url: str, secrets=None) -> tuple[str, bool]:
         username, password = get_git_username_password_from_token(token)
 
     if username:
-        return f"https://{username}:{password}@{url_obj.hostname}{url_obj.path}", True
+        return f"https://{username}:{password}@{url_obj.netloc}{url_obj.path}", True
     return url, False
 
 
@@ -165,14 +165,17 @@ def clone_git(url: str, context: str, secrets=None, clone: bool = True):
 
     branch = None
     tag = None
+    commit = None
     if url_obj.fragment:
         refs = url_obj.fragment
         if refs.startswith("refs/heads/"):
             branch = refs.replace("refs/heads/", "")
         elif refs.startswith("refs/tags/"):
             tag = refs.replace("refs/tags/", "")
+        elif refs.startswith("refs/commits/"):
+            commit = refs.replace("refs/commits/", "")
         else:
-            url = url.replace("#" + refs, f"#refs/heads/{refs}")
+            url = url.replace(f"#{refs}", f"#refs/heads/{refs}")
             branch = refs
 
     # when using the CLI and clone path was not enriched, username/password input will be requested via shell
@@ -182,8 +185,8 @@ def clone_git(url: str, context: str, secrets=None, clone: bool = True):
         # override enriched clone path for security reasons
         repo.remotes[0].set_url(clone_path, final_clone_path)
 
-    if tag:
-        repo.git.checkout(tag)
+    if tag_or_commit := tag or commit:
+        repo.git.checkout(tag_or_commit)
 
     return url, repo