mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/utils/helpers.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@ import enum
 import hashlib
 import inspect
 import json
+import os
+import pathlib
 import re
 import sys
 import time
@@ -38,6 +40,7 @@ from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
 import mlrun
+import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.version.version
 from mlrun.errors import err_to_str
@@ -96,8 +99,11 @@ def get_artifact_target(item: dict, project=None):
         tree = item["metadata"].get("tree")
 
     kind = item.get("kind")
-    if kind in ["dataset", "model"] and db_key:
-        return f"{DB_SCHEMA}://{StorePrefix.Artifact}/{project_str}/{db_key}:{tree}"
+    if kind in ["dataset", "model", "artifact"] and db_key:
+        target = f"{DB_SCHEMA}://{StorePrefix.Artifact}/{project_str}/{db_key}"
+        if tree:
+            target = f"{target}:{tree}"
+        return target
 
     return (
         item.get("target_path")
@@ -144,6 +150,7 @@ def verify_field_regex(
     patterns,
     raise_on_failure: bool = True,
     log_message: str = "Field is malformed. Does not match required pattern",
+    mode: mlrun.common.schemas.RegexMatchModes = mlrun.common.schemas.RegexMatchModes.all,
 ) -> bool:
     for pattern in patterns:
         if not re.match(pattern, str(field_value)):
@@ -154,13 +161,52 @@ def verify_field_regex(
                 field_value=field_value,
                 pattern=pattern,
             )
-            if raise_on_failure:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    f"Field '{field_name}' is malformed. Does not match required pattern: {pattern}"
-                )
-            else:
+            if mode == mlrun.common.schemas.RegexMatchModes.all:
+                if raise_on_failure:
+                    raise mlrun.errors.MLRunInvalidArgumentError(
+                        f"Field '{field_name}' is malformed. {field_value} does not match required pattern: {pattern}"
+                    )
                 return False
-    return True
+        elif mode == mlrun.common.schemas.RegexMatchModes.any:
+            return True
+    if mode == mlrun.common.schemas.RegexMatchModes.all:
+        return True
+    elif mode == mlrun.common.schemas.RegexMatchModes.any:
+        if raise_on_failure:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Field '{field_name}' is malformed. {field_value} does not match any of the"
+                f" required patterns: {patterns}"
+            )
+        return False
+
+
+def validate_builder_source(
+    source: str, pull_at_runtime: bool = False, workdir: str = None
+):
+    if pull_at_runtime or not source:
+        return
+
+    if "://" not in source:
+        if not path.isabs(source):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Source '{source}' must be a valid URL or absolute path when 'pull_at_runtime' is False "
+                "set 'source' to a remote URL to clone/copy the source to the base image, "
+                "or set 'pull_at_runtime' to True to pull the source at runtime."
+            )
+
+        else:
+            logger.warn(
+                "Loading local source at build time requires the source to be on the base image, "
+                "in which case it is recommended to use 'workdir' instead",
+                source=source,
+                workdir=workdir,
+            )
+
+    if source.endswith(".zip"):
+        logger.warn(
+            "zip files are not natively extracted by docker, use tar.gz for faster loading during build",
+            source=source,
+        )
 
 
 def validate_tag_name(
@@ -200,6 +246,34 @@ def get_regex_list_as_string(regex_list: List) -> str:
     return "".join(["(?={regex})".format(regex=regex) for regex in regex_list]) + ".*$"
 
 
+def is_file_path_invalid(code_path: str, file_path: str) -> bool:
+    """
+    The function checks if the given file_path is a valid path.
+    If the file_path is a relative path, it is completed by joining it with the code_path.
+    Otherwise, the file_path is used as is.
+    Additionally, it checks if the resulting path exists as a file, unless the file_path is a remote URL.
+    If the file_path has no suffix, it is considered invalid.
+
+    :param code_path: The base directory or code path to search for the file in case of relative file_path
+    :param file_path: The file path to be validated
+    :return: True if the file path is invalid, False otherwise
+    """
+    if not file_path:
+        return True
+
+    if file_path.startswith("./") or (
+        "://" not in file_path and os.path.basename(file_path) == file_path
+    ):
+        abs_path = os.path.join(code_path, file_path.lstrip("./"))
+    else:
+        abs_path = file_path
+
+    return (
+        not (os.path.isfile(abs_path) or "://" in file_path)
+        or not pathlib.Path(file_path).suffix
+    )
+
+
 def tag_name_regex_as_string() -> str:
     return get_regex_list_as_string(mlrun.utils.regex.tag_name)
 
@@ -208,6 +282,23 @@ def is_yaml_path(url):
     return url.endswith(".yaml") or url.endswith(".yml")
 
 
+def remove_image_protocol_prefix(image: str) -> str:
+    if not image:
+        return image
+
+    prefixes = ["https://", "https://"]
+    if any(prefix in image for prefix in prefixes):
+        image = image.removeprefix("https://").removeprefix("http://")
+        logger.warning(
+            "The image has an unexpected protocol prefix ('http://' or 'https://'). "
+            "If you wish to use the default configured registry, no protocol prefix is required "
+            "(note that you can also use '.<image-name>' instead of the full URL where <image-name> is a placeholder). "
+            "Removing protocol prefix from image.",
+            image=image,
+        )
+    return image
+
+
 # Verifying that a field input is of the expected type. If not the method raises a detailed MLRunInvalidArgumentError
 def verify_field_of_type(field_name: str, field_value, expected_type: type):
     if not isinstance(field_value, expected_type):
@@ -954,7 +1045,7 @@ def retry_until_successful(
         f" last_exception: {last_exception},"
         f" function_name: {_function.__name__},"
         f" timeout: {timeout}"
-    )
+    ) from last_exception
 
 
 def get_ui_url(project, uid=None):
@@ -1000,7 +1091,7 @@ def create_class(pkg_class: str):
     return class_
 
 
-def create_function(pkg_func: list):
+def create_function(pkg_func: str):
     """Create a function from a package.module.function string
 
     :param pkg_func:  full function location,
@@ -1014,9 +1105,16 @@ def create_function(pkg_func: list):
     return function_
 
 
-def get_caller_globals(level=2):
+def get_caller_globals():
+    """Returns a dictionary containing the first non-mlrun caller function's namespace."""
     try:
-        return inspect.stack()[level][0].f_globals
+        stack = inspect.stack()
+        # If an API function called this function directly, the first non-mlrun caller will be 2 levels up the stack.
+        # Otherwise, we keep going up the stack until we find it.
+        for level in range(2, len(stack)):
+            namespace = stack[level][0].f_globals
+            if not namespace["__name__"].startswith("mlrun."):
+                return namespace
     except Exception:
         return None
 
@@ -1209,6 +1307,43 @@ def is_legacy_artifact(artifact):
         return not hasattr(artifact, "metadata")
 
 
+def format_run(run: dict, with_project=False) -> dict:
+    fields = [
+        "id",
+        "name",
+        "status",
+        "error",
+        "created_at",
+        "scheduled_at",
+        "finished_at",
+        "description",
+    ]
+
+    if with_project:
+        fields.append("project")
+
+    # create a run object that contains all fields,
+    run = {
+        key: str(value) if value is not None else value
+        for key, value in run.items()
+        if key in fields
+    }
+
+    # if the time_keys values is from 1970, this indicates that the field has not yet been specified yet,
+    # and we want to return a None value instead
+    time_keys = ["scheduled_at", "finished_at", "created_at"]
+
+    for key, value in run.items():
+        if (
+            key in time_keys
+            and isinstance(value, (str, datetime))
+            and parser.parse(str(value)).year == 1970
+        ):
+            run[key] = None
+
+    return run
+
+
 def get_in_artifact(artifact: dict, key, default=None, raise_on_missing=False):
     """artifact can be dict or Artifact object"""
     if is_legacy_artifact(artifact):
@@ -1245,6 +1380,18 @@ def is_relative_path(path):
     return not (path.startswith("/") or ":\\" in path or "://" in path)
 
 
+def is_running_in_jupyter_notebook() -> bool:
+    """
+    Check if the code is running inside a Jupyter Notebook.
+    :return: True if running inside a Jupyter Notebook, False otherwise.
+    """
+    import IPython
+
+    ipy = IPython.get_ipython()
+    # if its IPython terminal, it isn't a Jupyter ipython
+    return ipy and "Terminal" not in str(type(ipy))
+
+
 def as_number(field_name, field_value):
     if isinstance(field_value, str) and not field_value.isnumeric():
         raise ValueError(f"{field_name} must be numeric (str/int types)")
@@ -1295,3 +1442,31 @@ def ensure_git_branch(url: str, repo: git.Repo) -> str:
     if not branch and not reference:
         url = f"{url}#refs/heads/{repo.active_branch}"
     return url
+
+
+def is_file_path(filepath):
+    root, ext = os.path.splitext(filepath)
+    return os.path.isfile(filepath) and ext
+
+
+class DeprecationHelper(object):
+    """A helper class to deprecate old schemas"""
+
+    def __init__(self, new_target, version="1.4.0"):
+        self._new_target = new_target
+        self._version = version
+
+    def _warn(self):
+        warnings.warn(
+            f"mlrun.api.schemas.{self._new_target.__name__} is deprecated in version {self._version}, "
+            f"Please use mlrun.common.schemas.{self._new_target.__name__} instead.",
+            FutureWarning,
+        )
+
+    def __call__(self, *args, **kwargs):
+        self._warn()
+        return self._new_target(*args, **kwargs)
+
+    def __getattr__(self, attr):
+        self._warn()
+        return getattr(self._new_target, attr)

mlrun/utils/http.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,10 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
 import time
+import typing
 
 import requests
 import requests.adapters
+import requests.utils
 import urllib3.exceptions
 import urllib3.util.retry
 
@@ -66,12 +69,12 @@ class HTTPSessionWithRetry(requests.Session):
     ):
         """
         Initialize a new HTTP session with retry logic.
-        :param max_retries: Maximum number of retries to attempt.
-        :param retry_backoff_factor: Wait interval retries in seconds.
-        :param retry_on_exception: Retry on the HTTP_RETRYABLE_EXCEPTIONS. defaults to True.
-        :param retry_on_status: Retry on error status codes. defaults to True.
-        :param retry_on_post: Retry on POST requests. defaults to False.
-        :param verbose: Print debug messages.
+        :param max_retries:             Maximum number of retries to attempt.
+        :param retry_backoff_factor:    Wait interval retries in seconds.
+        :param retry_on_exception:      Retry on the HTTP_RETRYABLE_EXCEPTIONS. defaults to True.
+        :param retry_on_status:         Retry on error status codes. defaults to True.
+        :param retry_on_post:           Retry on POST requests. defaults to False.
+        :param verbose:                 Print debug messages.
         """
         super().__init__()
 
@@ -79,14 +82,16 @@ class HTTPSessionWithRetry(requests.Session):
         self.retry_backoff_factor = retry_backoff_factor
         self.retry_on_exception = retry_on_exception
         self.verbose = verbose
+        self._logger = logger.get_child("http-client")
+        self._retry_methods = self._resolve_retry_methods(retry_on_post)
 
         if retry_on_status:
-            http_adapter = requests.adapters.HTTPAdapter(
+            self._http_adapter = requests.adapters.HTTPAdapter(
                 max_retries=urllib3.util.retry.Retry(
                     total=self.max_retries,
                     backoff_factor=self.retry_backoff_factor,
                     status_forcelist=config.http_retry_defaults.status_codes,
-                    method_whitelist=self._get_retry_methods(retry_on_post),
+                    method_whitelist=self._retry_methods,
                     # we want to retry but not to raise since we do want that last response (to parse details on the
                     # error from response body) we'll handle raising ourselves
                     raise_on_status=False,
@@ -94,54 +99,30 @@ class HTTPSessionWithRetry(requests.Session):
                 pool_maxsize=int(config.httpdb.max_workers),
             )
 
-            self.mount("http://", http_adapter)
-            self.mount("https://", http_adapter)
+            self.mount("http://", self._http_adapter)
+            self.mount("https://", self._http_adapter)
 
     def request(self, method, url, **kwargs):
         retry_count = 0
+        kwargs.setdefault("headers", {})
+        kwargs["headers"][
+            "User-Agent"
+        ] = f"{requests.utils.default_user_agent()} mlrun/{config.version}"
         while True:
             try:
                 response = super().request(method, url, **kwargs)
                 return response
             except Exception as exc:
-                if not self.retry_on_exception:
-                    self._log_exception(
-                        "warning",
-                        exc,
-                        f"{method} {url} request failed, http retries disabled,"
-                        f" raising exception: {err_to_str(exc)}",
-                        retry_count,
-                    )
+                if not self._error_is_retryable(url, method, exc, retry_count):
                     raise exc
 
-                if retry_count >= self.max_retries:
-                    self._log_exception(
-                        "warning",
-                        exc,
-                        f"{method} {url} request failed, max retries reached,"
-                        f" raising exception: {err_to_str(exc)}",
-                        retry_count,
-                    )
-                    raise exc
-
-                # only retryable exceptions
-                exception_is_retryable = any(
-                    msg in str(exc) for msg in self.HTTP_RETRYABLE_EXCEPTION_STRINGS
-                ) or any(
-                    isinstance(exc, retryable_exc)
-                    for retryable_exc in self.HTTP_RETRYABLE_EXCEPTIONS
+                self._logger.warning(
+                    "Error during request handling, retrying",
+                    exc=str(exc),
+                    retry_count=retry_count,
+                    url=url,
+                    method=method,
                 )
-
-                if not exception_is_retryable:
-                    self._log_exception(
-                        "warning",
-                        exc,
-                        f"{method} {url} request failed on non-retryable exception,"
-                        f" raising exception: {err_to_str(exc)}",
-                        retry_count,
-                    )
-                    raise exc
-
                 if self.verbose:
                     self._log_exception(
                         "debug",
@@ -153,17 +134,71 @@ class HTTPSessionWithRetry(requests.Session):
                 retry_count += 1
                 time.sleep(self.retry_backoff_factor)
 
-    @staticmethod
-    def _get_retry_methods(retry_on_post=False):
-        return (
-            # setting to False in order to retry on all methods, otherwise every method except POST.
-            False
-            if retry_on_post
-            else urllib3.util.retry.Retry.DEFAULT_METHOD_WHITELIST
+    def _error_is_retryable(self, url, method, exc, retry_count):
+        if not self.retry_on_exception:
+            self._log_exception(
+                "warning",
+                exc,
+                f"{method} {url} request failed, http retries disabled,"
+                f" raising exception: {err_to_str(exc)}",
+                retry_count,
+            )
+            return False
+
+        # if the response is not retryable, stop retrying
+        # this is done to prevent the retry logic from running on non-idempotent methods (such as POST).
+        if not self._method_retryable(method):
+            self._log_exception(
+                "warning",
+                exc,
+                f"{method} {url} request failed, http retries disabled for {method} method.",
+                retry_count,
+            )
+            return False
+
+        if retry_count >= self.max_retries:
+            self._log_exception(
+                "warning",
+                exc,
+                f"{method} {url} request failed, max retries reached,"
+                f" raising exception: {err_to_str(exc)}",
+                retry_count,
+            )
+            return False
+
+        # only retryable exceptions
+        exception_is_retryable = any(
+            msg in str(exc) for msg in self.HTTP_RETRYABLE_EXCEPTION_STRINGS
+        ) or any(
+            isinstance(exc, retryable_exc)
+            for retryable_exc in self.HTTP_RETRYABLE_EXCEPTIONS
         )
 
+        if not exception_is_retryable:
+            self._log_exception(
+                "warning",
+                exc,
+                f"{method} {url} request failed on non-retryable exception,"
+                f" raising exception: {err_to_str(exc)}",
+                retry_count,
+            )
+            return False
+        return True
+
+    def _method_retryable(self, method: str):
+        return method in self._retry_methods
+
+    def _resolve_retry_methods(
+        self, retry_on_post: bool = False
+    ) -> typing.FrozenSet[str]:
+        methods = urllib3.util.retry.Retry.DEFAULT_ALLOWED_METHODS
+        methods = methods.union({"PATCH"})
+        if retry_on_post:
+            methods = methods.union({"POST"})
+        return frozenset(methods)
+
     def _log_exception(self, level, exc, message, retry_count):
-        getattr(logger, level)(
+        getattr(self._logger, level)(
             message,
             exception_type=type(exc),
             exception_message=err_to_str(exc),

mlrun/utils/logger.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ import logging
 from enum import Enum
 from sys import stdout
 from traceback import format_exception
-from typing import IO, Union
+from typing import IO, Optional, Union
 
 from mlrun.config import config
 
@@ -42,20 +42,39 @@ class JSONFormatter(logging.Formatter):
 
 
 class HumanReadableFormatter(logging.Formatter):
-    def __init__(self):
-        super(HumanReadableFormatter, self).__init__()
-
     def format(self, record):
+        record_with = self._record_with(record)
+        more = f": {record_with}" if record_with else ""
+        return f"> {self.formatTime(record, self.datefmt)} [{record.levelname.lower()}] {record.getMessage()}{more}"
+
+    def _record_with(self, record):
         record_with = getattr(record, "with", {})
         if record.exc_info:
             record_with.update(exc_info=format_exception(*record.exc_info))
+        return record_with
+
+
+class HumanReadableExtendedFormatter(HumanReadableFormatter):
+    def format(self, record):
+        record_with = self._record_with(record)
         more = f": {record_with}" if record_with else ""
-        return f"> {self.formatTime(record, self.datefmt)} [{record.levelname.lower()}] {record.getMessage()}{more}"
+        return (
+            "> "
+            f"{self.formatTime(record, self.datefmt)} "
+            f"[{record.name}:{record.levelname.lower()}] "
+            f"{record.getMessage()}{more}"
+        )
 
 
 class Logger(object):
-    def __init__(self, level, name="mlrun", propagate=True):
-        self._logger = logging.getLogger(name)
+    def __init__(
+        self,
+        level,
+        name="mlrun",
+        propagate=True,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self._logger = logger or logging.getLogger(name)
         self._logger.propagate = propagate
         self._logger.setLevel(level)
         self._bound_variables = {}
@@ -90,6 +109,25 @@ class Logger(object):
         # add the handler to the logger
         self._logger.addHandler(stream_handler)
 
+    def get_child(self, suffix):
+        """
+        Get a child logger with the given suffix.
+        This is useful for when you want to have a logger for a specific component.
+        Once the formatter will support logger name, it will be easier to understand
+        which component logged the message.
+
+        :param suffix: The suffix to add to the logger name.
+        """
+        return Logger(
+            self.level,
+            # name is not set as it is provided by the "getChild"
+            name="",
+            # allowing child to delegate events logged to ancestor logger
+            # not doing so, will leave log lines not being handled
+            propagate=True,
+            logger=self._logger.getChild(suffix),
+        )
+
     @property
     def level(self):
         return self._logger.level
@@ -143,12 +181,14 @@ class Logger(object):
 
 class FormatterKinds(Enum):
     HUMAN = "human"
+    HUMAN_EXTENDED = "human_extended"
     JSON = "json"
 
 
 def _create_formatter_instance(formatter_kind: FormatterKinds) -> logging.Formatter:
     return {
         FormatterKinds.HUMAN: HumanReadableFormatter(),
+        FormatterKinds.HUMAN_EXTENDED: HumanReadableExtendedFormatter(),
         FormatterKinds.JSON: JSONFormatter(),
     }[formatter_kind]