mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/utils/helpers.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
 import enum
 import functools
 import hashlib
@@ -22,16 +23,14 @@ import os
 import re
 import string
 import sys
-import time
 import typing
 import warnings
 from datetime import datetime, timezone
 from importlib import import_module
 from os import path
 from types import ModuleType
-from typing import Any, List, Optional, Tuple
+from typing import Any, Optional
 
-import anyio
 import git
 import inflection
 import numpy as np
@@ -40,7 +39,7 @@ import pandas
 import semver
 import yaml
 from dateutil import parser
-from deprecated import deprecated
+from mlrun_pipelines.models import PipelineRun
 from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
@@ -50,10 +49,17 @@ import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.regex
 import mlrun.utils.version.version
+from mlrun.common.constants import MYSQL_MEDIUMBLOB_SIZE_BYTES
 from mlrun.config import config
-from mlrun.errors import err_to_str
 
 from .logger import create_logger
+from .retryer import (  # noqa: F401
+    AsyncRetryer,
+    Retryer,
+    create_exponential_backoff,
+    create_linear_backoff,
+    create_step_backoff,
+)
 
 yaml.Dumper.ignore_aliases = lambda *args: True
 _missing = object()
@@ -70,19 +76,6 @@ class OverwriteBuildParamsWarning(FutureWarning):
     pass
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'parse_versioned_object_uri' will be removed from this file in 1.7.0, use "
-    "'mlrun.common.helpers.parse_versioned_object_uri' instead",
-    category=FutureWarning,
-)
-def parse_versioned_object_uri(uri: str, default_project: str = ""):
-    return mlrun.common.helpers.parse_versioned_object_uri(
-        uri=uri, default_project=default_project
-    )
-
-
 class StorePrefix:
     """map mlrun store objects to prefixes"""
 
@@ -113,14 +106,9 @@ class StorePrefix:
 
 
 def get_artifact_target(item: dict, project=None):
-    if is_legacy_artifact(item):
-        db_key = item.get("db_key")
-        project_str = project or item.get("project")
-        tree = item.get("tree")
-    else:
-        db_key = item["spec"].get("db_key")
-        project_str = project or item["metadata"].get("project")
-        tree = item["metadata"].get("tree")
+    db_key = item["spec"].get("db_key")
+    project_str = project or item["metadata"].get("project")
+    tree = item["metadata"].get("tree")
 
     kind = item.get("kind")
     if kind in ["dataset", "model", "artifact"] and db_key:
@@ -129,11 +117,15 @@ def get_artifact_target(item: dict, project=None):
             target = f"{target}@{tree}"
         return target
 
-    return (
-        item.get("target_path")
-        if is_legacy_artifact(item)
-        else item["spec"].get("target_path")
-    )
+    return item["spec"].get("target_path")
+
+
+# TODO: left for migrations testing purposes. Remove in 1.8.0.
+def is_legacy_artifact(artifact):
+    if isinstance(artifact, dict):
+        return "metadata" not in artifact
+    else:
+        return not hasattr(artifact, "metadata")
 
 
 logger = create_logger(config.log_level, config.log_formatter, "mlrun", sys.stdout)
@@ -189,8 +181,12 @@ def verify_field_regex(
             )
             if mode == mlrun.common.schemas.RegexMatchModes.all:
                 if raise_on_failure:
+                    if len(field_name) > max_chars:
+                        field_name = field_name[:max_chars] + "...truncated"
+                    if len(field_value) > max_chars:
+                        field_value = field_value[:max_chars] + "...truncated"
                     raise mlrun.errors.MLRunInvalidArgumentError(
-                        f"Field '{field_name[:max_chars]}' is malformed. '{field_value[:max_chars]}' "
+                        f"Field '{field_name}' is malformed. '{field_value}' "
                         f"does not match required pattern: {pattern}"
                     )
                 return False
@@ -265,6 +261,17 @@ def validate_artifact_key_name(
     )
 
 
+def validate_inline_artifact_body_size(body: typing.Union[str, bytes, None]) -> None:
+    if body and len(body) > MYSQL_MEDIUMBLOB_SIZE_BYTES:
+        raise mlrun.errors.MLRunBadRequestError(
+            "The body of the artifact exceeds the maximum allowed size. "
+            "Avoid embedding the artifact body. "
+            "This increases the size of the project yaml file and could affect the project during loading and saving. "
+            "More information is available at"
+            "https://docs.mlrun.org/en/latest/projects/automate-project-git-source.html#setting-and-registering-the-project-artifacts"
+        )
+
+
 def validate_v3io_stream_consumer_group(
     value: str, raise_on_failure: bool = True
 ) -> bool:
@@ -276,12 +283,12 @@ def validate_v3io_stream_consumer_group(
     )
 
 
-def get_regex_list_as_string(regex_list: List) -> str:
+def get_regex_list_as_string(regex_list: list) -> str:
     """
     This function is used to combine a list of regex strings into a single regex,
     with and condition between them.
     """
-    return "".join(["(?={regex})".format(regex=regex) for regex in regex_list]) + ".*$"
+    return "".join([f"(?={regex})" for regex in regex_list]) + ".*$"
 
 
 def tag_name_regex_as_string() -> str:
@@ -420,7 +427,7 @@ class LogBatchWriter:
 
 def get_in(obj, keys, default=None):
     """
-    >>> get_in({'a': {'b': 1}}, 'a.b')
+    >>> get_in({"a": {"b": 1}}, "a.b")
     1
     """
     if isinstance(keys, str):
@@ -698,7 +705,7 @@ def generate_artifact_uri(project, key, tag=None, iter=None, tree=None):
     return artifact_uri
 
 
-def extend_hub_uri_if_needed(uri) -> Tuple[str, bool]:
+def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
     """
     Retrieve the full uri of the item's yaml in the hub.
 
@@ -784,34 +791,6 @@ def gen_html_table(header, rows=None):
     return style + '<table class="tg">\n' + out + "</table>\n\n"
 
 
-def new_pipe_metadata(
-    artifact_path: str = None,
-    cleanup_ttl: int = None,
-    op_transformers: typing.List[typing.Callable] = None,
-):
-    from kfp.dsl import PipelineConf
-
-    def _set_artifact_path(task):
-        from kubernetes import client as k8s_client
-
-        task.add_env_variable(
-            k8s_client.V1EnvVar(name="MLRUN_ARTIFACT_PATH", value=artifact_path)
-        )
-        return task
-
-    conf = PipelineConf()
-    cleanup_ttl = cleanup_ttl or int(config.kfp_ttl)
-
-    if cleanup_ttl:
-        conf.set_ttl_seconds_after_finished(cleanup_ttl)
-    if artifact_path:
-        conf.add_op_transformer(_set_artifact_path)
-    if op_transformers:
-        for op_transformer in op_transformers:
-            conf.add_op_transformer(op_transformer)
-    return conf
-
-
 def _convert_python_package_version_to_image_tag(version: typing.Optional[str]):
     return (
         version.replace("+", "-").replace("0.0.0-", "") if version is not None else None
@@ -893,7 +872,7 @@ def get_docker_repository_or_default(repository: str) -> str:
     return repository
 
 
-def get_parsed_docker_registry() -> Tuple[Optional[str], Optional[str]]:
+def get_parsed_docker_registry() -> tuple[Optional[str], Optional[str]]:
     # according to https://stackoverflow.com/questions/37861791/how-are-docker-image-names-parsed
     docker_registry = config.httpdb.builder.docker_registry or ""
     first_slash_index = docker_registry.find("/")
@@ -947,65 +926,27 @@ def fill_function_hash(function_dict, tag=""):
     return fill_object_hash(function_dict, "hash", tag)
 
 
-def create_linear_backoff(base=2, coefficient=2, stop_value=120):
-    """
-    Create a generator of linear backoff. Check out usage example in test_helpers.py
-    """
-    x = 0
-    comparison = min if coefficient >= 0 else max
-
-    while True:
-        next_value = comparison(base + x * coefficient, stop_value)
-        yield next_value
-        x += 1
-
-
-def create_step_backoff(steps=None):
-    """
-    Create a generator of steps backoff.
-    Example: steps = [[2, 5], [20, 10], [120, None]] will produce a generator in which the first 5
-    values will be 2, the next 10 values will be 20 and the rest will be 120.
-    :param steps: a list of lists [step_value, number_of_iteration_in_this_step]
-    """
-    steps = steps if steps is not None else [[2, 10], [10, 10], [120, None]]
-    steps = iter(steps)
-
-    # Get first step
-    step = next(steps)
-    while True:
-        current_step_value, current_step_remain = step
-        if current_step_remain == 0:
-            # No more in this step, moving on
-            step = next(steps)
-        elif current_step_remain is None:
-            # We are in the last step, staying here forever
-            yield current_step_value
-        elif current_step_remain > 0:
-            # Still more remains in this step, just reduce the remaining number
-            step[1] -= 1
-            yield current_step_value
-
-
-def create_exponential_backoff(base=2, max_value=120, scale_factor=1):
+def retry_until_successful(
+    backoff: int, timeout: int, logger, verbose: bool, _function, *args, **kwargs
+):
     """
-    Create a generator of exponential backoff. Check out usage example in test_helpers.py
-    :param base: exponent base
-    :param max_value: max limit on the result
-    :param scale_factor: factor to be used as linear scaling coefficient
+    Runs function with given *args and **kwargs.
+    Tries to run it until success or timeout reached (timeout is optional)
+    :param backoff: can either be a:
+            - number (int / float) that will be used as interval.
+            - generator of waiting intervals. (support next())
+    :param timeout: pass None if timeout is not wanted, number of seconds if it is
+    :param logger: a logger so we can log the failures
+    :param verbose: whether to log the failure on each retry
+    :param _function: function to run
+    :param args: functions args
+    :param kwargs: functions kwargs
+    :return: function result
     """
-    exponent = 1
-    while True:
-        # This "complex" implementation (unlike the one in linear backoff) is to avoid exponent growing too fast and
-        # risking going behind max_int
-        next_value = scale_factor * (base**exponent)
-        if next_value < max_value:
-            exponent += 1
-            yield next_value
-        else:
-            yield max_value
+    return Retryer(backoff, timeout, logger, verbose, _function, *args, **kwargs).run()
 
 
-def retry_until_successful(
+async def retry_until_successful_async(
     backoff: int, timeout: int, logger, verbose: bool, _function, *args, **kwargs
 ):
     """
@@ -1022,80 +963,41 @@ def retry_until_successful(
     :param kwargs: functions kwargs
     :return: function result
     """
-    start_time = time.time()
-    last_exception = None
-
-    # Check if backoff is just a simple interval
-    if isinstance(backoff, int) or isinstance(backoff, float):
-        backoff = create_linear_backoff(base=backoff, coefficient=0)
-
-    first_interval = next(backoff)
-    if timeout and timeout <= first_interval:
-        logger.warning(
-            f"Timeout ({timeout}) must be higher than backoff ({first_interval})."
-            f" Set timeout to be higher than backoff."
-        )
-
-    # If deadline was not provided or deadline not reached
-    while timeout is None or time.time() < start_time + timeout:
-        next_interval = first_interval or next(backoff)
-        first_interval = None
-        try:
-            result = _function(*args, **kwargs)
-            return result
-
-        except mlrun.errors.MLRunFatalFailureError as exc:
-            raise exc.original_exception
-        except Exception as exc:
-            last_exception = exc
-
-            # If next interval is within allowed time period - wait on interval, abort otherwise
-            if timeout is None or time.time() + next_interval < start_time + timeout:
-                if logger is not None and verbose:
-                    logger.debug(
-                        f"Operation not yet successful, Retrying in {next_interval} seconds."
-                        f" exc: {err_to_str(exc)}"
-                    )
-
-                time.sleep(next_interval)
-            else:
-                break
-
-    if logger is not None:
-        logger.warning(
-            f"Operation did not complete on time. last exception: {last_exception}"
-        )
-
-    raise mlrun.errors.MLRunRetryExhaustedError(
-        f"Failed to execute command by the given deadline."
-        f" last_exception: {last_exception},"
-        f" function_name: {_function.__name__},"
-        f" timeout: {timeout}"
-    ) from last_exception
+    return await AsyncRetryer(
+        backoff, timeout, logger, verbose, _function, *args, **kwargs
+    ).run()
 
 
 def get_ui_url(project, uid=None):
     url = ""
     if mlrun.mlconf.resolve_ui_url():
-        url = "{}/{}/{}/jobs".format(
-            mlrun.mlconf.resolve_ui_url(), mlrun.mlconf.ui.projects_prefix, project
-        )
+        url = f"{mlrun.mlconf.resolve_ui_url()}/{mlrun.mlconf.ui.projects_prefix}/{project}/jobs"
         if uid:
             url += f"/monitor/{uid}/overview"
     return url
 
 
+def get_model_endpoint_url(project, model_name, model_endpoint_id):
+    url = ""
+    if mlrun.mlconf.resolve_ui_url():
+        url = f"{mlrun.mlconf.resolve_ui_url()}/{mlrun.mlconf.ui.projects_prefix}/{project}/models"
+        if model_name:
+            url += f"/model-endpoints/{model_name}/{model_endpoint_id}/overview"
+    return url
+
+
 def get_workflow_url(project, id=None):
     url = ""
     if mlrun.mlconf.resolve_ui_url():
-        url = "{}/{}/{}/jobs/monitor-workflows/workflow/{}".format(
-            mlrun.mlconf.resolve_ui_url(), mlrun.mlconf.ui.projects_prefix, project, id
+        url = (
+            f"{mlrun.mlconf.resolve_ui_url()}/{mlrun.mlconf.ui.projects_prefix}"
+            f"/{project}/jobs/monitor-workflows/workflow/{id}"
         )
     return url
 
 
 def are_strings_in_exception_chain_messages(
-    exception: Exception, strings_list=typing.List[str]
+    exception: Exception, strings_list: list[str]
 ) -> bool:
     while exception is not None:
         if any([string in str(exception) for string in strings_list]):
@@ -1208,7 +1110,7 @@ def get_function(function, namespace):
 
 
 def get_handler_extended(
-    handler_path: str, context=None, class_args: dict = {}, namespaces=None
+    handler_path: str, context=None, class_args: dict = None, namespaces=None
 ):
     """get function handler from [class_name::]handler string
 
@@ -1218,6 +1120,7 @@ def get_handler_extended(
     :param namespaces:    one or list of namespaces/modules to search the handler in
     :return: function handler (callable)
     """
+    class_args = class_args or {}
     if "::" not in handler_path:
         return get_function(handler_path, namespaces)
 
@@ -1275,7 +1178,7 @@ def has_timezone(timestamp):
         return False
 
 
-def as_list(element: Any) -> List[Any]:
+def as_list(element: Any) -> list[Any]:
     return element if isinstance(element, list) else [element]
 
 
@@ -1294,7 +1197,7 @@ def calculate_dataframe_hash(dataframe: pandas.DataFrame):
     return hashlib.sha1(pandas.util.hash_pandas_object(dataframe).values).hexdigest()
 
 
-def template_artifact_path(artifact_path, project, run_uid="project"):
+def template_artifact_path(artifact_path, project, run_uid=None):
     """
     Replace {{run.uid}} with the run uid and {{project}} with the project name in the artifact path.
     If no run uid is provided, the word `project` will be used instead as it is assumed to be a project
@@ -1302,6 +1205,7 @@ def template_artifact_path(artifact_path, project, run_uid="project"):
     """
     if not artifact_path:
         return artifact_path
+    run_uid = run_uid or "project"
     artifact_path = artifact_path.replace("{{run.uid}}", run_uid)
     artifact_path = _fill_project_path_template(artifact_path, project)
     return artifact_path
@@ -1361,13 +1265,6 @@ def str_to_timestamp(time_str: str, now_time: Timestamp = None):
     return Timestamp(time_str)
 
 
-def is_legacy_artifact(artifact):
-    if isinstance(artifact, dict):
-        return "metadata" not in artifact
-    else:
-        return not hasattr(artifact, "metadata")
-
-
 def is_link_artifact(artifact):
     if isinstance(artifact, dict):
         return (
@@ -1377,7 +1274,7 @@ def is_link_artifact(artifact):
         return artifact.kind == mlrun.common.schemas.ArtifactCategories.link.value
 
 
-def format_run(run: dict, with_project=False) -> dict:
+def format_run(run: PipelineRun, with_project=False) -> dict:
     fields = [
         "id",
         "name",
@@ -1414,17 +1311,17 @@ def format_run(run: dict, with_project=False) -> dict:
     # pipelines are yet to populate the status or workflow has failed
     # as observed https://jira.iguazeng.com/browse/ML-5195
     # set to unknown to ensure a status is returned
-    if run["status"] is None:
-        run["status"] = inflection.titleize(mlrun.runtimes.constants.RunStates.unknown)
+    if run.get("status", None) is None:
+        run["status"] = inflection.titleize(
+            mlrun.common.runtimes.constants.RunStates.unknown
+        )
 
     return run
 
 
 def get_in_artifact(artifact: dict, key, default=None, raise_on_missing=False):
     """artifact can be dict or Artifact object"""
-    if is_legacy_artifact(artifact):
-        return artifact.get(key, default)
-    elif key == "kind":
+    if key == "kind":
         return artifact.get(key, default)
     else:
         for block in ["metadata", "spec", "status"]:
@@ -1558,13 +1455,15 @@ def normalize_project_username(username: str):
     return username
 
 
-# run_in threadpool is taken from fastapi to allow us to run sync functions in a threadpool
-# without importing fastapi in the client
 async def run_in_threadpool(func, *args, **kwargs):
+    """
+    Run a sync-function in the loop default thread pool executor pool and await its result.
+    Note that this function is not suitable for CPU-bound tasks, as it will block the event loop.
+    """
+    loop = asyncio.get_running_loop()
     if kwargs:
-        # run_sync doesn't accept 'kwargs', so bind them in here
         func = functools.partial(func, **kwargs)
-    return await anyio.to_thread.run_sync(func, *args)
+    return await loop.run_in_executor(None, func, *args)
 
 
 def is_explicit_ack_supported(context):
@@ -1630,7 +1529,7 @@ def is_ecr_url(registry: str) -> bool:
     return ".ecr." in registry and ".amazonaws.com" in registry
 
 
-def get_local_file_schema() -> List:
+def get_local_file_schema() -> list:
     # The expression `list(string.ascii_lowercase)` generates a list of lowercase alphabets,
     # which corresponds to drive letters in Windows file paths such as `C:/Windows/path`.
     return ["file"] + list(string.ascii_lowercase)
@@ -1642,3 +1541,90 @@ def is_safe_path(base, filepath, is_symlink=False):
         os.path.abspath(filepath) if not is_symlink else os.path.realpath(filepath)
     )
     return base == os.path.commonpath((base, resolved_filepath))
+
+
+def get_serving_spec():
+    data = None
+
+    # we will have the serving spec in either mounted config map
+    # or env depending on the size of the spec and configuration
+
+    try:
+        with open(mlrun.common.constants.MLRUN_SERVING_SPEC_PATH) as f:
+            data = f.read()
+    except FileNotFoundError:
+        pass
+
+    if data is None:
+        data = os.environ.get("SERVING_SPEC_ENV", "")
+        if not data:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Failed to find serving spec in env var or config file"
+            )
+    spec = json.loads(data)
+    return spec
+
+
+def additional_filters_warning(additional_filters, class_name):
+    if additional_filters and any(additional_filters):
+        mlrun.utils.logger.warn(
+            f"additional_filters parameter is not supported in {class_name},"
+            f" parameter has been ignored."
+        )
+
+
+def validate_component_version_compatibility(
+    component_name: typing.Literal["iguazio", "nuclio"], *min_versions: str
+):
+    """
+    :param component_name: Name of the component to validate compatibility for.
+    :param min_versions: Valid minimum version(s) required, assuming no 2 versions has equal major and minor.
+    """
+    parsed_min_versions = [
+        semver.VersionInfo.parse(min_version) for min_version in min_versions
+    ]
+    parsed_current_version = None
+    component_current_version = None
+    try:
+        if component_name == "iguazio":
+            component_current_version = mlrun.mlconf.igz_version
+            parsed_current_version = mlrun.mlconf.get_parsed_igz_version()
+
+            if parsed_current_version:
+                # ignore pre-release and build metadata, as iguazio version always has them, and we only care about the
+                # major, minor, and patch versions
+                parsed_current_version = semver.VersionInfo.parse(
+                    f"{parsed_current_version.major}.{parsed_current_version.minor}.{parsed_current_version.patch}"
+                )
+        if component_name == "nuclio":
+            component_current_version = mlrun.mlconf.nuclio_version
+            parsed_current_version = semver.VersionInfo.parse(
+                mlrun.mlconf.nuclio_version
+            )
+        if not parsed_current_version:
+            return True
+    except ValueError:
+        # only log when version is set but invalid
+        if component_current_version:
+            logger.warning(
+                "Unable to parse current version, assuming compatibility",
+                component_name=component_name,
+                current_version=component_current_version,
+                min_versions=min_versions,
+            )
+        return True
+
+    parsed_min_versions.sort(reverse=True)
+    for parsed_min_version in parsed_min_versions:
+        if parsed_current_version < parsed_min_version:
+            return False
+    return True
+
+
+def format_alert_summary(
+    alert: mlrun.common.schemas.AlertConfig, event_data: mlrun.common.schemas.Event
+) -> str:
+    result = alert.summary.replace("{{project}}", alert.project)
+    result = result.replace("{{name}}", alert.name)
+    result = result.replace("{{entity}}", event_data.entity.ids[0])
+    return result

mlrun/utils/http.py

@@ -14,7 +14,6 @@
 #
 
 import time
-import typing
 
 import requests
 import requests.adapters
@@ -123,7 +122,7 @@ class HTTPSessionWithRetry(requests.Session):
 
                 self._logger.warning(
                     "Error during request handling, retrying",
-                    exc=str(exc),
+                    exc=err_to_str(exc),
                     retry_count=retry_count,
                     url=url,
                     method=method,
@@ -202,9 +201,7 @@ class HTTPSessionWithRetry(requests.Session):
     def _method_retryable(self, method: str):
         return method in self._retry_methods
 
-    def _resolve_retry_methods(
-        self, retry_on_post: bool = False
-    ) -> typing.FrozenSet[str]:
+    def _resolve_retry_methods(self, retry_on_post: bool = False) -> frozenset[str]:
         methods = urllib3.util.retry.Retry.DEFAULT_ALLOWED_METHODS
         methods = methods.union({"PATCH"})
         if retry_on_post: