mlrun 1.10.0rc13__py3-none-any.whl → 1.10.0rc42__py3-none-any.whl

mlrun/utils/helpers.py

@@ -15,13 +15,13 @@
 import asyncio
 import base64
 import enum
-import functools
 import gzip
 import hashlib
 import inspect
 import itertools
 import json
 import os
+import pathlib
 import re
 import string
 import sys
@@ -29,6 +29,7 @@ import traceback
 import typing
 import uuid
 import warnings
+from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from importlib import import_module, reload
 from os import path
@@ -45,6 +46,8 @@ import pytz
 import semver
 import yaml
 from dateutil import parser
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
 from pandas import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
@@ -61,6 +64,7 @@ import mlrun_pipelines.models
 import mlrun_pipelines.utils
 from mlrun.common.constants import MYSQL_MEDIUMBLOB_SIZE_BYTES
 from mlrun.common.schemas import ArtifactCategories
+from mlrun.common.schemas.hub import HubSourceType
 from mlrun.config import config
 from mlrun_pipelines.models import PipelineRun
 
@@ -162,14 +166,6 @@ def get_artifact_target(item: dict, project=None):
     return item["spec"].get("target_path")
 
 
-# TODO: Remove once data migration v5 is obsolete
-def is_legacy_artifact(artifact):
-    if isinstance(artifact, dict):
-        return "metadata" not in artifact
-    else:
-        return not hasattr(artifact, "metadata")
-
-
 logger = create_logger(config.log_level, config.log_formatter, "mlrun", sys.stdout)
 missing = object()
 
@@ -257,6 +253,40 @@ def verify_field_regex(
         return False
 
 
+def validate_function_name(name: str) -> None:
+    """
+    Validate that a function name conforms to Kubernetes DNS-1123 label requirements.
+
+    Function names for Kubernetes resources must:
+    - Be lowercase alphanumeric characters or '-'
+    - Start and end with an alphanumeric character
+    - Be at most 63 characters long
+
+    This validation should be called AFTER normalize_name() has been applied.
+
+    Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
+
+    :param name: The function name to validate (after normalization)
+    :raises MLRunInvalidArgumentError: If the function name is invalid for Kubernetes
+    """
+    if not name:
+        return
+
+    verify_field_regex(
+        "function.metadata.name",
+        name,
+        mlrun.utils.regex.dns_1123_label,
+        raise_on_failure=True,
+        log_message=(
+            f"Function name '{name}' is invalid. "
+            "Kubernetes function names must be DNS-1123 labels: "
+            "lowercase alphanumeric characters or '-', "
+            "starting and ending with an alphanumeric character, "
+            "and at most 63 characters long."
+        ),
+    )
+
+
 def validate_builder_source(
     source: str, pull_at_runtime: bool = False, workdir: Optional[str] = None
 ):
@@ -471,21 +501,49 @@ def to_date_str(d):
     return ""
 
 
-def normalize_name(name: str, verbose: bool = True):
+def normalize_name(name: str):
     # TODO: Must match
     # [a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?
     name = re.sub(r"\s+", "-", name)
     if "_" in name:
-        if verbose:
-            warnings.warn(
-                "Names with underscore '_' are about to be deprecated, use dashes '-' instead. "
-                f"Replacing '{name}' underscores with dashes.",
-                FutureWarning,
-            )
         name = name.replace("_", "-")
     return name.lower()
 
 
+def ensure_batch_job_suffix(
+    function_name: typing.Optional[str],
+) -> tuple[typing.Optional[str], bool, str]:
+    """
+    Ensure that a function name has the batch job suffix appended to prevent database collision.
+
+    This helper is used by to_job() methods in runtimes that convert online functions (serving, local)
+    to batch processing jobs. The suffix prevents the job from overwriting the original function in
+    the database when both are stored with the same (project, name) key.
+
+    :param function_name: The original function name (can be None or empty string)
+
+    :return: A tuple of (modified_name, was_renamed, suffix) where:
+        - modified_name: The function name with the batch suffix (if not already present),
+          or empty string if input was empty
+        - was_renamed: True if the suffix was added, False if it was already present or if name was empty/None
+        - suffix: The suffix value that was used (or would have been used)
+
+    """
+    suffix = mlrun_constants.RESERVED_BATCH_JOB_SUFFIX
+
+    # Handle None or empty string
+    if not function_name:
+        return function_name, False, suffix
+
+    if not function_name.endswith(suffix):
+        return (
+            f"{function_name}{suffix}",
+            True,
+            suffix,
+        )
+    return function_name, False, suffix
+
+
 class LogBatchWriter:
     def __init__(self, func, batch=16, maxtime=5):
         self.batch = batch
@@ -515,9 +573,14 @@ def get_in(obj, keys, default=None):
     if isinstance(keys, str):
         keys = keys.split(".")
     for key in keys:
-        if not obj or key not in obj:
+        if obj is None:
             return default
-        obj = obj[key]
+        if isinstance(obj, dict):
+            if key not in obj:
+                return default
+            obj = obj[key]
+        else:
+            obj = getattr(obj, key, default)
     return obj
 
 
@@ -794,17 +857,48 @@ def generate_artifact_uri(
     return artifact_uri
 
 
-def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
+def remove_tag_from_artifact_uri(uri: str) -> Optional[str]:
     """
-    Retrieve the full uri of the item's yaml in the hub.
+    Remove the `:<tag>` part from a URI with pattern:
+    [store://][<project>/]<key>[#<iter>][:<tag>][@<tree>][^<uid>]
+
+    Returns the URI without the tag section.
+
+    Examples:
+        "store://proj/key:latest" => "store://proj/key"
+        "key#1:dev@tree^uid" => "key#1@tree^uid"
+        "store://key:tag" => "store://key"
+        "store://models/remote-model-project/my_model#0@tree" => unchanged (no tag)
+    """
+    add_store = False
+    if mlrun.datastore.is_store_uri(uri):
+        uri = uri.removeprefix(DB_SCHEMA + "://")
+        add_store = True
+    uri = re.sub(r"(#[^:@\s]*)?:[^@^:\s]+(?=(@|\^|$))", lambda m: m.group(1) or "", uri)
+    return uri if not add_store else DB_SCHEMA + "://" + uri
+
+
+def check_if_hub_uri(uri: str) -> bool:
+    return uri.startswith(hub_prefix)
+
+
+def extend_hub_uri_if_needed(
+    uri: str,
+    asset_type: HubSourceType = HubSourceType.functions,
+    file: str = "function.yaml",
+) -> tuple[str, bool]:
+    """
+    Retrieve the full uri of an object in the hub.
 
     :param uri: structure: "hub://[<source>/]<item-name>[:<tag>]"
+    :param asset_type:  The type of the hub item (functions, modules, etc.)
+    :param file: The file name inside the hub item directory (default: function.yaml)
 
     :return: A tuple of:
                [0] = Extended URI of item
                [1] =  Is hub item (bool)
     """
-    is_hub_uri = uri.startswith(hub_prefix)
+    is_hub_uri = check_if_hub_uri(uri)
     if not is_hub_uri:
         return uri, is_hub_uri
 
@@ -821,10 +915,10 @@ def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Invalid character '/' in function name or source name"
             ) from exc
-    name = normalize_name(name=name, verbose=False)
+    name = normalize_name(name=name)
     if not source_name:
         # Searching item in all sources
-        sources = db.list_hub_sources(item_name=name, tag=tag)
+        sources = db.list_hub_sources(item_name=name, tag=tag, item_type=asset_type)
         if not sources:
             raise mlrun.errors.MLRunNotFoundError(
                 f"Item={name}, tag={tag} not found in any hub source"
@@ -834,10 +928,10 @@ def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
     else:
         # Specific source is given
         indexed_source = db.get_hub_source(source_name)
-    # hub function directory name are with underscores instead of hyphens
+    # hub directories name are with underscores instead of hyphens
     name = name.replace("-", "_")
-    function_suffix = f"{name}/{tag}/src/function.yaml"
-    return indexed_source.source.get_full_uri(function_suffix), is_hub_uri
+    suffix = f"{name}/{tag}/src/{file}"
+    return indexed_source.source.get_full_uri(suffix, asset_type), is_hub_uri
 
 
 def gen_md_table(header, rows=None):
@@ -905,10 +999,20 @@ def enrich_image_url(
     mlrun_version = config.images_tag or client_version or server_version
     tag = mlrun_version or ""
 
-    # TODO: Remove condition when mlrun/mlrun-kfp image is also supported
-    if "mlrun-kfp" not in image_url:
+    # starting mlrun 1.10.0-rc0 we want to enrich the kfp image with the python version
+    # e.g for 1.9 we have a single mlrun-kfp image that supports only python 3.9
+    enrich_kfp_python_version = (
+        "mlrun-kfp" in image_url
+        and mlrun_version
+        and semver.VersionInfo.is_valid(mlrun_version)
+        and semver.VersionInfo.parse(mlrun_version)
+        >= semver.VersionInfo.parse("1.10.0-rc0")
+    )
+
+    if "mlrun-kfp" not in image_url or enrich_kfp_python_version:
         tag += resolve_image_tag_suffix(
-            mlrun_version=mlrun_version, python_version=client_python_version
+            mlrun_version=mlrun_version,
+            python_version=client_python_version,
         )
 
     # it's an mlrun image if the repository is mlrun
@@ -934,8 +1038,15 @@ def enrich_image_url(
         else:
             image_url = "mlrun/mlrun"
 
-    if is_mlrun_image and tag and ":" not in image_url:
-        image_url = f"{image_url}:{tag}"
+    if is_mlrun_image and tag:
+        if ":" not in image_url:
+            image_url = f"{image_url}:{tag}"
+        elif enrich_kfp_python_version:
+            # For mlrun-kfp >= 1.10.0-rc0, append python suffix to existing tag
+            python_suffix = resolve_image_tag_suffix(
+                mlrun_version, client_python_version
+            )
+            image_url = f"{image_url}{python_suffix}" if python_suffix else image_url
 
     registry = (
         config.images_registry if is_mlrun_image else config.vendor_images_registry
@@ -1050,7 +1161,14 @@ def fill_function_hash(function_dict, tag=""):
 
 
 def retry_until_successful(
-    backoff: int, timeout: int, logger, verbose: bool, _function, *args, **kwargs
+    backoff: int,
+    timeout: int,
+    logger,
+    verbose: bool,
+    _function,
+    *args,
+    fatal_exceptions=(),
+    **kwargs,
 ):
     """
     Runs function with given *args and **kwargs.
@@ -1063,14 +1181,31 @@ def retry_until_successful(
     :param verbose: whether to log the failure on each retry
     :param _function: function to run
     :param args: functions args
+    :param fatal_exceptions: exception types that should not be retried
     :param kwargs: functions kwargs
     :return: function result
     """
-    return Retryer(backoff, timeout, logger, verbose, _function, *args, **kwargs).run()
+    return Retryer(
+        backoff,
+        timeout,
+        logger,
+        verbose,
+        _function,
+        *args,
+        fatal_exceptions=fatal_exceptions,
+        **kwargs,
+    ).run()
 
 
 async def retry_until_successful_async(
-    backoff: int, timeout: int, logger, verbose: bool, _function, *args, **kwargs
+    backoff: int,
+    timeout: int,
+    logger,
+    verbose: bool,
+    _function,
+    *args,
+    fatal_exceptions=(),
+    **kwargs,
 ):
     """
     Runs function with given *args and **kwargs.
@@ -1082,12 +1217,20 @@ async def retry_until_successful_async(
     :param logger: a logger so we can log the failures
     :param verbose: whether to log the failure on each retry
     :param _function: function to run
+    :param fatal_exceptions: exception types that should not be retried
     :param args: functions args
     :param kwargs: functions kwargs
     :return: function result
     """
     return await AsyncRetryer(
-        backoff, timeout, logger, verbose, _function, *args, **kwargs
+        backoff,
+        timeout,
+        logger,
+        verbose,
+        _function,
+        *args,
+        fatal_exceptions=fatal_exceptions,
+        **kwargs,
     ).run()
 
 
@@ -1173,55 +1316,6 @@ def get_workflow_url(
     return url
 
 
-def get_kfp_list_runs_filter(
-    project_name: Optional[str] = None,
-    end_date: Optional[str] = None,
-    start_date: Optional[str] = None,
-) -> str:
-    """
-    Generates a filter for listing Kubeflow Pipelines (KFP) runs.
-
-    :param project_name: The name of the project. If "*", it won't filter by project.
-    :param end_date: The latest creation date for filtering runs (ISO 8601 format).
-    :param start_date: The earliest creation date for filtering runs (ISO 8601 format).
-    :return: A JSON-formatted filter string for KFP.
-    """
-
-    # KFP filter operation codes
-    kfp_less_than_or_equal_op = 7  # '<='
-    kfp_greater_than_or_equal_op = 5  # '>='
-    kfp_substring_op = 9  # Substring match
-
-    filters = {"predicates": []}
-
-    if end_date:
-        filters["predicates"].append(
-            {
-                "key": "created_at",
-                "op": kfp_less_than_or_equal_op,
-                "timestamp_value": end_date,
-            }
-        )
-
-    if project_name and project_name != "*":
-        filters["predicates"].append(
-            {
-                "key": "name",
-                "op": kfp_substring_op,
-                "string_value": project_name,
-            }
-        )
-    if start_date:
-        filters["predicates"].append(
-            {
-                "key": "created_at",
-                "op": kfp_greater_than_or_equal_op,
-                "timestamp_value": start_date,
-            }
-        )
-    return json.dumps(filters)
-
-
 def validate_and_convert_date(date_input: str) -> str:
     """
     Converts any recognizable date string into a standardized RFC 3339 format.
@@ -1819,10 +1913,7 @@ async def run_in_threadpool(func, *args, **kwargs):
     Run a sync-function in the loop default thread pool executor pool and await its result.
     Note that this function is not suitable for CPU-bound tasks, as it will block the event loop.
     """
-    loop = asyncio.get_running_loop()
-    if kwargs:
-        func = functools.partial(func, **kwargs)
-    return await loop.run_in_executor(None, func, *args)
+    return await asyncio.to_thread(func, *args, **kwargs)
 
 
 def is_explicit_ack_supported(context):
@@ -2352,3 +2443,112 @@ def encode_user_code(
             "Consider using `with_source_archive` to add user code as a remote source to the function."
         )
     return encoded
+
+
+def split_path(path: str) -> typing.Union[str, list[str], None]:
+    if path is not None:
+        parsed_path = path.split(".")
+        if len(parsed_path) == 1:
+            parsed_path = parsed_path[0]
+        return parsed_path
+    return path
+
+
+def get_data_from_path(path: typing.Union[str, list[str], None], data: dict) -> Any:
+    if isinstance(path, str):
+        output_data = data.get(path)
+    elif isinstance(path, list):
+        output_data = deepcopy(data)
+        for key in path:
+            output_data = output_data.get(key, {})
+    elif path is None:
+        output_data = data
+    else:
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            "Expected path be of type str or list of str or None"
+        )
+    return output_data
+
+
+def is_valid_port(port: int, raise_on_error: bool = False) -> bool:
+    if not port:
+        return False
+    if 0 <= port <= 65535:
+        return True
+    if raise_on_error:
+        raise ValueError("Port must be in the range 0–65535")
+    return False
+
+
+def set_data_by_path(
+    path: typing.Union[str, list[str], None], data: dict, value
+) -> None:
+    if path is None:
+        if not isinstance(value, dict):
+            raise ValueError("When path is None, value must be a dictionary.")
+        data.update(value)
+
+    elif isinstance(path, str):
+        data[path] = value
+
+    elif isinstance(path, list):
+        current = data
+        for key in path[:-1]:
+            if key not in current or not isinstance(current[key], dict):
+                current[key] = {}
+            current = current[key]
+        current[path[-1]] = value
+    else:
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            "Expected path to be of type str or list of str"
+        )
+
+
+def _normalize_requirements(reqs: typing.Union[str, list[str], None]) -> list[str]:
+    if reqs is None:
+        return []
+    if isinstance(reqs, str):
+        s = reqs.strip()
+        return [s] if s else []
+    return [s.strip() for s in reqs if s and s.strip()]
+
+
+def merge_requirements(
+    reqs_priority: typing.Union[str, list[str], None],
+    reqs_secondary: typing.Union[str, list[str], None],
+) -> list[str]:
+    """
+    Merge two requirement collections into a union. If the same package
+    appears in both, the specifier from reqs_priority wins.
+
+    Args:
+        reqs_priority: str | list[str] | None  (priority input)
+        reqs_secondary: str | list[str] | None
+
+    Returns:
+        list[str]: pip-style requirements.
+    """
+    merged: dict[str, Requirement] = {}
+
+    for r in _normalize_requirements(reqs_secondary) + _normalize_requirements(
+        reqs_priority
+    ):
+        req = Requirement(r)
+        merged[canonicalize_name(req.name)] = req
+
+    return [str(req) for req in merged.values()]
+
+
+def get_source_and_working_dir_paths(source_file_path) -> (pathlib.Path, pathlib.Path):
+    source_file_path_object = pathlib.Path(source_file_path).resolve()
+    working_dir_path_object = pathlib.Path(".").resolve()
+    return source_file_path_object, working_dir_path_object
+
+
+def get_relative_module_name_from_path(
+    source_file_path_object, working_dir_path_object
+) -> str:
+    relative_path_to_source_file = source_file_path_object.relative_to(
+        working_dir_path_object
+    )
+    return ".".join(relative_path_to_source_file.with_suffix("").parts)

mlrun/utils/logger.py

@@ -393,12 +393,14 @@ def resolve_formatter_by_kind(
 
 
 def create_test_logger(name: str = "mlrun", stream: IO[str] = stdout) -> Logger:
-    return create_logger(
+    logger = create_logger(
         level="debug",
         formatter_kind=FormatterKinds.HUMAN_EXTENDED.name,
         name=name,
         stream=stream,
     )
+    logger._logger.propagate = True  # pass records up to pytest’s handler
+    return logger
 
 
 def create_logger(

mlrun/utils/notifications/notification/base.py

@@ -15,11 +15,29 @@
 import asyncio
 import typing
 from copy import deepcopy
+from typing import Optional
+
+import aiohttp
 
 import mlrun.common.schemas
 import mlrun.lists
 
 
+class TimedHTTPClient:
+    def __init__(self, timeout: Optional[float] = 30.0):
+        """
+        HTTP client wrapper with built-in timeout.
+
+        Args:
+            timeout: Request timeout in seconds (default: 30.0)
+        """
+        self.timeout = aiohttp.ClientTimeout(total=timeout)
+
+    def session(self, **kwargs) -> aiohttp.ClientSession:
+        """Create a new ClientSession with the configured timeout and additional parameters."""
+        return aiohttp.ClientSession(timeout=self.timeout, **kwargs)
+
+
 class NotificationBase:
     def __init__(
         self,

mlrun/utils/notifications/notification/git.py

@@ -16,13 +16,11 @@ import json
 import os
 import typing
 
-import aiohttp
-
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.lists
 
-from .base import NotificationBase
+from .base import NotificationBase, TimedHTTPClient
 
 
 class GitNotification(NotificationBase):
@@ -148,7 +146,7 @@ class GitNotification(NotificationBase):
             }
             url = f"https://{server}/repos/{repo}/issues/{issue}/comments"
 
-        async with aiohttp.ClientSession() as session:
+        async with TimedHTTPClient().session() as session:
             resp = await session.post(url, headers=headers, json={"body": message})
             if not resp.ok:
                 resp_text = await resp.text()

mlrun/utils/notifications/notification/slack.py

@@ -14,14 +14,12 @@
 
 import typing
 
-import aiohttp
-
 import mlrun.common.runtimes.constants as runtimes_constants
 import mlrun.common.schemas
 import mlrun.lists
 import mlrun.utils.helpers
 
-from .base import NotificationBase
+from .base import NotificationBase, TimedHTTPClient
 
 
 class SlackNotification(NotificationBase):
@@ -67,7 +65,7 @@ class SlackNotification(NotificationBase):
 
         data = self._generate_slack_data(message, severity, runs, alert, event_data)
 
-        async with aiohttp.ClientSession() as session:
+        async with TimedHTTPClient().session() as session:
             async with session.post(webhook, json=data) as response:
                 response.raise_for_status()
 

mlrun/utils/notifications/notification/webhook.py

@@ -15,14 +15,13 @@
 import re
 import typing
 
-import aiohttp
 import orjson
 
 import mlrun.common.schemas
 import mlrun.lists
 import mlrun.utils.helpers
 
-from .base import NotificationBase
+from .base import NotificationBase, TimedHTTPClient
 
 
 class WebhookNotification(NotificationBase):
@@ -87,9 +86,7 @@ class WebhookNotification(NotificationBase):
         # we automatically handle it as `ssl=None` for their convenience.
         verify_ssl = verify_ssl and None if url.startswith("https") else None
 
-        async with aiohttp.ClientSession(
-            json_serialize=self._encoder,
-        ) as session:
+        async with TimedHTTPClient().session(json_serialize=self._encoder) as session:
             response = await getattr(session, method)(
                 url,
                 headers=headers,

mlrun/utils/notifications/notification_pusher.py

@@ -308,7 +308,7 @@ class NotificationPusher(_NotificationPusherBase):
                 and retry_count >= max_retries
             ):
                 message += (
-                    "\nRetry limit reached — run has failed after all retry attempts."
+                    "\nRetry limit reached - run has failed after all retry attempts."
                 )
 
         severity = (