mlrun 1.10.0rc16__py3-none-any.whl → 1.10.1rc4__py3-none-any.whl

mlrun/utils/helpers.py

@@ -15,13 +15,13 @@
 import asyncio
 import base64
 import enum
-import functools
 import gzip
 import hashlib
 import inspect
 import itertools
 import json
 import os
+import pathlib
 import re
 import string
 import sys
@@ -46,6 +46,8 @@ import pytz
 import semver
 import yaml
 from dateutil import parser
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
 from pandas import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
@@ -62,6 +64,7 @@ import mlrun_pipelines.models
 import mlrun_pipelines.utils
 from mlrun.common.constants import MYSQL_MEDIUMBLOB_SIZE_BYTES
 from mlrun.common.schemas import ArtifactCategories
+from mlrun.common.schemas.hub import HubSourceType
 from mlrun.config import config
 from mlrun_pipelines.models import PipelineRun
 
@@ -250,6 +253,40 @@ def verify_field_regex(
         return False
 
 
+def validate_function_name(name: str) -> None:
+    """
+    Validate that a function name conforms to Kubernetes DNS-1123 label requirements.
+
+    Function names for Kubernetes resources must:
+    - Be lowercase alphanumeric characters or '-'
+    - Start and end with an alphanumeric character
+    - Be at most 63 characters long
+
+    This validation should be called AFTER normalize_name() has been applied.
+
+    Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
+
+    :param name: The function name to validate (after normalization)
+    :raises MLRunInvalidArgumentError: If the function name is invalid for Kubernetes
+    """
+    if not name:
+        return
+
+    verify_field_regex(
+        "function.metadata.name",
+        name,
+        mlrun.utils.regex.dns_1123_label,
+        raise_on_failure=True,
+        log_message=(
+            f"Function name '{name}' is invalid. "
+            "Kubernetes function names must be DNS-1123 labels: "
+            "lowercase alphanumeric characters or '-', "
+            "starting and ending with an alphanumeric character, "
+            "and at most 63 characters long."
+        ),
+    )
+
+
 def validate_builder_source(
     source: str, pull_at_runtime: bool = False, workdir: Optional[str] = None
 ):
@@ -464,21 +501,49 @@ def to_date_str(d):
     return ""
 
 
-def normalize_name(name: str, verbose: bool = True):
+def normalize_name(name: str):
     # TODO: Must match
     # [a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?
     name = re.sub(r"\s+", "-", name)
     if "_" in name:
-        if verbose:
-            warnings.warn(
-                "Names with underscore '_' are about to be deprecated, use dashes '-' instead. "
-                f"Replacing '{name}' underscores with dashes.",
-                FutureWarning,
-            )
         name = name.replace("_", "-")
     return name.lower()
 
 
+def ensure_batch_job_suffix(
+    function_name: typing.Optional[str],
+) -> tuple[typing.Optional[str], bool, str]:
+    """
+    Ensure that a function name has the batch job suffix appended to prevent database collision.
+
+    This helper is used by to_job() methods in runtimes that convert online functions (serving, local)
+    to batch processing jobs. The suffix prevents the job from overwriting the original function in
+    the database when both are stored with the same (project, name) key.
+
+    :param function_name: The original function name (can be None or empty string)
+
+    :return: A tuple of (modified_name, was_renamed, suffix) where:
+        - modified_name: The function name with the batch suffix (if not already present),
+          or empty string if input was empty
+        - was_renamed: True if the suffix was added, False if it was already present or if name was empty/None
+        - suffix: The suffix value that was used (or would have been used)
+
+    """
+    suffix = mlrun_constants.RESERVED_BATCH_JOB_SUFFIX
+
+    # Handle None or empty string
+    if not function_name:
+        return function_name, False, suffix
+
+    if not function_name.endswith(suffix):
+        return (
+            f"{function_name}{suffix}",
+            True,
+            suffix,
+        )
+    return function_name, False, suffix
+
+
 class LogBatchWriter:
     def __init__(self, func, batch=16, maxtime=5):
         self.batch = batch
@@ -508,9 +573,14 @@ def get_in(obj, keys, default=None):
     if isinstance(keys, str):
         keys = keys.split(".")
     for key in keys:
-        if not obj or key not in obj:
+        if obj is None:
             return default
-        obj = obj[key]
+        if isinstance(obj, dict):
+            if key not in obj:
+                return default
+            obj = obj[key]
+        else:
+            obj = getattr(obj, key, default)
     return obj
 
 
@@ -800,20 +870,35 @@ def remove_tag_from_artifact_uri(uri: str) -> Optional[str]:
         "store://key:tag" => "store://key"
         "store://models/remote-model-project/my_model#0@tree" => unchanged (no tag)
     """
-    return re.sub(r"(?<=/[^/:]\+):[^@^:\s]+(?=(@|\^|$))", "", uri)
+    add_store = False
+    if mlrun.datastore.is_store_uri(uri):
+        uri = uri.removeprefix(DB_SCHEMA + "://")
+        add_store = True
+    uri = re.sub(r"(#[^:@\s]*)?:[^@^:\s]+(?=(@|\^|$))", lambda m: m.group(1) or "", uri)
+    return uri if not add_store else DB_SCHEMA + "://" + uri
+
 
+def check_if_hub_uri(uri: str) -> bool:
+    return uri.startswith(hub_prefix)
 
-def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
+
+def extend_hub_uri_if_needed(
+    uri: str,
+    asset_type: HubSourceType = HubSourceType.functions,
+    file: str = "function.yaml",
+) -> tuple[str, bool]:
     """
-    Retrieve the full uri of the item's yaml in the hub.
+    Retrieve the full uri of an object in the hub.
 
     :param uri: structure: "hub://[<source>/]<item-name>[:<tag>]"
+    :param asset_type:  The type of the hub item (functions, modules, etc.)
+    :param file: The file name inside the hub item directory (default: function.yaml)
 
     :return: A tuple of:
                [0] = Extended URI of item
                [1] =  Is hub item (bool)
     """
-    is_hub_uri = uri.startswith(hub_prefix)
+    is_hub_uri = check_if_hub_uri(uri)
     if not is_hub_uri:
         return uri, is_hub_uri
 
@@ -830,10 +915,10 @@ def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Invalid character '/' in function name or source name"
             ) from exc
-    name = normalize_name(name=name, verbose=False)
+    name = normalize_name(name=name)
     if not source_name:
         # Searching item in all sources
-        sources = db.list_hub_sources(item_name=name, tag=tag)
+        sources = db.list_hub_sources(item_name=name, tag=tag, item_type=asset_type)
         if not sources:
             raise mlrun.errors.MLRunNotFoundError(
                 f"Item={name}, tag={tag} not found in any hub source"
@@ -843,10 +928,10 @@ def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
     else:
         # Specific source is given
         indexed_source = db.get_hub_source(source_name)
-    # hub function directory name are with underscores instead of hyphens
+    # hub directories name are with underscores instead of hyphens
     name = name.replace("-", "_")
-    function_suffix = f"{name}/{tag}/src/function.yaml"
-    return indexed_source.source.get_full_uri(function_suffix), is_hub_uri
+    suffix = f"{name}/{tag}/src/{file}"
+    return indexed_source.source.get_full_uri(suffix, asset_type), is_hub_uri
 
 
 def gen_md_table(header, rows=None):
@@ -914,10 +999,20 @@ def enrich_image_url(
     mlrun_version = config.images_tag or client_version or server_version
     tag = mlrun_version or ""
 
-    # TODO: Remove condition when mlrun/mlrun-kfp image is also supported
-    if "mlrun-kfp" not in image_url:
+    # starting mlrun 1.10.0-rc0 we want to enrich the kfp image with the python version
+    # e.g for 1.9 we have a single mlrun-kfp image that supports only python 3.9
+    enrich_kfp_python_version = (
+        "mlrun-kfp" in image_url
+        and mlrun_version
+        and semver.VersionInfo.is_valid(mlrun_version)
+        and semver.VersionInfo.parse(mlrun_version)
+        >= semver.VersionInfo.parse("1.10.0-rc0")
+    )
+
+    if "mlrun-kfp" not in image_url or enrich_kfp_python_version:
         tag += resolve_image_tag_suffix(
-            mlrun_version=mlrun_version, python_version=client_python_version
+            mlrun_version=mlrun_version,
+            python_version=client_python_version,
         )
 
     # it's an mlrun image if the repository is mlrun
@@ -930,8 +1025,10 @@ def enrich_image_url(
         # use the tag from image URL if available, else fallback to the given tag
         tag = image_tag or tag
         if tag:
+            # Remove '-pyXY' suffix if present, since the compatibility check expects a valid semver string
+            tag_for_compatibility = re.sub(r"-py\d+$", "", tag)
             if mlrun.utils.helpers.validate_component_version_compatibility(
-                "mlrun-client", "1.10.0-rc0", mlrun_client_version=tag
+                "mlrun-client", "1.10.0-rc0", mlrun_client_version=tag_for_compatibility
             ):
                 warnings.warn(
                     "'mlrun/ml-base' image is deprecated in 1.10.0 and will be removed in 1.12.0, "
@@ -943,8 +1040,15 @@ def enrich_image_url(
         else:
             image_url = "mlrun/mlrun"
 
-    if is_mlrun_image and tag and ":" not in image_url:
-        image_url = f"{image_url}:{tag}"
+    if is_mlrun_image and tag:
+        if ":" not in image_url:
+            image_url = f"{image_url}:{tag}"
+        elif enrich_kfp_python_version:
+            # For mlrun-kfp >= 1.10.0-rc0, append python suffix to existing tag
+            python_suffix = resolve_image_tag_suffix(
+                mlrun_version, client_python_version
+            )
+            image_url = f"{image_url}{python_suffix}" if python_suffix else image_url
 
     registry = (
         config.images_registry if is_mlrun_image else config.vendor_images_registry
@@ -1214,55 +1318,6 @@ def get_workflow_url(
     return url
 
 
-def get_kfp_list_runs_filter(
-    project_name: Optional[str] = None,
-    end_date: Optional[str] = None,
-    start_date: Optional[str] = None,
-) -> str:
-    """
-    Generates a filter for listing Kubeflow Pipelines (KFP) runs.
-
-    :param project_name: The name of the project. If "*", it won't filter by project.
-    :param end_date: The latest creation date for filtering runs (ISO 8601 format).
-    :param start_date: The earliest creation date for filtering runs (ISO 8601 format).
-    :return: A JSON-formatted filter string for KFP.
-    """
-
-    # KFP filter operation codes
-    kfp_less_than_or_equal_op = 7  # '<='
-    kfp_greater_than_or_equal_op = 5  # '>='
-    kfp_substring_op = 9  # Substring match
-
-    filters = {"predicates": []}
-
-    if end_date:
-        filters["predicates"].append(
-            {
-                "key": "created_at",
-                "op": kfp_less_than_or_equal_op,
-                "timestamp_value": end_date,
-            }
-        )
-
-    if project_name and project_name != "*":
-        filters["predicates"].append(
-            {
-                "key": "name",
-                "op": kfp_substring_op,
-                "string_value": project_name,
-            }
-        )
-    if start_date:
-        filters["predicates"].append(
-            {
-                "key": "created_at",
-                "op": kfp_greater_than_or_equal_op,
-                "timestamp_value": start_date,
-            }
-        )
-    return json.dumps(filters)
-
-
 def validate_and_convert_date(date_input: str) -> str:
     """
     Converts any recognizable date string into a standardized RFC 3339 format.
@@ -1860,10 +1915,7 @@ async def run_in_threadpool(func, *args, **kwargs):
     Run a sync-function in the loop default thread pool executor pool and await its result.
     Note that this function is not suitable for CPU-bound tasks, as it will block the event loop.
     """
-    loop = asyncio.get_running_loop()
-    if kwargs:
-        func = functools.partial(func, **kwargs)
-    return await loop.run_in_executor(None, func, *args)
+    return await asyncio.to_thread(func, *args, **kwargs)
 
 
 def is_explicit_ack_supported(context):
@@ -2404,9 +2456,7 @@ def split_path(path: str) -> typing.Union[str, list[str], None]:
     return path
 
 
-def get_data_from_path(
-    path: typing.Union[str, list[str], None], data: dict
-) -> dict[str, Any]:
+def get_data_from_path(path: typing.Union[str, list[str], None], data: dict) -> Any:
     if isinstance(path, str):
         output_data = data.get(path)
     elif isinstance(path, list):
@@ -2419,6 +2469,88 @@ def get_data_from_path(
         raise mlrun.errors.MLRunInvalidArgumentError(
             "Expected path be of type str or list of str or None"
         )
-    if isinstance(output_data, (int, float)):
-        output_data = [output_data]
     return output_data
+
+
+def is_valid_port(port: int, raise_on_error: bool = False) -> bool:
+    if not port:
+        return False
+    if 0 <= port <= 65535:
+        return True
+    if raise_on_error:
+        raise ValueError("Port must be in the range 0–65535")
+    return False
+
+
+def set_data_by_path(
+    path: typing.Union[str, list[str], None], data: dict, value
+) -> None:
+    if path is None:
+        if not isinstance(value, dict):
+            raise ValueError("When path is None, value must be a dictionary.")
+        data.update(value)
+
+    elif isinstance(path, str):
+        data[path] = value
+
+    elif isinstance(path, list):
+        current = data
+        for key in path[:-1]:
+            if key not in current or not isinstance(current[key], dict):
+                current[key] = {}
+            current = current[key]
+        current[path[-1]] = value
+    else:
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            "Expected path to be of type str or list of str"
+        )
+
+
+def _normalize_requirements(reqs: typing.Union[str, list[str], None]) -> list[str]:
+    if reqs is None:
+        return []
+    if isinstance(reqs, str):
+        s = reqs.strip()
+        return [s] if s else []
+    return [s.strip() for s in reqs if s and s.strip()]
+
+
+def merge_requirements(
+    reqs_priority: typing.Union[str, list[str], None],
+    reqs_secondary: typing.Union[str, list[str], None],
+) -> list[str]:
+    """
+    Merge two requirement collections into a union. If the same package
+    appears in both, the specifier from reqs_priority wins.
+
+    Args:
+        reqs_priority: str | list[str] | None  (priority input)
+        reqs_secondary: str | list[str] | None
+
+    Returns:
+        list[str]: pip-style requirements.
+    """
+    merged: dict[str, Requirement] = {}
+
+    for r in _normalize_requirements(reqs_secondary) + _normalize_requirements(
+        reqs_priority
+    ):
+        req = Requirement(r)
+        merged[canonicalize_name(req.name)] = req
+
+    return [str(req) for req in merged.values()]
+
+
+def get_source_and_working_dir_paths(source_file_path) -> (pathlib.Path, pathlib.Path):
+    source_file_path_object = pathlib.Path(source_file_path).resolve()
+    working_dir_path_object = pathlib.Path(".").resolve()
+    return source_file_path_object, working_dir_path_object
+
+
+def get_relative_module_name_from_path(
+    source_file_path_object, working_dir_path_object
+) -> str:
+    relative_path_to_source_file = source_file_path_object.relative_to(
+        working_dir_path_object
+    )
+    return ".".join(relative_path_to_source_file.with_suffix("").parts)

mlrun/utils/logger.py

@@ -393,12 +393,14 @@ def resolve_formatter_by_kind(
 
 
 def create_test_logger(name: str = "mlrun", stream: IO[str] = stdout) -> Logger:
-    return create_logger(
+    logger = create_logger(
         level="debug",
         formatter_kind=FormatterKinds.HUMAN_EXTENDED.name,
         name=name,
         stream=stream,
     )
+    logger._logger.propagate = True  # pass records up to pytest’s handler
+    return logger
 
 
 def create_logger(

mlrun/utils/notifications/notification/base.py

@@ -15,11 +15,29 @@
 import asyncio
 import typing
 from copy import deepcopy
+from typing import Optional
+
+import aiohttp
 
 import mlrun.common.schemas
 import mlrun.lists
 
 
+class TimedHTTPClient:
+    def __init__(self, timeout: Optional[float] = 30.0):
+        """
+        HTTP client wrapper with built-in timeout.
+
+        Args:
+            timeout: Request timeout in seconds (default: 30.0)
+        """
+        self.timeout = aiohttp.ClientTimeout(total=timeout)
+
+    def session(self, **kwargs) -> aiohttp.ClientSession:
+        """Create a new ClientSession with the configured timeout and additional parameters."""
+        return aiohttp.ClientSession(timeout=self.timeout, **kwargs)
+
+
 class NotificationBase:
     def __init__(
         self,

mlrun/utils/notifications/notification/git.py

@@ -16,13 +16,11 @@ import json
 import os
 import typing
 
-import aiohttp
-
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.lists
 
-from .base import NotificationBase
+from .base import NotificationBase, TimedHTTPClient
 
 
 class GitNotification(NotificationBase):
@@ -148,7 +146,7 @@ class GitNotification(NotificationBase):
             }
             url = f"https://{server}/repos/{repo}/issues/{issue}/comments"
 
-        async with aiohttp.ClientSession() as session:
+        async with TimedHTTPClient().session() as session:
             resp = await session.post(url, headers=headers, json={"body": message})
             if not resp.ok:
                 resp_text = await resp.text()

mlrun/utils/notifications/notification/mail.py

@@ -34,17 +34,18 @@ class MailNotification(base.NotificationBase):
 
     boolean_params = ["use_tls", "start_tls", "validate_certs"]
 
+    optional_auth_params = ["username", "password"]
+
     required_params = [
         "server_host",
         "server_port",
         "sender_address",
-        "username",
-        "password",
         "email_addresses",
     ] + boolean_params
 
     @classmethod
     def validate_params(cls, params):
+        cls._enrich_params(params)
         for required_param in cls.required_params:
             if required_param not in params:
                 raise ValueError(
@@ -57,6 +58,13 @@ class MailNotification(base.NotificationBase):
                     f"Parameter '{boolean_param}' must be a boolean for MailNotification"
                 )
 
+        # Allow no auth, username only, or username + password
+        # Some SMTP servers allow username without password
+        if params["password"] and not params["username"]:
+            raise ValueError(
+                "Parameter 'username' is required when 'password' is provided for MailNotification"
+            )
+
         cls._validate_emails(params)
 
     async def push(
@@ -78,6 +86,8 @@ class MailNotification(base.NotificationBase):
         )
         self.params["body"] = runs_html
 
+        self._enrich_params(self.params)
+
         if message_body_override:
             self.params["body"] = message_body_override.replace(
                 "{{ runs }}", runs_html
@@ -147,8 +157,8 @@ class MailNotification(base.NotificationBase):
         sender_address: str,
         server_host: str,
         server_port: int,
-        username: str,
-        password: str,
+        username: typing.Optional[str],
+        password: typing.Optional[str],
         use_tls: bool,
         start_tls: bool,
         validate_certs: bool,
@@ -163,14 +173,27 @@ class MailNotification(base.NotificationBase):
         message["Subject"] = subject
         message.attach(MIMEText(body, "html"))
 
-        # Send the email
-        await aiosmtplib.send(
-            message,
-            hostname=server_host,
-            port=server_port,
-            username=username,
-            password=password,
-            use_tls=use_tls,
-            validate_certs=validate_certs,
-            start_tls=start_tls,
-        )
+        send_kwargs = {
+            "hostname": server_host,
+            "port": server_port,
+            "use_tls": use_tls,
+            "validate_certs": validate_certs,
+            "start_tls": start_tls,
+        }
+
+        # Only include auth parameters when provided to avoid forcing SMTP AUTH
+        if username is not None:
+            send_kwargs["username"] = username
+        if password is not None:
+            send_kwargs["password"] = password
+
+        await aiosmtplib.send(message, **send_kwargs)
+
+    @staticmethod
+    def _enrich_params(params):
+        # if username/password are not provided or empty strings, set them to None.
+        # this ensures consistent behavior in _send_email and avoids
+        # forcing SMTP auth when the server does not require authentication.
+        for param in ["username", "password"]:
+            if param not in params or not params[param]:
+                params[param] = None

mlrun/utils/notifications/notification/slack.py

@@ -14,14 +14,12 @@
 
 import typing
 
-import aiohttp
-
 import mlrun.common.runtimes.constants as runtimes_constants
 import mlrun.common.schemas
 import mlrun.lists
 import mlrun.utils.helpers
 
-from .base import NotificationBase
+from .base import NotificationBase, TimedHTTPClient
 
 
 class SlackNotification(NotificationBase):
@@ -67,7 +65,7 @@ class SlackNotification(NotificationBase):
 
         data = self._generate_slack_data(message, severity, runs, alert, event_data)
 
-        async with aiohttp.ClientSession() as session:
+        async with TimedHTTPClient().session() as session:
             async with session.post(webhook, json=data) as response:
                 response.raise_for_status()
 

mlrun/utils/notifications/notification/webhook.py

@@ -15,14 +15,13 @@
 import re
 import typing
 
-import aiohttp
 import orjson
 
 import mlrun.common.schemas
 import mlrun.lists
 import mlrun.utils.helpers
 
-from .base import NotificationBase
+from .base import NotificationBase, TimedHTTPClient
 
 
 class WebhookNotification(NotificationBase):
@@ -87,9 +86,7 @@ class WebhookNotification(NotificationBase):
         # we automatically handle it as `ssl=None` for their convenience.
         verify_ssl = verify_ssl and None if url.startswith("https") else None
 
-        async with aiohttp.ClientSession(
-            json_serialize=self._encoder,
-        ) as session:
+        async with TimedHTTPClient().session(json_serialize=self._encoder) as session:
             response = await getattr(session, method)(
                 url,
                 headers=headers,

mlrun/utils/notifications/notification_pusher.py

@@ -308,7 +308,7 @@ class NotificationPusher(_NotificationPusherBase):
                 and retry_count >= max_retries
             ):
                 message += (
-                    "\nRetry limit reached — run has failed after all retry attempts."
+                    "\nRetry limit reached - run has failed after all retry attempts."
                 )
 
         severity = (

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "78045e1e85e7c81eee93682240c4ebe7b22fa67c",
-  "version": "1.10.0-rc16"
+  "git_commit": "d48a638b75d73b43de9731dc297755e2c0c86084",
+  "version": "1.10.1-rc4"
 }