mlrun 1.10.0rc16__py3-none-any.whl → 1.10.1rc4__py3-none-any.whl

mlrun/run.py

@@ -17,6 +17,7 @@ import json
 import os
 import pathlib
 import socket
+import sys
 import tempfile
 import time
 import typing
@@ -117,14 +118,31 @@ def function_to_module(code="", workdir=None, secrets=None, silent=False):
         raise ValueError("nothing to run, specify command or function")
 
     command = os.path.join(workdir or "", command)
-    path = Path(command)
-    mod_name = path.name
-    if path.suffix:
-        mod_name = mod_name[: -len(path.suffix)]
+
+    source_file_path_object, working_dir_path_object = (
+        mlrun.utils.helpers.get_source_and_working_dir_paths(command)
+    )
+    if source_file_path_object.is_relative_to(working_dir_path_object):
+        mod_name = mlrun.utils.helpers.get_relative_module_name_from_path(
+            source_file_path_object, working_dir_path_object
+        )
+    elif source_file_path_object.is_relative_to(
+        pathlib.Path(tempfile.gettempdir()).resolve()
+    ):
+        mod_name = Path(command).stem
+    else:
+        raise mlrun.errors.MLRunRuntimeError(
+            f"Cannot run source file '{command}': it must be located either under the current working "
+            f"directory ('{working_dir_path_object}') or the system temporary directory ('{tempfile.gettempdir()}'). "
+            f"This is required when running with local=True."
+        )
+
     spec = imputil.spec_from_file_location(mod_name, command)
     if spec is None:
         raise OSError(f"cannot import from {command!r}")
     mod = imputil.module_from_spec(spec)
+    # add to system modules, which can be necessary when running in a MockServer (ML-10937)
+    sys.modules[mod_name] = mod
     spec.loader.exec_module(mod)
 
     return mod
@@ -141,7 +159,7 @@ def load_func_code(command="", workdir=None, secrets=None, name="name"):
         else:
             is_remote = "://" in command
             data = get_object(command, secrets)
-            runtime = yaml.load(data, Loader=yaml.FullLoader)
+            runtime = yaml.safe_load(data)
             runtime = new_function(runtime=runtime)
 
         command = runtime.spec.command or ""
@@ -222,7 +240,8 @@ def get_or_create_ctx(
     :param spec:     dictionary holding run spec
     :param with_env: look for context in environment vars, default True
     :param rundb:    path/url to the metadata and artifact database
-    :param project:  project to initiate the context in (by default `mlrun.mlconf.active_project`)
+    :param project:  project to initiate the context in (by default `mlrun.mlconf.active_project`).
+                              If not set, an active project must exist.
     :param upload_artifacts:  when using local context (not as part of a job/run), upload artifacts to the
                               system default artifact path location
     :return: execution context
@@ -277,6 +296,16 @@ def get_or_create_ctx(
     if newspec and not isinstance(newspec, dict):
         newspec = json.loads(newspec)
 
+    if (
+        not newspec.get("metadata", {}).get("project")
+        and not project
+        and not mlconf.active_project
+    ):
+        raise mlrun.errors.MLRunMissingProjectError(
+            """No active project found. Make sure to set an active project using: mlrun.get_or_create_project()
+            You can verify the active project with: mlrun.mlconf.active_project"""
+        )
+
     if not newspec:
         newspec = {}
         if upload_artifacts:
@@ -316,7 +345,7 @@ def get_or_create_ctx(
 def import_function(url="", secrets=None, db="", project=None, new_name=None):
     """Create function object from DB or local/remote YAML file
 
-    Functions can be imported from function repositories (mlrun Function Hub (formerly Marketplace) or local db),
+    Functions can be imported from function repositories (MLRun Hub) or local db),
     or be read from a remote URL (http(s), s3, git, v3io, ..) containing the function YAML
 
     special URLs::
@@ -332,7 +361,7 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
             "https://raw.githubusercontent.com/org/repo/func.yaml"
         )
 
-    :param url: path/url to Function Hub, db or function YAML file
+    :param url: path/url to MLRun Hub, db or function YAML file
     :param secrets: optional, credentials dict for DB or URL (s3, v3io, ...)
     :param db: optional, mlrun api/db path
     :param project: optional, target project for the function
@@ -362,10 +391,13 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
     return function
 
 
-def import_function_to_dict(url, secrets=None):
+def import_function_to_dict(
+    url: str,
+    secrets: Optional[dict] = None,
+) -> dict:
     """Load function spec from local/remote YAML file"""
     obj = get_object(url, secrets)
-    runtime = yaml.load(obj, Loader=yaml.FullLoader)
+    runtime = yaml.safe_load(obj)
     remote = "://" in url
 
     code = get_in(runtime, "spec.build.functionSourceCode")
@@ -388,20 +420,40 @@ def import_function_to_dict(url, secrets=None):
                 raise ValueError("exec path (spec.command) must be relative")
             url = url[: url.rfind("/") + 1] + code_file
             code = get_object(url, secrets)
+            code_file = _ensure_path_confined_to_base_dir(
+                base_directory=".",
+                relative_path=code_file,
+                error_message_on_escape="Path traversal detected in spec.command",
+            )
             dir = path.dirname(code_file)
             if dir:
                 makedirs(dir, exist_ok=True)
             with open(code_file, "wb") as fp:
                 fp.write(code)
         elif cmd:
-            if not path.isfile(code_file):
-                # look for the file in a relative path to the yaml
-                slash = url.rfind("/")
-                if slash >= 0 and path.isfile(url[: url.rfind("/") + 1] + code_file):
-                    raise ValueError(
-                        f"exec file spec.command={code_file} is relative, change working dir"
-                    )
+            slash_index = url.rfind("/")
+            if slash_index < 0:
+                raise ValueError(f"no file in exec path (spec.command={code_file})")
+            base_dir = os.path.normpath(url[: slash_index + 1])
+
+            # Validate and resolve the candidate path before checking existence
+            candidate_path = _ensure_path_confined_to_base_dir(
+                base_directory=base_dir,
+                relative_path=code_file,
+                error_message_on_escape=(
+                    f"exec file spec.command={code_file} is outside of allowed directory"
+                ),
+            )
+
+            # Only now it's safe to check file existence
+            if not path.isfile(candidate_path):
                 raise ValueError(f"no file in exec path (spec.command={code_file})")
+
+            # Check that the path is absolute
+            if not os.path.isabs(code_file):
+                raise ValueError(
+                    f"exec file spec.command={code_file} is relative, it must be absolute. Change working dir"
+                )
         else:
             raise ValueError("command or code not specified in function spec")
 
@@ -503,6 +555,7 @@ def new_function(
 
     # make sure function name is valid
     name = mlrun.utils.helpers.normalize_name(name)
+    mlrun.utils.helpers.validate_function_name(name)
 
     runner.metadata.name = name
     runner.metadata.project = (
@@ -542,6 +595,7 @@ def new_function(
         )
 
     runner.prepare_image_for_deploy()
+
     return runner
 
 
@@ -575,7 +629,7 @@ def code_to_function(
     code_output: Optional[str] = "",
     embed_code: bool = True,
     description: Optional[str] = "",
-    requirements: Optional[Union[str, list[str]]] = None,
+    requirements: Optional[list[str]] = None,
     categories: Optional[list[str]] = None,
     labels: Optional[dict[str, str]] = None,
     with_doc: Optional[bool] = True,
@@ -638,7 +692,7 @@ def code_to_function(
     :param description:  short function description, defaults to ''
     :param requirements: a list of python packages
     :param requirements_file: path to a python requirements file
-    :param categories:   list of categories for mlrun Function Hub, defaults to None
+    :param categories:   list of categories for MLRun Hub, defaults to None
     :param labels:       name/value pairs dict to tag the function with useful metadata, defaults to None
     :param with_doc:     indicates whether to document the function parameters, defaults to True
     :param ignored_tags: notebook cells to ignore when converting notebooks to py code (separated by ';')
@@ -746,6 +800,7 @@ def code_to_function(
         kind=sub_kind,
         ignored_tags=ignored_tags,
     )
+
     spec["spec"]["env"].append(
         {
             "name": "MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK",
@@ -798,6 +853,7 @@ def code_to_function(
         runtime.spec.build.code_origin = code_origin
         runtime.spec.build.origin_filename = filename or (name + ".ipynb")
         update_common(runtime, spec)
+
         return runtime
 
     if kind is None or kind in ["", "Function"]:
@@ -811,6 +867,7 @@ def code_to_function(
 
     if not name:
         raise ValueError("name must be specified")
+
     h = get_in(spec, "spec.handler", "").split(":")
     runtime.handler = h[0] if len(h) <= 1 else h[1]
     runtime.metadata = get_in(spec, "spec.metadata")
@@ -1184,11 +1241,13 @@ def get_model_provider(
     raise_missing_schema_exception=True,
 ) -> ModelProvider:
     """get mlrun dataitem object (from path/url)"""
-    store_manager.set(secrets, db=db)
+    #  without caching secrets
+    store_manager.set(db=db)
     return store_manager.model_provider_object(
         url=url,
         default_invoke_kwargs=default_invoke_kwargs,
         raise_missing_schema_exception=raise_missing_schema_exception,
+        secrets=secrets,
     )
 
 
@@ -1256,3 +1315,21 @@ def wait_for_runs_completion(
         runs = running
 
     return completed
+
+
+def _ensure_path_confined_to_base_dir(
+    base_directory: str,
+    relative_path: str,
+    error_message_on_escape: str,
+) -> str:
+    """
+    Join `user_supplied_relative_path` to `allowed_base_directory`, normalise the result,
+    and guarantee it stays inside `allowed_base_directory`.
+    """
+    absolute_base_directory = path.abspath(base_directory)
+    absolute_candidate_path = path.abspath(
+        path.join(absolute_base_directory, relative_path)
+    )
+    if not absolute_candidate_path.startswith(absolute_base_directory + path.sep):
+        raise ValueError(error_message_on_escape)
+    return absolute_candidate_path

mlrun/runtimes/__init__.py

@@ -221,6 +221,24 @@ class RuntimeKinds:
             return True
         return False
 
+    @staticmethod
+    def requires_k8s_name_validation(kind: str) -> bool:
+        """
+        Returns True if the runtime kind creates Kubernetes resources that use the function name.
+
+        Function names for k8s-deployed runtimes must conform to DNS-1123 label requirements:
+        - Lowercase alphanumeric characters or '-'
+        - Start and end with an alphanumeric character
+        - Maximum 63 characters
+
+        Local runtimes (local, handler) run on the local machine and don't create k8s resources,
+        so they don't require k8s naming validation.
+
+        :param kind: Runtime kind string (job, spark, serving, local, etc.)
+        :return: True if function name needs k8s DNS-1123 validation, False otherwise
+        """
+        return not RuntimeKinds.is_local_runtime(kind)
+
     @staticmethod
     def requires_absolute_artifacts_path(kind):
         """

mlrun/runtimes/base.py

@@ -142,9 +142,6 @@ class FunctionSpec(ModelObj):
     def build(self, build):
         self._build = self._verify_dict(build, "build", ImageBuilder)
 
-    def enrich_function_preemption_spec(self):
-        pass
-
     def validate_service_account(self, allowed_service_accounts):
         pass
 
@@ -379,7 +376,12 @@ class BaseRuntime(ModelObj):
                              This ensures latest code changes are executed. This argument must be used in
                              conjunction with the local=True argument.
         :param output_path:    Default artifact output path.
-        :param retry:          Retry configuration for the run, can be a dict or an instance of mlrun.model.Retry.
+        :param retry:          Retry configuration for the run, can be a dict or an instance of
+                               :py:class:`~mlrun.model.Retry`.
+                               The `count` field in the `Retry` object specifies the number of retry attempts.
+                               If `count=0`, the run will not be retried.
+                               The `backoff` field specifies the retry backoff strategy between retry attempts.
+                               If not provided, the default backoff delay is 30 seconds.
         :return: Run context object (RunObject) with run metadata, results and status
         """
         if artifact_path or out_path:
@@ -391,6 +393,7 @@ class BaseRuntime(ModelObj):
                 FutureWarning,
             )
         output_path = output_path or out_path or artifact_path
+
         launcher = mlrun.launcher.factory.LauncherFactory().create_launcher(
             self._is_remote, local=local, **launcher_kwargs
         )
@@ -446,15 +449,20 @@ class BaseRuntime(ModelObj):
         :param runobj: Run context object (RunObject) with run metadata and status
         :return: Dictionary with all the variables that could be parsed
         """
+        active_project = self.metadata.project or config.active_project
         runtime_env = {
-            "MLRUN_ACTIVE_PROJECT": self.metadata.project or config.active_project
+            mlrun_constants.MLRUN_ACTIVE_PROJECT: active_project,
+            # TODO: Remove this in 1.12.0 as MLRUN_DEFAULT_PROJECT is deprecated and should not be injected anymore
+            "MLRUN_DEFAULT_PROJECT": active_project,
         }
         if runobj:
             runtime_env["MLRUN_EXEC_CONFIG"] = runobj.to_json(
                 exclude_notifications_params=True
             )
             if runobj.metadata.project:
-                runtime_env["MLRUN_ACTIVE_PROJECT"] = runobj.metadata.project
+                runtime_env[mlrun_constants.MLRUN_ACTIVE_PROJECT] = (
+                    runobj.metadata.project
+                )
             if runobj.spec.verbose:
                 runtime_env["MLRUN_LOG_LEVEL"] = "DEBUG"
         if config.httpdb.api_url:

mlrun/runtimes/daskjob.py

@@ -541,6 +541,7 @@ class DaskCluster(KubejobRuntime):
             notifications=notifications,
             returns=returns,
             state_thresholds=state_thresholds,
+            retry=retry,
             **launcher_kwargs,
         )
 

mlrun/runtimes/local.py

@@ -29,12 +29,12 @@ from os import environ, remove
 from pathlib import Path
 from subprocess import PIPE, Popen
 from sys import executable
+from typing import Optional
 
 from nuclio import Event
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
-import mlrun.common.runtimes.constants
 from mlrun.lists import RunList
 
 from ..errors import err_to_str
@@ -201,9 +201,12 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
     kind = "local"
     _is_remote = False
 
-    def to_job(self, image=""):
+    def to_job(self, image="", func_name: Optional[str] = None):
         struct = self.to_dict()
         obj = KubejobRuntime.from_dict(struct)
+        obj.kind = "job"  # Ensure kind is set to 'job' for KubejobRuntime
+        if func_name:
+            obj.metadata.name = func_name
         if image:
             obj.spec.image = image
         return obj

mlrun/runtimes/mounts.py

@@ -14,8 +14,11 @@
 
 import os
 import typing
+import warnings
 from collections import namedtuple
 
+import mlrun.common.secrets
+import mlrun.errors
 from mlrun.config import config
 from mlrun.config import config as mlconf
 from mlrun.errors import MLRunInvalidArgumentError
@@ -247,10 +250,22 @@ def mount_s3(
     def _use_s3_cred(runtime: "KubeResource"):
         _access_key = aws_access_key or os.environ.get(prefix + "AWS_ACCESS_KEY_ID")
         _secret_key = aws_secret_key or os.environ.get(prefix + "AWS_SECRET_ACCESS_KEY")
-        _endpoint_url = endpoint_url or os.environ.get(prefix + "S3_ENDPOINT_URL")
+
+        # Check for endpoint URL with backward compatibility
+        _endpoint_url = endpoint_url or os.environ.get(prefix + "AWS_ENDPOINT_URL_S3")
+        if not _endpoint_url:
+            # Check for deprecated environment variable
+            _endpoint_url = os.environ.get(prefix + "S3_ENDPOINT_URL")
+            if _endpoint_url:
+                warnings.warn(
+                    "S3_ENDPOINT_URL is deprecated in 1.10.0 and will be removed in 1.12.0, "
+                    "use AWS_ENDPOINT_URL_S3 instead.",
+                    # TODO: Remove this in 1.12.0
+                    FutureWarning,
+                )
 
         if _endpoint_url:
-            runtime.set_env(prefix + "S3_ENDPOINT_URL", _endpoint_url)
+            runtime.set_env(prefix + "AWS_ENDPOINT_URL_S3", _endpoint_url)
         if aws_region:
             runtime.set_env(prefix + "AWS_REGION", aws_region)
         if non_anonymous:
@@ -399,6 +414,9 @@ def mount_secret(
                          the specified paths, and unlisted keys will not be
                          present."""
 
+    if secret_name:
+        mlrun.common.secrets.validate_not_forbidden_secret(secret_name.strip())
+
     def _mount_secret(runtime: "KubeResource"):
         # Define the secret volume source
         secret_volume_source = {

mlrun/runtimes/nuclio/__init__.py

@@ -16,6 +16,7 @@ from .serving import ServingRuntime, new_v2_model_server  # noqa
 from .nuclio import nuclio_init_hook  # noqa
 from .function import (
     min_nuclio_versions,
+    multiple_port_sidecar_is_supported,
     RemoteRuntime,
 )  # noqa
 from .api_gateway import APIGateway