mlrun 1.10.0rc18__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/runtimes/utils.py

@@ -26,6 +26,7 @@ import pandas as pd
 import mlrun
 import mlrun.common.constants
 import mlrun.common.constants as mlrun_constants
+import mlrun.common.runtimes.constants
 import mlrun.common.schemas
 import mlrun.utils.regex
 from mlrun.artifacts import TableArtifact
@@ -153,6 +154,7 @@ def results_to_iter(results, runspec, execution):
 
     iter = []
     failed = 0
+    pending_retry = 0
     running = 0
     for task in results:
         if task:
@@ -164,17 +166,26 @@ def results_to_iter(results, runspec, execution):
                 "state": state,
                 "iter": id,
             }
-            if state == "error":
+            if state == mlrun.common.runtimes.constants.RunStates.error:
                 failed += 1
                 err = get_in(task, ["status", "error"], "")
-                logger.error(f"error in task  {execution.uid}:{id} - {err_to_str(err)}")
-            elif state != "completed":
+                logger.error(f"error in task {execution.uid}:{id} - {err_to_str(err)}")
+            elif state == mlrun.common.runtimes.constants.RunStates.pending_retry:
+                pending_retry += 1
+                err = get_in(task, ["status", "error"], "")
+                retry_count = get_in(task, ["status", "retry_count"], 0)
+                logger.warning(
+                    f"pending retry in task {execution.uid}:{id} - {err_to_str(err)}. Retry count: {retry_count}"
+                )
+            elif state != mlrun.common.runtimes.constants.RunStates.completed:
                 running += 1
 
             iter.append(struct)
 
     if not iter:
-        execution.set_state("completed", commit=True)
+        execution.set_state(
+            mlrun.common.runtimes.constants.RunStates.completed, commit=True
+        )
         logger.warning("Warning!, zero iteration results")
         return
     if hasattr(pd, "json_normalize"):
@@ -204,8 +215,14 @@ def results_to_iter(results, runspec, execution):
             error=f"{failed} of {len(results)} tasks failed, check logs in db for details",
             commit=False,
         )
+    elif pending_retry:
+        execution.set_state(
+            mlrun.common.runtimes.constants.RunStates.pending_retry, commit=False
+        )
     elif running == 0:
-        execution.set_state("completed", commit=False)
+        execution.set_state(
+            mlrun.common.runtimes.constants.RunStates.completed, commit=False
+        )
     execution.commit()
 
 
@@ -431,22 +448,45 @@ def enrich_function_from_dict(function, function_dict):
     return function
 
 
+def resolve_owner(
+    labels: dict,
+    owner_to_enrich: Optional[str] = None,
+):
+    """
+    Resolve the owner label value
+    :param labels: The run labels dict
+    :param auth_username: The authenticated username
+    :return: The resolved owner label value
+    """
+
+    if owner_to_enrich and (
+        labels.get("job-type") == mlrun.common.constants.JOB_TYPE_WORKFLOW_RUNNER
+        or labels.get("job-type")
+        == mlrun.common.constants.JOB_TYPE_RERUN_WORKFLOW_RUNNER
+    ):
+        return owner_to_enrich
+    else:
+        return os.environ.get("V3IO_USERNAME") or getpass.getuser()
+
+
 def enrich_run_labels(
     labels: dict,
     labels_to_enrich: Optional[list[mlrun_constants.MLRunInternalLabels]] = None,
+    owner_to_enrich: Optional[str] = None,
 ):
     """
-    Enrich the run labels with the internal labels and the labels enrichment extension
+    Enrich the run labels with the internal labels and the labels enrichment extension.
     :param labels: The run labels dict
     :param labels_to_enrich: The label keys to enrich from MLRunInternalLabels.default_run_labels_to_enrich
+    :param owner_to_enrich: Optional owner to enrich the labels with, if not provided will try to resolve it.
     :return: The enriched labels dict
     """
     # Merge the labels with the labels enrichment extension
     labels_enrichment = {
-        mlrun_constants.MLRunInternalLabels.owner: os.environ.get("V3IO_USERNAME")
-        or getpass.getuser(),
+        mlrun_constants.MLRunInternalLabels.owner: resolve_owner(
+            labels, owner_to_enrich
+        ),
     }
-
     # Resolve which label keys to enrich
     if labels_to_enrich is None:
         labels_to_enrich = (

mlrun/secrets.py

@@ -11,10 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import json
+import os
 from ast import literal_eval
-from os import environ, getenv
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from os import environ
+from typing import Optional, Union
+
+import mlrun.auth.utils
+import mlrun.utils.helpers
+from mlrun.config import is_running_as_api
 
 from .utils import AzureVaultStore, list2dict
 
@@ -48,6 +54,11 @@ class SecretsStore:
                 self._secrets[prefix + k] = str(v)
 
         elif kind == "file":
+            # Ensure files cannot be open from inside the API
+            if is_running_as_api():
+                raise RuntimeError(
+                    "add_source of kind 'file' is not allowed from the API"
+                )
             with open(source) as fp:
                 lines = fp.read().splitlines()
                 secrets_dict = list2dict(lines)
@@ -161,6 +172,9 @@ def get_secret_or_env(
     4. An MLRun-generated env. variable, mounted from a project secret (to be used in MLRun runtimes)
     5. The default value
 
+    Also supports discovering the value inside any environment variable that contains a JSON-encoded list
+    of dicts with fields: {'name': 'KEY', 'value': 'VAL', 'value_from': ...}. This fallback is applied
+    after checking normal environment variables and before returning the default.
     Example::
 
         secrets = {"KEY1": "VALUE1"}
@@ -187,18 +201,89 @@ def get_secret_or_env(
     if prefix:
         key = f"{prefix}_{key}"
 
-    value = None
     if secret_provider:
-        if isinstance(secret_provider, (dict, SecretsStore)):
-            value = secret_provider.get(key)
+        if isinstance(secret_provider, dict | SecretsStore):
+            secret_value = secret_provider.get(key)
         else:
-            value = secret_provider(key)
-        if value:
-            return value
+            secret_value = secret_provider(key)
+        if secret_value:
+            return secret_value
+
+    direct_environment_value = environ.get(key)
+    if direct_environment_value:
+        return direct_environment_value
+
+    json_list_value = _find_value_in_json_env_lists(key)
+    if json_list_value is not None:
+        return json_list_value
 
-    return (
-        value
-        or getenv(key)
-        or getenv(SecretsStore.k8s_env_variable_name_for_secret(key))
-        or default
+    mlrun_env_key = SecretsStore.k8s_env_variable_name_for_secret(key)
+    mlrun_env_value = environ.get(mlrun_env_key)
+    if mlrun_env_value:
+        return mlrun_env_value
+
+    return default
+
+
+def _find_value_in_json_env_lists(
+    secret_name: str,
+) -> Optional[str]:
+    """
+    Scan all environment variables. If any env var contains a JSON-encoded list
+    of dicts shaped like {'name': str, 'value': str|None, 'value_from': ...},
+    return the 'value' for the entry whose 'name' matches secret_name.
+    """
+    for environment_variable_value in environ.values():
+        if not environment_variable_value or not isinstance(
+            environment_variable_value, str
+        ):
+            continue
+        # Fast precheck to skip obvious non-JSON strings
+        first_char = environment_variable_value.lstrip()[:1]
+        if first_char not in ("[", "{"):
+            continue
+        try:
+            parsed_value = json.loads(environment_variable_value)
+        except ValueError:
+            continue
+        if isinstance(parsed_value, list):
+            for entry in parsed_value:
+                if isinstance(entry, dict) and entry.get("name") == secret_name:
+                    value_in_entry = entry.get("value")
+                    # Match original semantics: empty string is treated as "not found"
+                    if value_in_entry:
+                        return value_in_entry
+    return None
+
+
+@mlrun.utils.iguazio_v4_only
+def sync_secret_tokens() -> None:
+    """
+    Synchronize local secret tokens with the backend. Doesn't sync when running from a runtime.
+
+    This function:
+      1. Reads the local token file (defaults to `mlrun.mlconf.auth_with_oauth_token.token_file` value).
+      2. Validates its content and converts validated tokens into `SecretToken` objects.
+      3. Uploads the tokens to the backend.
+      4. Logs a warning if any tokens were updated on the backend due to newer
+         expiration times found locally.
+    """
+
+    # Do not sync tokens from the file when using the offline token environment variable.
+    # The offline token from the env var takes precedence over the file.
+    # Using the env var is not the recommended approach, and tokens from the env var
+    # will not be saved as secrets in the backend.
+    if os.getenv("MLRUN_AUTH_OFFLINE_TOKEN") or mlrun.utils.is_running_in_runtime():
+        return
+
+    # The import is needed here to prevent a circular import, since this method is called from the mlrun.db connection.
+    from mlrun.db import get_run_db
+
+    secret_tokens = mlrun.auth.utils.load_and_prepare_secret_tokens(
+        auth_user_id=get_run_db().token_provider.authenticated_user_id
     )
+
+    # The log_warning=False flag ensures the SDK doesn't log
+    # unnecessary warnings about local file updates, since
+    # this method reads from the file, not updates it.
+    get_run_db().store_secret_tokens(secret_tokens, log_warning=False)

mlrun/serving/__init__.py

@@ -27,6 +27,7 @@ __all__ = [
     "ModelRunner",
     "Model",
     "ModelSelector",
+    "ModelRunnerSelector",
     "MonitoredStep",
     "LLModel",
 ]
@@ -47,6 +48,7 @@ from .states import (
     ModelRunner,
     Model,
     ModelSelector,
+    ModelRunnerSelector,
     MonitoredStep,
     LLModel,
 )  # noqa

mlrun/serving/remote.py

@@ -23,10 +23,14 @@ import storey
 from storey.flow import _ConcurrentJobExecution
 
 import mlrun
+import mlrun.common.schemas
 import mlrun.config
+import mlrun.platforms
+import mlrun.utils.async_http
 from mlrun.errors import err_to_str
-from mlrun.utils import logger
+from mlrun.utils import dict_to_json, logger
 
+from ..config import config
 from .utils import (
     _extract_input_data,
     _update_result_body,
@@ -73,7 +77,9 @@ class RemoteStep(storey.SendToHttp):
 
         :param url:     http(s) url or function [project/]name to call
         :param subpath: path (which follows the url), use `$path` to use the event.path
-        :param method:  HTTP method (GET, POST, ..), default to POST
+        :param method:  The HTTP method to use for the request (e.g., "GET", "POST", "PUT", "DELETE").
+                        If not provided, the step will try to use `event.method` at runtime, and if that
+                        is also missing, it defaults to `"POST"`.
         :param headers: dictionary with http header values
         :param url_expression: an expression for getting the url from the event, e.g. "event['url']"
         :param body_expression: an expression for getting the request body from the event, e.g. "event['data']"
@@ -150,8 +156,8 @@ class RemoteStep(storey.SendToHttp):
     async def _process_event(self, event):
         # async implementation (with storey)
         body = self._get_event_or_body(event)
-        method, url, headers, body = self._generate_request(event, body)
-        kwargs = {}
+        method, url, headers, body, kwargs = self._generate_request(event, body)
+        kwargs = kwargs or {}
         if self.timeout:
             kwargs["timeout"] = aiohttp.ClientTimeout(total=self.timeout)
         try:
@@ -162,7 +168,7 @@ class RemoteStep(storey.SendToHttp):
                 text = await resp.text()
                 raise RuntimeError(f"bad http response {resp.status}: {text}")
             return resp
-        except asyncio.TimeoutError as exc:
+        except TimeoutError as exc:
             logger.error(f"http request to {url} timed out in RemoteStep {self.name}")
             raise exc
 
@@ -191,7 +197,7 @@ class RemoteStep(storey.SendToHttp):
             )
 
         body = _extract_input_data(self._input_path, event.body)
-        method, url, headers, body = self._generate_request(event, body)
+        method, url, headers, body, kwargs = self._generate_request(event, body)
         try:
             resp = self._session.request(
                 method,
@@ -200,6 +206,7 @@ class RemoteStep(storey.SendToHttp):
                 headers=headers,
                 data=body,
                 timeout=self.timeout,
+                **kwargs,
             )
         except requests.exceptions.ReadTimeout as err:
             raise requests.exceptions.ReadTimeout(
@@ -234,19 +241,19 @@ class RemoteStep(storey.SendToHttp):
             headers[event_id_key] = event.id
         if method == "GET":
             body = None
-        elif body is not None and not isinstance(body, (str, bytes)):
+        elif body is not None and not isinstance(body, str | bytes):
             if self._body_function_handler:
                 body = self._body_function_handler(body)
             body = json.dumps(body)
             headers["Content-Type"] = "application/json"
 
-        return method, url, headers, body
+        return method, url, headers, body, {}
 
     def _get_data(self, data, headers):
         if (
             self.return_json
             or headers.get("content-type", "").lower() == "application/json"
-        ) and isinstance(data, (str, bytes)):
+        ) and isinstance(data, str | bytes):
             data = json.loads(data)
         return data
 
@@ -383,7 +390,7 @@ class BatchHttpRequests(_ConcurrentJobExecution):
 
             if is_get:
                 body = None
-            elif body is not None and not isinstance(body, (str, bytes)):
+            elif body is not None and not isinstance(body, str | bytes):
                 if self._body_function_handler:
                     body = self._body_function_handler(body)
                 body = json.dumps(body)
@@ -451,6 +458,72 @@ class BatchHttpRequests(_ConcurrentJobExecution):
         if (
             self.return_json
             or headers.get("content-type", "").lower() == "application/json"
-        ) and isinstance(data, (str, bytes)):
+        ) and isinstance(data, str | bytes):
             data = json.loads(data)
         return data
+
+
+class MLRunAPIRemoteStep(RemoteStep):
+    def __init__(
+        self, method: str, path: str, fill_placeholders: Optional[bool] = None, **kwargs
+    ):
+        """
+        Graph step implementation for calling MLRun API endpoints
+
+        :param method:  The HTTP method to use for the request (e.g., "GET", "POST", "PUT", "DELETE").
+                        If not provided, the step will try to use `event.method` at runtime, and if that
+                        is also missing, it defaults to `"POST"`.
+        :param path:    API path (e.g. /api/projects)
+        :param fill_placeholders: if True, fill placeholders in the path using event fields (default to False)
+        :param kwargs:  other arguments passed to RemoteStep
+        """
+        super().__init__(url="", method=method, **kwargs)
+        self.rundb = None
+        self.path = path
+        self.fill_placeholders = fill_placeholders
+
+    def _generate_request(self, event, body):
+        method = self.method or event.method or "POST"
+        kw = {
+            key: value
+            for key, value in (
+                ("params", body.get("params")),
+                ("json", body.get("json")),
+            )
+            if value is not None
+        }
+
+        headers = self.headers or {}
+        headers.update(body.get("headers", {}))
+
+        if self.rundb.user:
+            kw["auth"] = (self.rundb.user, self.rundb.password)
+        elif self.rundb.token_provider:
+            token = self.rundb.token_provider.get_token()
+            if token:
+                # Iguazio auth doesn't support passing token through bearer, so use cookie instead
+                if self.rundb.token_provider.is_iguazio_session():
+                    session_cookie = f'session=j:{{"sid": "{token}"}}'
+                    headers["cookie"] = session_cookie
+                else:
+                    if "Authorization" not in kw.setdefault("headers", {}):
+                        headers.update({"Authorization": "Bearer " + token})
+
+        if mlrun.common.schemas.HeaderNames.client_version not in headers:
+            headers.update(
+                {
+                    mlrun.common.schemas.HeaderNames.client_version: self.rundb.client_version,
+                    mlrun.common.schemas.HeaderNames.python_version: self.rundb.python_version,
+                    "User-Agent": f"{requests.utils.default_user_agent()} mlrun/{config.version}",
+                }
+            )
+
+        url = self.url.format(**body) if self.fill_placeholders else self.url
+        headers["Content-Type"] = "application/json"
+        return method, url, headers, dict_to_json(body), kw
+
+    def post_init(self, mode="sync", **kwargs):
+        super().post_init(mode=mode, **kwargs)
+        self.fill_placeholders = self.fill_placeholders or False
+        self.rundb = mlrun.get_run_db()
+        self.url = self.rundb.get_base_api_url(self.path)

mlrun/serving/routers.py

@@ -31,6 +31,9 @@ import mlrun.common.model_monitoring
 import mlrun.common.schemas.model_monitoring
 from mlrun.utils import logger, now_date
 
+from ..common.model_monitoring.helpers import (
+    get_model_endpoints_creation_task_status,
+)
 from .utils import RouterToDict, _extract_input_data, _update_result_body
 from .v2_serving import _ModelLogPusher
 
@@ -171,46 +174,6 @@ class BaseModelRouter(RouterToDict):
         """run tasks after processing the event"""
         return event
 
-    def _get_background_task_status(
-        self,
-    ) -> mlrun.common.schemas.BackgroundTaskState:
-        self._background_task_check_timestamp = now_date()
-        server: mlrun.serving.GraphServer = getattr(
-            self.context, "_server", None
-        ) or getattr(self.context, "server", None)
-        if not self.context.is_mock:
-            if server.model_endpoint_creation_task_name:
-                background_task = mlrun.get_run_db().get_project_background_task(
-                    server.project, server.model_endpoint_creation_task_name
-                )
-                logger.debug(
-                    "Checking model endpoint creation task status",
-                    task_name=server.model_endpoint_creation_task_name,
-                )
-                if (
-                    background_task.status.state
-                    in mlrun.common.schemas.BackgroundTaskState.terminal_states()
-                ):
-                    logger.info(
-                        f"Model endpoint creation task completed with state {background_task.status.state}"
-                    )
-                else:  # in progress
-                    logger.info(
-                        f"Model endpoint creation task is still in progress with the current state: "
-                        f"{background_task.status.state}. Events will not be monitored for the next "
-                        f"{mlrun.mlconf.model_endpoint_monitoring.model_endpoint_creation_check_period} seconds",
-                        name=self.name,
-                        background_task_check_timestamp=self._background_task_check_timestamp.isoformat(),
-                    )
-                return background_task.status.state
-            else:
-                logger.error(
-                    "Model endpoint creation task name not provided. This function is not being monitored.",
-                )
-        elif self.context.monitoring_mock:
-            return mlrun.common.schemas.BackgroundTaskState.succeeded
-        return mlrun.common.schemas.BackgroundTaskState.failed
-
     def _update_background_task_state(self, event):
         if not self.background_task_reached_terminal_state and (
             self._background_task_check_timestamp is None
@@ -219,7 +182,26 @@ class BaseModelRouter(RouterToDict):
                 seconds=mlrun.mlconf.model_endpoint_monitoring.model_endpoint_creation_check_period
             )
         ):
-            self._background_task_current_state = self._get_background_task_status()
+            server: mlrun.serving.GraphServer = getattr(
+                self.context, "_server", None
+            ) or getattr(self.context, "server", None)
+            if not self.context.is_mock:
+                (
+                    self._background_task_current_state,
+                    self._background_task_check_timestamp,
+                    _,
+                ) = get_model_endpoints_creation_task_status(server)
+            elif self.context.monitoring_mock:
+                self._background_task_current_state = (
+                    mlrun.common.schemas.BackgroundTaskState.succeeded
+                )
+                self._background_task_check_timestamp = mlrun.utils.now_date()
+            else:
+                self._background_task_current_state = (
+                    mlrun.common.schemas.BackgroundTaskState.failed
+                )
+                self._background_task_check_timestamp = mlrun.utils.now_date()
+
         if event.body:
             event.body["background_task_state"] = (
                 self._background_task_current_state
@@ -1004,7 +986,7 @@ class VotingEnsemble(ParallelRun):
         List
             The model's predictions
         """
-        if isinstance(response, (list, numpy.ndarray)):
+        if isinstance(response, list | numpy.ndarray):
             return response
         try:
             self.format_response_with_col_name_flag = True
@@ -1141,7 +1123,7 @@ class EnrichmentModelRouter(ModelRouter):
 
     def preprocess(self, event):
         """Turn an entity identifier (source) to a Feature Vector"""
-        if isinstance(event.body, (str, bytes)):
+        if isinstance(event.body, str | bytes):
             event.body = json.loads(event.body)
         event.body["inputs"] = self._feature_service.get(
             event.body["inputs"], as_list=True
@@ -1293,7 +1275,7 @@ class EnrichmentVotingEnsemble(VotingEnsemble):
         """
         Turn an entity identifier (source) to a Feature Vector
         """
-        if isinstance(event.body, (str, bytes)):
+        if isinstance(event.body, str | bytes):
             event.body = json.loads(event.body)
         event.body["inputs"] = self._feature_service.get(
             event.body["inputs"], as_list=True