mlrun 1.8.0rc26__py3-none-any.whl → 1.8.0rc28__py3-none-any.whl

mlrun/__main__.py

@@ -772,10 +772,11 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
 
         runs = run_db.list_runs(uid=uid, project=project, labels=selector)
         df = runs.to_df()[
-            ["name", "uid", "iter", "start", "state", "parameters", "results"]
+            ["name", "uid", "iter", "start", "end", "state", "parameters", "results"]
         ]
         # df['uid'] = df['uid'].apply(lambda x: f'..{x[-6:]}')
-        df["start"] = df["start"].apply(time_str)
+        for time_column in ["start", "end"]:
+            df[time_column] = df[time_column].apply(time_str)
         df["parameters"] = df["parameters"].apply(dict_to_str)
         df["results"] = df["results"].apply(dict_to_str)
         print(tabulate(df, headers="keys"))

mlrun/artifacts/document.py

@@ -34,7 +34,9 @@ class DocumentLoaderSpec(ModelObj):
 
     This class is responsible for loading documents from a given source path using a specified loader class.
     The loader class is dynamically imported and instantiated with the provided arguments. The loaded documents
-    can be optionally uploaded as artifacts.
+    can be optionally uploaded as artifacts. Note that only loader classes that return single results
+    (e.g., TextLoader, UnstructuredHTMLLoader, WebBaseLoader(scalar)) are supported - loaders returning multiple
+    results like DirectoryLoader or WebBaseLoader(list) are not compatible.
 
     Attributes:
         loader_class_name (str): The name of the loader class to use for loading documents.
@@ -61,7 +63,7 @@ class DocumentLoaderSpec(ModelObj):
             kwargs (Optional[dict]): Additional keyword arguments to pass to the loader class.
             download_object (bool, optional): If True, the file will be downloaded before launching
                 the loader. If False, the loader accepts a link that should not be downloaded.
-                Defaults to False.
+                Defaults to True.
         Example:
             >>> # Create a loader specification for PDF documents
             >>> loader_spec = DocumentLoaderSpec(
@@ -97,7 +99,7 @@ class MLRunLoader:
     Args:
         artifact_key (str, optional): The key for the artifact to be logged. Special characters and symbols
             not valid in artifact names will be encoded as their hexadecimal representation. The '%%' pattern
-            in the key will be replaced by the hex-encoded version of the source path. Defaults to "doc%%".
+            in the key will be replaced by the hex-encoded version of the source path. Defaults to "%%".
         local_path (str): The source path of the document to be loaded.
         loader_spec (DocumentLoaderSpec): Specification for the document loader.
         producer (Optional[Union[MlrunProject, str, MLClientCtx]], optional): The producer of the document.
@@ -129,7 +131,7 @@ class MLRunLoader:
         >>> loader = MLRunLoader(
         ...     source_path="/path/to/document.txt",
         ...     loader_spec=loader_spec,
-        ...     artifact_key="doc%%",  # %% will be replaced with encoded path
+        ...     artifact_key="%%",  # %% will be replaced with encoded path
         ...     producer=project,
         ... )
         >>> documents = loader.load()
@@ -141,7 +143,7 @@ class MLRunLoader:
         ...     loader_cls=MLRunLoader,
         ...     loader_kwargs={
         ...         "loader_spec": loader_spec,
-        ...         "artifact_key": "doc%%",
+        ...         "artifact_key": "%%",
         ...         "producer": project,
         ...         "upload": True,
         ...     },
@@ -154,7 +156,7 @@ class MLRunLoader:
         cls,
         source_path: str,
         loader_spec: "DocumentLoaderSpec",
-        artifact_key="doc%%",
+        artifact_key="%%",
         producer: Optional[Union["MlrunProject", str, "MLClientCtx"]] = None,  # noqa: F821
         upload: bool = False,
         tag: str = "",
@@ -271,6 +273,7 @@ class DocumentArtifact(Artifact):
             result.append("_")
 
         resolved_path = "".join(result)
+        resolved_path = resolved_path.lstrip("_")
         return resolved_path
 
     class DocumentArtifactSpec(ArtifactSpec):

mlrun/artifacts/model.py

@@ -30,6 +30,7 @@ from ..utils import StorePrefix, is_relative_path
 from .base import Artifact, ArtifactSpec, upload_extra_data
 
 model_spec_filename = "model_spec.yaml"
+MODEL_OPTIONAL_SUFFIXES = [".tar.gz", ".pkl", ".bin", ".pickle"]
 
 
 class ModelArtifactSpec(ArtifactSpec):
@@ -426,7 +427,17 @@ def get_model(model_dir, suffix=""):
     model_file = ""
     model_spec = None
     extra_dataitems = {}
-    suffix = suffix or ".pkl"
+    default_suffix = ".pkl"
+
+    alternative_suffix = next(
+        (
+            optional_suffix
+            for optional_suffix in MODEL_OPTIONAL_SUFFIXES
+            if model_dir.lower().endswith(optional_suffix)
+        ),
+        None,
+    )
+
     if hasattr(model_dir, "artifact_url"):
         model_dir = model_dir.artifact_url
 
@@ -440,15 +451,19 @@ def get_model(model_dir, suffix=""):
             target, model_spec.model_target_file or model_spec.model_file
         )
         extra_dataitems = _get_extra(target, model_spec.extra_data)
-
+        suffix = suffix or default_suffix
     elif model_dir.lower().endswith(".yaml"):
         model_spec = _load_model_spec(model_dir)
         model_file = _get_file_path(model_dir, model_spec.model_file)
         extra_dataitems = _get_extra(model_dir, model_spec.extra_data)
-
-    elif model_dir.endswith(suffix):
+        suffix = suffix or default_suffix
+    elif suffix and model_dir.endswith(suffix):
+        model_file = model_dir
+    elif not suffix and alternative_suffix:
+        suffix = alternative_suffix
         model_file = model_dir
     else:
+        suffix = suffix or default_suffix
         dirobj = mlrun.datastore.store_manager.object(url=model_dir)
         model_dir_list = dirobj.listdir()
         if model_spec_filename in model_dir_list:

mlrun/common/model_monitoring/helpers.py

@@ -47,9 +47,9 @@ def parse_monitoring_stream_path(
             function_name is None
             or function_name == mm_constants.MonitoringFunctionNames.STREAM
         ):
-            stream_uri += f"?topic=monitoring_stream_{project}"
+            stream_uri += f"?topic=monitoring_stream_{project}_v1"
         else:
-            stream_uri += f"?topic=monitoring_stream_{project}_{function_name}"
+            stream_uri += f"?topic=monitoring_stream_{project}_{function_name}_v1"
 
     return stream_uri
 

mlrun/common/schemas/model_monitoring/constants.py

@@ -146,7 +146,6 @@ class EventFieldType:
 
 class FeatureSetFeatures(MonitoringStrEnum):
     LATENCY = EventFieldType.LATENCY
-    ERROR_COUNT = EventFieldType.ERROR_COUNT
     METRICS = EventFieldType.METRICS
 
     @classmethod

mlrun/common/schemas/serving.py

@@ -0,0 +1,22 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pydantic.v1 import BaseModel
+
+from .background_task import BackgroundTaskList
+
+
+class DeployResponse(BaseModel):
+    data: dict
+    background_tasks: BackgroundTaskList

mlrun/config.py

@@ -232,6 +232,7 @@ default_config = {
                 "abort_grace_period": "10",
                 "delete_project": "900",
                 "delete_function": "900",
+                "model_endpoint_creation": "600",
             },
             "runtimes": {"dask": "600"},
             "push_notifications": "60",
@@ -870,6 +871,14 @@ class Config:
             return self.__class__(val)
         return val
 
+    def __deepcopy__(self, memo):
+        cls = self.__class__
+        # create a new Config without calling __init__ (avoid recursion)
+        result = cls.__new__(cls)
+        # manually deep-copy _cfg
+        object.__setattr__(result, "_cfg", copy.deepcopy(self._cfg, memo))
+        return result
+
     def __setattr__(self, attr, value):
         # in order for the dbpath setter to work
         if attr == "dbpath":
@@ -1304,7 +1313,7 @@ class Config:
                     project=project,
                     kind=kind
                     if function_name is None
-                    else f"{kind}-{function_name.lower()}",
+                    else f"{kind}-{function_name.lower()}-v1",
                 )
             elif (
                 kind == "stream"
@@ -1313,19 +1322,23 @@ class Config:
             ):
                 return mlrun.mlconf.model_endpoint_monitoring.store_prefixes.user_space.format(
                     project=project,
-                    kind=kind,
+                    kind=f"{kind}-v1",
                 )
-            else:
-                if (
-                    function_name
-                    == mlrun.common.schemas.model_monitoring.constants.MonitoringFunctionNames.APPLICATION_CONTROLLER
-                ):
-                    kind = function_name
+            elif (
+                function_name
+                == mlrun.common.schemas.model_monitoring.constants.MonitoringFunctionNames.APPLICATION_CONTROLLER
+                and kind == "stream"
+            ):
                 return mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
                     project=project,
-                    kind=kind,
+                    kind=f"{kind}-{function_name.lower()}-v1",
                 )
 
+            return mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
+                project=project,
+                kind=kind,
+            )
+
         # Get the current offline path from the configuration
         file_path = mlrun.mlconf.model_endpoint_monitoring.offline_storage_path.format(
             project=project, kind=kind

mlrun/datastore/base.py

@@ -677,13 +677,6 @@ class DataItem:
         return f"'{self.url}'"
 
 
-def get_range(size, offset):
-    byterange = f"bytes={offset}-"
-    if size:
-        byterange += str(offset + size)
-    return byterange
-
-
 def basic_auth_header(user, password):
     username = user.encode("latin1")
     password = password.encode("latin1")

mlrun/datastore/s3.py

@@ -21,7 +21,7 @@ from fsspec.registry import get_filesystem_class
 
 import mlrun.errors
 
-from .base import DataStore, FileStats, get_range, make_datastore_schema_sanitizer
+from .base import DataStore, FileStats, make_datastore_schema_sanitizer
 
 
 class S3Store(DataStore):
@@ -108,6 +108,13 @@ class S3Store(DataStore):
                     "choose-signer.s3.*", disable_signing
                 )
 
+    @staticmethod
+    def get_range(size, offset):
+        byterange = f"bytes={offset}-"
+        if size:
+            byterange += str(offset + size - 1)
+        return byterange
+
     def get_spark_options(self):
         res = {}
         st = self.get_storage_options()
@@ -185,7 +192,7 @@ class S3Store(DataStore):
         bucket, key = self.get_bucket_and_key(key)
         obj = self.s3.Object(bucket, key)
         if size or offset:
-            return obj.get(Range=get_range(size, offset))["Body"].read()
+            return obj.get(Range=S3Store.get_range(size, offset))["Body"].read()
         return obj.get()["Body"].read()
 
     def put(self, key, data, append=False):

mlrun/db/base.py

@@ -108,6 +108,8 @@ class RunDBInterface(ABC):
         start_time_to: Optional[datetime.datetime] = None,
         last_update_time_from: Optional[datetime.datetime] = None,
         last_update_time_to: Optional[datetime.datetime] = None,
+        end_time_from: Optional[datetime.datetime] = None,
+        end_time_to: Optional[datetime.datetime] = None,
         partition_by: Union[mlrun.common.schemas.RunPartitionByField, str] = None,
         rows_per_partition: int = 1,
         partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
@@ -1075,7 +1077,6 @@ class RunDBInterface(ABC):
         base_period: int = 10,
         image: str = "mlrun/mlrun",
         deploy_histogram_data_drift_app: bool = True,
-        rebuild_images: bool = False,
         fetch_credentials_from_sys_config: bool = False,
     ) -> None:
         pass

mlrun/db/httpdb.py

@@ -905,6 +905,8 @@ class HTTPRunDB(RunDBInterface):
         start_time_to: Optional[datetime] = None,
         last_update_time_from: Optional[datetime] = None,
         last_update_time_to: Optional[datetime] = None,
+        end_time_from: Optional[datetime] = None,
+        end_time_to: Optional[datetime] = None,
         partition_by: Optional[
             Union[mlrun.common.schemas.RunPartitionByField, str]
         ] = None,
@@ -951,6 +953,8 @@ class HTTPRunDB(RunDBInterface):
         :param last_update_time_from: Filter by run last update time in ``(last_update_time_from,
             last_update_time_to)``.
         :param last_update_time_to: Filter by run last update time in ``(last_update_time_from, last_update_time_to)``.
+        :param end_time_from: Filter by run end time in ``[end_time_from, end_time_to]``.
+        :param end_time_to: Filter by run end time in ``[end_time_from, end_time_to]``.
         :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
             parameter must be provided as well.
         :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
@@ -976,6 +980,8 @@ class HTTPRunDB(RunDBInterface):
             start_time_to=start_time_to,
             last_update_time_from=last_update_time_from,
             last_update_time_to=last_update_time_to,
+            end_time_from=end_time_from,
+            end_time_to=end_time_to,
             partition_by=partition_by,
             rows_per_partition=rows_per_partition,
             partition_sort_by=partition_sort_by,
@@ -2368,9 +2374,9 @@ class HTTPRunDB(RunDBInterface):
     def retry_pipeline(
         self,
         run_id: str,
+        project: str,
         namespace: Optional[str] = None,
         timeout: int = 30,
-        project: Optional[str] = None,
     ):
         """
         Retry a specific pipeline run using its run ID. This function sends an API request
@@ -2380,8 +2386,7 @@ class HTTPRunDB(RunDBInterface):
         :param run_id: The unique ID of the pipeline run to retry.
         :param namespace: Kubernetes namespace where the pipeline is running. Optional.
         :param timeout: Timeout (in seconds) for the API call. Defaults to 30 seconds.
-        :param project: Name of the MLRun project associated with the pipeline. Can be
-            ``*`` to query across all projects. Optional.
+        :param project: Name of the MLRun project associated with the pipeline.
 
         :raises ValueError: Raised if the API response is not successful or contains an
             error.
@@ -2392,14 +2397,13 @@ class HTTPRunDB(RunDBInterface):
         params = {}
         if namespace:
             params["namespace"] = namespace
-        project_path = project if project else "*"
 
         resp_text = ""
         resp_code = None
         try:
             resp = self.api_call(
                 "POST",
-                f"projects/{project_path}/pipelines/{run_id}/retry",
+                f"projects/{project}/pipelines/{run_id}/retry",
                 params=params,
                 timeout=timeout,
             )
@@ -2414,7 +2418,7 @@ class HTTPRunDB(RunDBInterface):
             logger.error(
                 "Retry pipeline API call encountered an error.",
                 run_id=run_id,
-                project=project_path,
+                project=project,
                 namespace=namespace,
                 response_code=resp_code,
                 response_text=resp_text,
@@ -2429,7 +2433,7 @@ class HTTPRunDB(RunDBInterface):
         logger.info(
             "Successfully retried pipeline run",
             run_id=run_id,
-            project=project_path,
+            project=project,
             namespace=namespace,
         )
         return resp.json()
@@ -3967,7 +3971,6 @@ class HTTPRunDB(RunDBInterface):
         base_period: int = 10,
         image: str = "mlrun/mlrun",
         deploy_histogram_data_drift_app: bool = True,
-        rebuild_images: bool = False,
         fetch_credentials_from_sys_config: bool = False,
     ) -> None:
         """
@@ -3985,7 +3988,6 @@ class HTTPRunDB(RunDBInterface):
                                                   stream functions, which are real time nuclio functions.
                                                   By default, the image is mlrun/mlrun.
         :param deploy_histogram_data_drift_app:   If true, deploy the default histogram-based data drift application.
-        :param rebuild_images:                    If true, force rebuild of model monitoring infrastructure images.
         :param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
 
         """
@@ -3996,7 +3998,6 @@ class HTTPRunDB(RunDBInterface):
                 "base_period": base_period,
                 "image": image,
                 "deploy_histogram_data_drift_app": deploy_histogram_data_drift_app,
-                "rebuild_images": rebuild_images,
                 "fetch_credentials_from_sys_config": fetch_credentials_from_sys_config,
             },
         )
@@ -5226,6 +5227,8 @@ class HTTPRunDB(RunDBInterface):
         start_time_to: Optional[datetime] = None,
         last_update_time_from: Optional[datetime] = None,
         last_update_time_to: Optional[datetime] = None,
+        end_time_from: Optional[datetime] = None,
+        end_time_to: Optional[datetime] = None,
         partition_by: Optional[
             Union[mlrun.common.schemas.RunPartitionByField, str]
         ] = None,
@@ -5277,6 +5280,8 @@ class HTTPRunDB(RunDBInterface):
             and not start_time_to
             and not last_update_time_from
             and not last_update_time_to
+            and not end_time_from
+            and not end_time_to
             and not partition_by
             and not partition_sort_by
             and not iter
@@ -5301,6 +5306,8 @@ class HTTPRunDB(RunDBInterface):
             "start_time_to": datetime_to_iso(start_time_to),
             "last_update_time_from": datetime_to_iso(last_update_time_from),
             "last_update_time_to": datetime_to_iso(last_update_time_to),
+            "end_time_from": datetime_to_iso(end_time_from),
+            "end_time_to": datetime_to_iso(end_time_to),
             "with-notifications": with_notifications,
             "page": page,
             "page-size": page_size,

mlrun/db/nopdb.py

@@ -138,6 +138,8 @@ class NopDB(RunDBInterface):
         start_time_to: Optional[datetime.datetime] = None,
         last_update_time_from: Optional[datetime.datetime] = None,
         last_update_time_to: Optional[datetime.datetime] = None,
+        end_time_from: Optional[datetime.datetime] = None,
+        end_time_to: Optional[datetime.datetime] = None,
         partition_by: Union[mlrun.common.schemas.RunPartitionByField, str] = None,
         rows_per_partition: int = 1,
         partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
@@ -855,7 +857,6 @@ class NopDB(RunDBInterface):
         base_period: int = 10,
         image: str = "mlrun/mlrun",
         deploy_histogram_data_drift_app: bool = True,
-        rebuild_images: bool = False,
         fetch_credentials_from_sys_config: bool = False,
     ) -> None:
         pass

mlrun/execution.py

@@ -914,7 +914,8 @@ class MLClientCtx:
                     kwargs={"extract_images": True}
                 )
         :param upload: Whether to upload the artifact
-        :param labels: Key-value labels
+        :param labels:  Key-value labels. A 'source' label is automatically added using either
+                        local_path or target_path to facilitate easier document searching.
         :param target_path: Path to the local file
         :param db_key: The key to use in the artifact DB table, by default its run name + '_' + key
                        db_key=False will not register it in the artifacts table
@@ -932,22 +933,32 @@ class MLClientCtx:
             ...     ),
             ... )
         """
+        original_source = local_path or target_path
 
-        if not key and not local_path and not target_path:
+        if not key and not original_source:
             raise ValueError(
                 "Must provide either 'key' parameter or 'local_path'/'target_path' to derive the key from"
             )
         if not key:
-            key = DocumentArtifact.key_from_source(local_path or target_path)
+            key = DocumentArtifact.key_from_source(original_source)
 
         doc_artifact = DocumentArtifact(
             key=key,
-            original_source=local_path or target_path,
+            original_source=original_source,
             document_loader_spec=document_loader_spec,
             collections=kwargs.pop("collections", None),
             **kwargs,
         )
 
+        # limit label to a max of 255 characters (for db reasons)
+        max_length = 255
+        labels = labels or {}
+        labels["source"] = (
+            original_source[: max_length - 3] + "..."
+            if len(original_source) > max_length
+            else original_source
+        )
+
         item = self._artifacts_manager.log_artifact(
             self,
             doc_artifact,

mlrun/lists.py

@@ -29,6 +29,7 @@ list_header = [
     "uid",
     "iter",
     "start",
+    "end",
     "state",
     "kind",
     "name",
@@ -58,6 +59,7 @@ class RunList(list):
                 get_in(run, "metadata.uid", ""),
                 get_in(run, "metadata.iteration", ""),
                 get_in(run, "status.start_time", ""),
+                get_in(run, "status.end_time", ""),
                 get_in(run, "status.state", ""),
                 get_in(run, "step_kind", get_in(run, "kind", "")),
                 get_in(run, "metadata.name", ""),
@@ -103,7 +105,8 @@ class RunList(list):
             return self._df
         rows = self.to_rows(extend_iterations=extend_iterations)
         df = pd.DataFrame(rows[1:], columns=rows[0])  # .set_index('iter')
-        df["start"] = pd.to_datetime(df["start"])
+        for time_column in ["start", "end"]:
+            df[time_column] = pd.to_datetime(df[time_column])
 
         if flat:
             df = flatten(df, "labels")

mlrun/model.py

@@ -1284,6 +1284,7 @@ class RunStatus(ModelObj):
         results=None,
         artifacts=None,
         start_time=None,
+        end_time=None,
         last_update=None,
         iterations=None,
         ui_url=None,
@@ -1299,6 +1300,7 @@ class RunStatus(ModelObj):
         self.results = results
         self._artifacts = artifacts
         self.start_time = start_time
+        self.end_time = end_time
         self.last_update = last_update
         self.iterations = iterations
         self.ui_url = ui_url

mlrun/model_monitoring/applications/_application_steps.py

@@ -126,6 +126,7 @@ class _PrepareMonitoringEvent(StepToDict):
         :param application_name: Application name.
         """
         self.graph_context = context
+        _ = self.graph_context.project_obj  # Ensure project exists
         self.application_name = application_name
         self.model_endpoints: dict[str, mlrun.common.schemas.ModelEndpoint] = {}