mlrun 1.8.0rc26__py3-none-any.whl → 1.8.0rc28__py3-none-any.whl

mlrun/projects/project.py

@@ -31,6 +31,7 @@ from os import environ, makedirs, path
 from typing import Callable, Optional, Union, cast
 from urllib.parse import urlparse
 
+import deprecated
 import dotenv
 import git
 import git.exc
@@ -1948,7 +1949,8 @@ class MlrunProject(ModelObj):
                     kwargs={"extract_images": True}
                 )
         :param upload: Whether to upload the artifact
-        :param labels: Key-value labels
+        :param labels:  Key-value labels. A 'source' label is automatically added using either
+                        local_path or target_path to facilitate easier document searching.
         :param target_path: Target file path
         :param kwargs: Additional keyword arguments
         :return: DocumentArtifact object
@@ -1978,13 +1980,24 @@ class MlrunProject(ModelObj):
                 "The document loader is configured to not support downloads but the upload flag is set to True."
                 "Either set loader.download_object=True or set upload=False"
             )
+        original_source = local_path or target_path
         doc_artifact = DocumentArtifact(
             key=key,
-            original_source=local_path or target_path,
+            original_source=original_source,
             document_loader_spec=document_loader_spec,
             collections=kwargs.pop("collections", None),
             **kwargs,
         )
+
+        # limit label to a max of 255 characters (for db reasons)
+        max_length = 255
+        labels = labels or {}
+        labels["source"] = (
+            original_source[: max_length - 3] + "..."
+            if len(original_source) > max_length
+            else original_source
+        )
+
         return self.log_artifact(
             item=doc_artifact,
             tag=tag,
@@ -2406,7 +2419,6 @@ class MlrunProject(ModelObj):
         *,
         deploy_histogram_data_drift_app: bool = True,
         wait_for_deployment: bool = False,
-        rebuild_images: bool = False,
         fetch_credentials_from_sys_config: bool = False,
     ) -> None:
         """
@@ -2428,7 +2440,6 @@ class MlrunProject(ModelObj):
         :param wait_for_deployment:               If true, return only after the deployment is done on the backend.
                                                   Otherwise, deploy the model monitoring infrastructure on the
                                                   background, including the histogram data drift app if selected.
-        :param rebuild_images:                    If true, force rebuild of model monitoring infrastructure images.
         :param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
         """
         if default_controller_image != "mlrun/mlrun":
@@ -2451,7 +2462,6 @@ class MlrunProject(ModelObj):
             image=image,
             base_period=base_period,
             deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
-            rebuild_images=rebuild_images,
             fetch_credentials_from_sys_config=fetch_credentials_from_sys_config,
         )
 
@@ -2839,6 +2849,13 @@ class MlrunProject(ModelObj):
 
         self.spec.set_function(name, function_object, func)
 
+    # TODO: Remove this in 1.10.0
+    @deprecated.deprecated(
+        version="1.8.0",
+        reason="'remove_function' is deprecated and will be removed in 1.10.0. "
+        "Please use `delete_function` instead.",
+        category=FutureWarning,
+    )
     def remove_function(self, name):
         """remove the specified function from the project
 
@@ -2846,6 +2863,18 @@ class MlrunProject(ModelObj):
         """
         self.spec.remove_function(name)
 
+    def delete_function(self, name, delete_from_db=False):
+        """deletes the specified function from the project
+
+        :param name: name of the function (under the project)
+        :param delete_from_db: default is False. If False, the function is removed
+                               only from the project's cache and spec.
+                               If True, the function is also removed from the database.
+        """
+        if delete_from_db:
+            mlrun.db.get_run_db().delete_function(name=name, project=self.metadata.name)
+        self.spec.remove_function(name)
+
     def remove_model_monitoring_function(self, name: Union[str, list[str]]):
         """delete the specified model-monitoring-app function/s
 
@@ -3762,8 +3791,8 @@ class MlrunProject(ModelObj):
                 "Please keep in mind that if you already had model monitoring functions "
                 "/ model monitoring infra / tracked model server "
                 "deployed on your project, you will need to redeploy them. "
-                "For redeploying the model monitoring infra, please use `enable_model_monitoring` API "
-                "and set `rebuild_images=True`"
+                "For redeploying the model monitoring infra, first disable it using "
+                "`project.disable_model_monitoring()` and then enable it using `project.enable_model_monitoring()`."
             )
 
     def list_model_endpoints(
@@ -3779,6 +3808,7 @@ class MlrunProject(ModelObj):
         top_level: bool = False,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
+        tsdb_metrics: bool = True,
     ) -> mlrun.common.schemas.ModelEndpointList:
         """
         Returns a list of `ModelEndpoint` objects. Each `ModelEndpoint` object represents the current state of a
@@ -3829,6 +3859,7 @@ class MlrunProject(ModelObj):
             top_level=top_level,
             uids=uids,
             latest_only=latest_only,
+            tsdb_metrics=tsdb_metrics,
         )
 
     def run_function(
@@ -4661,6 +4692,8 @@ class MlrunProject(ModelObj):
         start_time_to: Optional[datetime.datetime] = None,
         last_update_time_from: Optional[datetime.datetime] = None,
         last_update_time_to: Optional[datetime.datetime] = None,
+        end_time_from: Optional[datetime.datetime] = None,
+        end_time_to: Optional[datetime.datetime] = None,
         **kwargs,
     ) -> mlrun.lists.RunList:
         """Retrieve a list of runs.
@@ -4704,6 +4737,8 @@ class MlrunProject(ModelObj):
         :param last_update_time_from: Filter by run last update time in ``(last_update_time_from,
             last_update_time_to)``.
         :param last_update_time_to: Filter by run last update time in ``(last_update_time_from, last_update_time_to)``.
+        :param end_time_from: Filter by run end time in ``[end_time_from, end_time_to]``.
+        :param end_time_to: Filter by run end time in ``[end_time_from, end_time_to]``.
         """
         if state:
             # TODO: Remove this in 1.9.0
@@ -4730,6 +4765,8 @@ class MlrunProject(ModelObj):
             start_time_to=start_time_to,
             last_update_time_from=last_update_time_from,
             last_update_time_to=last_update_time_to,
+            end_time_from=end_time_from,
+            end_time_to=end_time_to,
             **kwargs,
         )
 

mlrun/runtimes/base.py

@@ -855,7 +855,7 @@ class BaseRuntime(ModelObj):
 
     def requires_build(self) -> bool:
         build = self.spec.build
-        return (
+        return bool(
             build.commands
             or build.requirements
             or (build.source and not build.load_source_on_run)

mlrun/runtimes/generators.py

@@ -182,7 +182,7 @@ class ListGenerator(TaskGenerator):
             yield newrun
 
 
-def get_run_copy(run):
+def get_run_copy(run: RunObject):
     newrun = deepcopy(run)
     newrun.spec.hyperparams = None
     newrun.spec.param_file = None

mlrun/runtimes/nuclio/function.py

@@ -599,6 +599,16 @@ class RemoteRuntime(KubeResource):
         # when a function is deployed, we wait for it to be ready by default
         # this also means that the function object will be updated with the function status
         self._wait_for_function_deployment(db, verbose=verbose)
+        # check if there are any background tasks related to creating model endpoints
+        background_tasks = mlrun.common.schemas.BackgroundTaskList(
+            **data.pop("background_tasks", {"background_tasks": []})
+        ).background_tasks
+        if background_tasks:
+            self._check_model_endpoint_task_state(
+                db=db,
+                background_task=background_tasks[0],
+                wait_for_completion=False,
+            )
 
         return self._enrich_command_from_status()
 
@@ -1285,6 +1295,33 @@ class RemoteRuntime(KubeResource):
             return mlrun.model.Credentials.generate_access_key
         return None
 
+    def _check_model_endpoint_task_state(
+        self,
+        db: mlrun.db.RunDBInterface,
+        background_task: mlrun.common.schemas.BackgroundTask,
+        wait_for_completion: bool,
+    ):
+        if wait_for_completion:
+            background_task = db._wait_for_background_task_to_reach_terminal_state(
+                name=background_task.metadata.name, project=self.metadata.project
+            )
+        else:
+            background_task = db.get_project_background_task(
+                project=self.metadata.project, name=background_task.metadata.name
+            )
+        if (
+            background_task.status.state
+            in mlrun.common.schemas.BackgroundTaskState.terminal_states()
+        ):
+            logger.info(
+                f"Model endpoint creation task completed with state {background_task.status.state}"
+            )
+        else:
+            logger.warning(
+                f"Model endpoint creation task is still running with state {background_task.status.state}"
+                f"You can use the serving function, but it won't be monitored for the next few minutes"
+            )
+
 
 def parse_logs(logs):
     logs = json.loads(logs)

mlrun/runtimes/nuclio/serving.py

@@ -152,6 +152,7 @@ class ServingSpec(NuclioSpec):
         clone_target_dir=None,
         state_thresholds=None,
         disable_default_http_trigger=None,
+        model_endpoint_creation_task_name=None,
     ):
         super().__init__(
             command=command,
@@ -209,6 +210,7 @@ class ServingSpec(NuclioSpec):
         self.tracking_policy = tracking_policy
         self.secret_sources = secret_sources or []
         self.default_content_type = default_content_type
+        self.model_endpoint_creation_task_name = model_endpoint_creation_task_name
 
     @property
     def graph(self) -> Union[RouterStep, RootFlowStep]:
@@ -696,6 +698,7 @@ class ServingRuntime(RemoteRuntime):
             "track_models": self.spec.track_models,
             "tracking_policy": None,
             "default_content_type": self.spec.default_content_type,
+            "model_endpoint_creation_task_name": self.spec.model_endpoint_creation_task_name,
         }
 
         if self.spec.secret_sources:

mlrun/runtimes/pod.py

@@ -214,9 +214,7 @@ class KubeResourceSpec(FunctionSpec):
         # default service account is set in mlrun.utils.process_function_service_account
         # due to project specific defaults
         self.service_account = service_account
-        self.image_pull_secret = (
-            image_pull_secret or mlrun.mlconf.function.spec.image_pull_secret.default
-        )
+        self.image_pull_secret = image_pull_secret
         self.node_name = node_name
         self.node_selector = node_selector or {}
         self._affinity = affinity

mlrun/serving/routers.py

@@ -389,7 +389,7 @@ class ParallelRun(BaseModelRouter):
                 self._pool = executor_class(
                     max_workers=len(self.routes),
                     initializer=ParallelRun.init_pool,
-                    initargs=(server, routes),
+                    initargs=(server, routes, self.context.is_mock),
                 )
             elif self.executor_type == ParallelRunnerModes.thread:
                 executor_class = concurrent.futures.ThreadPoolExecutor
@@ -452,9 +452,9 @@ class ParallelRun(BaseModelRouter):
         return results
 
     @staticmethod
-    def init_pool(server_spec, routes):
+    def init_pool(server_spec, routes, is_mock):
         server = mlrun.serving.GraphServer.from_dict(server_spec)
-        server.init_states(None, None)
+        server.init_states(None, None, is_mock=is_mock)
         global local_routes
         for route in routes.values():
             route.context = server.context
@@ -596,11 +596,6 @@ class VotingEnsemble(ParallelRun):
         self.vote_type = vote_type
         self.vote_flag = True if self.vote_type is not None else False
         self.weights = weights
-        self._model_logger = (
-            _ModelLogPusher(self, context)
-            if context and context.stream.enabled
-            else None
-        )
         self.version = kwargs.get("version", "v1")
         self.log_router = True
         self.prediction_col_name = prediction_col_name or "prediction"
@@ -608,8 +603,12 @@ class VotingEnsemble(ParallelRun):
         self.model_endpoint_uid = None
         self.model_endpoint = None
         self.shard_by_endpoint = shard_by_endpoint
+        self.initialized = False
 
     def post_init(self, mode="sync", **kwargs):
+        self._update_weights(self.weights)
+
+    def _lazy_init(self, event_id):
         server: mlrun.serving.GraphServer = getattr(
             self.context, "_server", None
         ) or getattr(self.context, "server", None)
@@ -617,14 +616,59 @@ class VotingEnsemble(ParallelRun):
             logger.warn("GraphServer not initialized for VotingEnsemble instance")
             return
         if not self.context.is_mock or self.context.monitoring_mock:
-            self.model_endpoint = mlrun.get_run_db().get_model_endpoint(
-                project=server.project,
-                name=self.name,
-                function_name=server.function_name,
-                function_tag=server.function_tag or "latest",
-            )
-            self.model_endpoint_uid = self.model_endpoint.metadata.uid
-        self._update_weights(self.weights)
+            if server.model_endpoint_creation_task_name:
+                background_task = mlrun.get_run_db().get_project_background_task(
+                    server.project, server.model_endpoint_creation_task_name
+                )
+                logger.info(
+                    "Checking model endpoint creation task status",
+                    task_name=server.model_endpoint_creation_task_name,
+                )
+                if (
+                    background_task.status.state
+                    in mlrun.common.schemas.BackgroundTaskState.terminal_states()
+                ):
+                    logger.info(
+                        f"Model endpoint creation task completed with state {background_task.status.state}"
+                    )
+                else:  # in progress
+                    logger.debug(
+                        f"Model endpoint creation task is still in progress with the current state: "
+                        f"{background_task.status.state}. This event will not be monitored.",
+                        name=self.name,
+                        event_id=event_id,
+                    )
+                    self.initialized = False
+                    return
+            else:
+                logger.info(
+                    "Model endpoint creation task name not provided",
+                )
+            try:
+                self.model_endpoint_uid = (
+                    mlrun.get_run_db()
+                    .get_model_endpoint(
+                        project=server.project,
+                        name=self.name,
+                        function_name=server.function_name,
+                        function_tag=server.function_tag or "latest",
+                        tsdb_metrics=False,
+                    )
+                    .metadata.uid
+                )
+            except mlrun.errors.MLRunNotFoundError:
+                logger.info(
+                    "Model endpoint not found for this step; monitoring for this model will not be performed",
+                    function_name=server.function_name,
+                    name=self.name,
+                )
+                self.model_endpoint_uid = None
+        self._model_logger = (
+            _ModelLogPusher(self, self.context)
+            if self.context and self.context.stream.enabled and self.model_endpoint_uid
+            else None
+        )
+        self.initialized = True
 
     def _resolve_route(self, body, urlpath):
         """Resolves the appropriate model to send the event to.
@@ -829,8 +873,9 @@ class VotingEnsemble(ParallelRun):
         Response
             Event response after running the requested logic
         """
+        if not self.initialized:
+            self._lazy_init(event.id)
         start = now_date()
-
         # Handle and verify the request
         original_body = event.body
         event.body = _extract_input_data(self._input_path, event.body)

mlrun/serving/server.py

@@ -112,6 +112,7 @@ class GraphServer(ModelObj):
         function_name=None,
         function_tag=None,
         project=None,
+        model_endpoint_creation_task_name=None,
     ):
         self._graph = None
         self.graph: Union[RouterStep, RootFlowStep] = graph
@@ -137,6 +138,7 @@ class GraphServer(ModelObj):
         self.function_name = function_name
         self.function_tag = function_tag
         self.project = project
+        self.model_endpoint_creation_task_name = model_endpoint_creation_task_name
 
     def set_current_function(self, function):
         """set which child function this server is currently running on"""
@@ -332,6 +334,7 @@ def v2_serving_init(context, namespace=None):
     context.logger.info("Initializing server from spec")
     spec = mlrun.utils.get_serving_spec()
     server = GraphServer.from_dict(spec)
+
     if config.log_level.lower() == "debug":
         server.verbose = True
     if hasattr(context, "trigger"):
@@ -544,11 +547,19 @@ class GraphContext:
         self.get_store_resource = None
         self.get_table = None
         self.is_mock = False
+        self.monitoring_mock = False
+        self._project_obj = None
 
     @property
     def server(self):
         return self._server
 
+    @property
+    def project_obj(self):
+        if not self._project_obj:
+            self._project_obj = mlrun.get_run_db().get_project(name=self.project)
+        return self._project_obj
+
     @property
     def project(self) -> str:
         """current project name (for the current function)"""

mlrun/serving/states.py

@@ -595,10 +595,6 @@ class TaskStep(BaseStep):
                 creation_strategy=self.model_endpoint_creation_strategy,
                 endpoint_type=self.endpoint_type,
             )
-            if hasattr(self._object, "model_endpoint_uid"):
-                self.endpoint_uid = self._object.model_endpoint_uid
-            if hasattr(self._object, "name"):
-                self.endpoint_name = self._object.name
 
     def respond(self):
         """mark this step as the responder.

mlrun/serving/v2_serving.py

@@ -115,9 +115,9 @@ class V2ModelServer(StepToDict):
             self.model = model
             self.ready = True
         self.model_endpoint_uid = None
-        self.model_endpoint = None
         self.shard_by_endpoint = shard_by_endpoint
         self._model_logger = None
+        self.initialized = False
 
     def _load_and_update_state(self):
         try:
@@ -139,36 +139,69 @@ class V2ModelServer(StepToDict):
             else:
                 self._load_and_update_state()
 
+    def _lazy_init(self, event_id):
         server: mlrun.serving.GraphServer = getattr(
             self.context, "_server", None
         ) or getattr(self.context, "server", None)
         if not server:
             logger.warn("GraphServer not initialized for VotingEnsemble instance")
             return
-
         if not self.context.is_mock and not self.model_spec:
             self.get_model()
         if not self.context.is_mock or self.context.monitoring_mock:
+            if server.model_endpoint_creation_task_name:
+                background_task = mlrun.get_run_db().get_project_background_task(
+                    server.project, server.model_endpoint_creation_task_name
+                )
+                logger.debug(
+                    "Checking model endpoint creation task status",
+                    task_name=server.model_endpoint_creation_task_name,
+                )
+                if (
+                    background_task.status.state
+                    in mlrun.common.schemas.BackgroundTaskState.terminal_states()
+                ):
+                    logger.debug(
+                        f"Model endpoint creation task completed with state {background_task.status.state}"
+                    )
+                else:  # in progress
+                    logger.debug(
+                        f"Model endpoint creation task is still in progress with the current state: "
+                        f"{background_task.status.state}. This event will not be monitored.",
+                        name=self.name,
+                        event_id=event_id,
+                    )
+                    self.initialized = False
+                    return
+            else:
+                logger.debug(
+                    "Model endpoint creation task name not provided",
+                )
             try:
-                self.model_endpoint = mlrun.get_run_db().get_model_endpoint(
-                    project=server.project,
-                    name=self.name,
-                    function_name=server.function_name,
-                    function_tag=server.function_tag or "latest",
+                self.model_endpoint_uid = (
+                    mlrun.get_run_db()
+                    .get_model_endpoint(
+                        project=server.project,
+                        name=self.name,
+                        function_name=server.function_name,
+                        function_tag=server.function_tag or "latest",
+                        tsdb_metrics=False,
+                    )
+                    .metadata.uid
                 )
-                self.model_endpoint_uid = self.model_endpoint.metadata.uid
             except mlrun.errors.MLRunNotFoundError:
                 logger.info(
-                    "Model Endpoint not found for this step we will not monitor this model",
+                    "Model endpoint not found for this step; monitoring for this model will not be performed",
                     function_name=server.function_name,
                     name=self.name,
                 )
-                self.model_endpoint, self.model_endpoint_uid = None, None
+                self.model_endpoint_uid = None
         self._model_logger = (
             _ModelLogPusher(self, self.context)
             if self.context and self.context.stream.enabled and self.model_endpoint_uid
             else None
         )
+        self.initialized = True
 
     def get_param(self, key: str, default=None):
         """get param by key (specified in the model or the function)"""
@@ -246,6 +279,8 @@ class V2ModelServer(StepToDict):
 
     def do_event(self, event, *args, **kwargs):
         """main model event handler method"""
+        if not self.initialized:
+            self._lazy_init(event.id)
         start = now_date()
         original_body = event.body
         event_body = _extract_input_data(self._input_path, event.body)

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "d9ddaa33eb375080a0577da3aa2849915b60e30d",
-  "version": "1.8.0-rc26"
+  "git_commit": "f7d4248834e0690d0e02b88fe62c16f9b40378e0",
+  "version": "1.8.0-rc28"
 }

{mlrun-1.8.0rc26.dist-info → mlrun-1.8.0rc28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mlrun
-Version: 1.8.0rc26
+Version: 1.8.0rc28
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,7 +44,7 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.8.8
+Requires-Dist: storey~=1.8.9
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
@@ -120,6 +120,7 @@ Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
 Requires-Dist: aiosmtplib~=3.0; extra == "api"
 Requires-Dist: pydantic<2,>=1; extra == "api"
 Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.5; python_version < "3.11" and extra == "api"
+Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
 Requires-Dist: adlfs==2023.9.0; extra == "all"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "all"
@@ -208,6 +209,7 @@ Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "comple
 Requires-Dist: google-cloud-storage==2.14.0; extra == "complete-api"
 Requires-Dist: google-cloud==0.34; extra == "complete-api"
 Requires-Dist: graphviz~=0.20.0; extra == "complete-api"
+Requires-Dist: grpcio~=1.70.0; extra == "complete-api"
 Requires-Dist: humanfriendly~=10.0; extra == "complete-api"
 Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
 Requires-Dist: kafka-python~=2.0; extra == "complete-api"