mlrun 1.8.0rc38__py3-none-any.whl → 1.8.0rc39__py3-none-any.whl

mlrun/__main__.py

@@ -17,7 +17,6 @@ import json
 import pathlib
 import socket
 import traceback
-import warnings
 from ast import literal_eval
 from base64 import b64decode
 from os import environ, path, remove
@@ -864,14 +863,8 @@ def version():
 )
 @click.option("--offset", type=int, default=0, help="byte offset")
 @click.option("--db", help="api and db service path/url")
-@click.option("--watch", "-w", is_flag=True, help="Deprecated. not in use")
-def logs(uid, project, offset, db, watch):
+def logs(uid, project, offset, db):
     """Get or watch task logs"""
-    if watch:
-        warnings.warn(
-            "'--watch' is deprecated in 1.6.0, and will be removed in 1.8.0, "
-            # TODO: Remove in 1.8.0
-        )
     mldb = get_run_db(db or mlconf.dbpath)
     if mldb.kind == "http":
         state, _ = mldb.watch_log(uid, project, watch=False, offset=offset)

mlrun/artifacts/base.py

@@ -893,7 +893,7 @@ def generate_target_path(item: Artifact, artifact_path, producer):
     return f"{artifact_path}{item.key}{suffix}"
 
 
-# TODO: left to support data migration from legacy artifacts to new artifacts. Remove in 1.8.0.
+# TODO: Remove once data migration v5 is obsolete
 def convert_legacy_artifact_to_new_format(
     legacy_artifact: dict,
 ) -> Artifact:
@@ -905,9 +905,9 @@ def convert_legacy_artifact_to_new_format(
     artifact_tag = legacy_artifact.get("tag", "")
     if artifact_tag:
         artifact_key = f"{artifact_key}:{artifact_tag}"
-    # TODO: remove in 1.8.0
+    # TODO: Remove once data migration v5 is obsolete
     warnings.warn(
-        f"Converting legacy artifact '{artifact_key}' to new format. This will not be supported in MLRun 1.8.0. "
+        f"Converting legacy artifact '{artifact_key}' to new format. This will not be supported in MLRun 1.9.0. "
         f"Make sure to save the artifact/project in the new format.",
         FutureWarning,
     )

mlrun/artifacts/manager.py

@@ -108,7 +108,7 @@ class ArtifactProducer:
 def dict_to_artifact(struct: dict) -> Artifact:
     kind = struct.get("kind", "")
 
-    # TODO: remove this in 1.8.0
+    # TODO: Remove once data migration v5 is obsolete
     if mlrun.utils.is_legacy_artifact(struct):
         return mlrun.artifacts.base.convert_legacy_artifact_to_new_format(struct)
 

mlrun/common/schemas/client_spec.py

@@ -66,3 +66,4 @@ class ClientSpec(pydantic.v1.BaseModel):
     external_platform_tracking: typing.Optional[dict]
     alerts_mode: typing.Optional[str]
     system_id: typing.Optional[str]
+    model_endpoint_monitoring_store_prefixes: typing.Optional[dict[str, str]]

mlrun/common/schemas/model_monitoring/constants.py

@@ -42,12 +42,10 @@ class ModelEndpointSchema(MonitoringStrEnum):
     # spec
     FUNCTION_NAME = "function_name"
     FUNCTION_TAG = "function_tag"
-    FUNCTION_UID = "function_uid"
     MODEL_NAME = "model_name"
-    MODEL_DB_KEY = "model_db_key"
-    MODEL_TAG = "model_tag"
+    MODEL_TAGS = "model_tags"
+    MODEL_PATH = "model_path"
     MODEL_CLASS = "model_class"
-    MODEL_UID = "model_uid"
     FEATURE_NAMES = "feature_names"
     LABEL_NAMES = "label_names"
     FEATURE_STATS = "feature_stats"

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -117,14 +117,13 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
 
 
 class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
-    model_uid: Optional[str] = ""
-    model_name: Optional[str] = ""
-    model_db_key: Optional[str] = ""
-    model_tag: Optional[str] = ""
     model_class: Optional[str] = ""
     function_name: Optional[str] = ""
     function_tag: Optional[str] = ""
-    function_uid: Optional[str] = ""
+    model_path: Optional[str] = ""
+    model_name: Optional[str] = ""
+    model_tags: Optional[list[str]] = []
+    _model_id: Optional[int] = ""
     feature_names: Optional[list[str]] = []
     label_names: Optional[list[str]] = []
     feature_stats: Optional[dict] = {}
@@ -137,12 +136,8 @@ class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
     @classmethod
     def mutable_fields(cls):
         return [
-            "model_uid",
-            "model_name",
-            "model_db_key",
-            "model_tag",
+            "model_path",
             "model_class",
-            "function_uid",
             "feature_names",
             "label_names",
             "children",
@@ -206,7 +201,6 @@ class ModelEndpoint(BaseModel):
             ModelEndpointSchema.CURRENT_STATS,
             ModelEndpointSchema.DRIFT_MEASURES,
             ModelEndpointSchema.FUNCTION_URI,
-            ModelEndpointSchema.MODEL_URI,
         }
         # Initialize a flattened dictionary that will be filled with the model endpoint dictionary attributes
         flatten_dict = {}

mlrun/datastore/base.py

@@ -24,7 +24,6 @@ import pandas as pd
 import pyarrow
 import pytz
 import requests
-from deprecated import deprecated
 
 import mlrun.config
 import mlrun.errors
@@ -95,16 +94,6 @@ class DataStore:
     def uri_to_ipython(endpoint, subpath):
         return ""
 
-    # TODO: remove in 1.8.0
-    @deprecated(
-        version="1.8.0",
-        reason="'get_filesystem()' will be removed in 1.8.0, use "
-        "'filesystem' property instead",
-        category=FutureWarning,
-    )
-    def get_filesystem(self):
-        return self.filesystem
-
     @property
     def filesystem(self) -> Optional[fsspec.AbstractFileSystem]:
         """return fsspec file system object, if supported"""

mlrun/db/httpdb.py

@@ -566,6 +566,17 @@ class HTTPRunDB(RunDBInterface):
             )
             config.alerts.mode = server_cfg.get("alerts_mode") or config.alerts.mode
             config.system_id = server_cfg.get("system_id") or config.system_id
+            model_monitoring_store_prefixes = (
+                server_cfg.get("model_endpoint_monitoring_store_prefixes") or {}
+            )
+            for prefix in ["default", "user_space", "monitoring_application"]:
+                store_prefix_value = model_monitoring_store_prefixes.get(prefix)
+                if server_prefix_value is not None:
+                    setattr(
+                        config.model_endpoint_monitoring.store_prefixes,
+                        prefix,
+                        store_prefix_value,
+                    )
 
         except Exception as exc:
             logger.warning(

mlrun/model_monitoring/api.py

@@ -118,8 +118,6 @@ def get_or_create_model_endpoint(
             model_endpoint_name=model_endpoint_name,
             function_name=function_name,
             function_tag=function_tag,
-            context=context,
-            sample_set_statistics=sample_set_statistics,
             monitoring_mode=monitoring_mode,
         )
     return model_endpoint
@@ -344,8 +342,6 @@ def _generate_model_endpoint(
     model_endpoint_name: str,
     function_name: str,
     function_tag: str,
-    context: "mlrun.MLClientCtx",
-    sample_set_statistics: dict[str, typing.Any],
     monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
 ) -> ModelEndpoint:
     """
@@ -358,21 +354,10 @@ def _generate_model_endpoint(
     :param model_endpoint_name:      Model endpoint name will be presented under the new model endpoint.
     :param function_name:            If a new model endpoint is created, use this function name.
     :param function_tag:             If a new model endpoint is created, use this function tag.
-    :param context:                  MLRun context. If function_name not provided, use the context to generate the
-                                     full function hash.
-    :param sample_set_statistics:    Dictionary of sample set statistics that will be used as a reference data for
-                                     the current model endpoint. Will be stored under
-                                     `model_endpoint.status.feature_stats`.
+    :param monitoring_mode:          Monitoring mode of the new model endpoint.
 
     :return `mlrun.common.schemas.ModelEndpoint` object.
     """
-    model_obj = None
-    if model_path:
-        model_obj: mlrun.artifacts.ModelArtifact = (
-            mlrun.datastore.store_resources.get_store_resource(
-                model_path, db=db_session
-            )
-        )
     current_time = datetime_now()
     model_endpoint = mlrun.common.schemas.ModelEndpoint(
         metadata=mlrun.common.schemas.ModelEndpointMetadata(
@@ -383,10 +368,7 @@ def _generate_model_endpoint(
         spec=mlrun.common.schemas.ModelEndpointSpec(
             function_name=function_name or "function",
             function_tag=function_tag or "latest",
-            model_name=model_obj.metadata.key if model_obj else None,
-            model_uid=model_obj.metadata.uid if model_obj else None,
-            model_tag=model_obj.metadata.tag if model_obj else None,
-            model_db_key=model_obj.spec.db_key if model_obj else None,
+            model_path=model_path,
             model_class="drift-analysis",
         ),
         status=mlrun.common.schemas.ModelEndpointStatus(

mlrun/model_monitoring/controller.py

@@ -16,6 +16,7 @@ import concurrent.futures
 import datetime
 import json
 import os
+import traceback
 from collections.abc import Iterator
 from contextlib import AbstractContextManager
 from types import TracebackType
@@ -551,14 +552,29 @@ class MonitoringApplicationController:
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=min(len(endpoints), 10)
         ) as pool:
-            for endpoint in endpoints:
+            futures = {
                 pool.submit(
                     MonitoringApplicationController.endpoint_to_regular_event,
                     endpoint,
                     policy,
                     set(applications_names),
                     self.v3io_access_key,
-                )
+                ): endpoint
+                for endpoint in endpoints
+            }
+            for future in concurrent.futures.as_completed(futures):
+                if future.exception():
+                    exception = future.exception()
+                    error = (
+                        f"Failed to push event. Endpoint name: {futures[future].metadata.name}, "
+                        f"endpoint uid: {futures[future].metadata.uid}, traceback:\n"
+                    )
+                    error += "".join(
+                        traceback.format_exception(
+                            None, exception, exception.__traceback__
+                        )
+                    )
+                    logger.error(error)
         logger.info("Finishing monitoring controller chief")
 
     @staticmethod

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -75,13 +75,8 @@ class TDEngineConnector(TSDBConnector):
         """Establish a connection to the TSDB server."""
         logger.debug("Creating a new connection to TDEngine", project=self.project)
         conn = TDEngineConnection(self._tdengine_connection_profile.dsn())
-        conn.run(
-            statements=f"CREATE DATABASE IF NOT EXISTS {self.database}",
-            timeout=self._timeout,
-            retries=self._retries,
-        )
         conn.prefix_statements = [f"USE {self.database}"]
-        logger.debug("Connected to TDEngine", project=self.project)
+
         return conn
 
     def _init_super_tables(self):
@@ -101,8 +96,27 @@ class TDEngineConnector(TSDBConnector):
             ),
         }
 
+    def _create_db_if_not_exists(self):
+        """Create the database if it does not exist."""
+        self.connection.prefix_statements = []
+        self.connection.run(
+            statements=f"CREATE DATABASE IF NOT EXISTS {self.database}",
+            timeout=self._timeout,
+            retries=self._retries,
+        )
+        self.connection.prefix_statements = [f"USE {self.database}"]
+        logger.debug(
+            "The TDEngine database is currently in use",
+            project=self.project,
+            database=self.database,
+        )
+
     def create_tables(self):
         """Create TDEngine supertables."""
+
+        # Create the database if it does not exist
+        self._create_db_if_not_exists()
+
         for table in self.tables:
             create_table_query = self.tables[table]._create_super_table_query()
             conn = self.connection
@@ -344,6 +358,7 @@ class TDEngineConnector(TSDBConnector):
                     project=self.project,
                     database=self.database,
                 )
+
             except Exception as e:
                 logger.warning(
                     "Failed to drop the database. You may need to drop it manually if it is empty.",

mlrun/projects/operations.py

@@ -294,9 +294,9 @@ def build_function(
     :param force_build: Force building the image, even when no changes were made
     """
     if not overwrite_build_params:
-        # TODO: change overwrite_build_params default to True in 1.8.0
+        # TODO: change overwrite_build_params default to True in 1.9.0
         warnings.warn(
-            "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.8.0.",
+            "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
             mlrun.utils.OverwriteBuildParamsWarning,
         )
 
@@ -325,7 +325,7 @@ def build_function(
             skip_deployed=skip_deployed,
         )
     else:
-        # TODO: remove filter once overwrite_build_params default is changed to True in 1.8.0
+        # TODO: remove filter once overwrite_build_params default is changed to True in 1.9.0
         with warnings.catch_warnings():
             warnings.simplefilter(
                 "ignore", category=mlrun.utils.OverwriteBuildParamsWarning

mlrun/projects/pipelines.py

@@ -1139,6 +1139,11 @@ def load_and_run_workflow(
         if "running" in notification.when
     ]
 
+    # Prevent redundant notifications for run completion by ensuring that notifications are only triggered when the run
+    # reaches the "running" state, as the server already handles the completion notifications.
+    for notification in start_notifications:
+        notification.when = ["running"]
+
     workflow_log_message = workflow_name or workflow_path
     context.logger.info(f"Running workflow {workflow_log_message} from remote")
     run = project.run(

mlrun/projects/project.py

@@ -4059,9 +4059,9 @@ class MlrunProject(ModelObj):
             (by default `/home/mlrun_code`)
         """
         if not overwrite_build_params:
-            # TODO: change overwrite_build_params default to True in 1.8.0
+            # TODO: change overwrite_build_params default to True in 1.9.0
             warnings.warn(
-                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.8.0.",
+                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
         default_image_name = mlrun.mlconf.default_project_image_name.format(
@@ -4136,9 +4136,9 @@ class MlrunProject(ModelObj):
             )
 
         if not overwrite_build_params:
-            # TODO: change overwrite_build_params default to True in 1.8.0
+            # TODO: change overwrite_build_params default to True in 1.9.0
             warnings.warn(
-                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.8.0.",
+                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
 

mlrun/run.py

@@ -36,9 +36,9 @@ import mlrun.common.formatters
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.helpers
+import mlrun_pipelines.utils
 from mlrun_pipelines.common.models import RunStatuses
 from mlrun_pipelines.common.ops import format_summary_from_kfp_run, show_kfp_run
-from mlrun_pipelines.utils import get_client
 
 from .common.helpers import parse_versioned_object_uri
 from .config import config as mlconf
@@ -437,7 +437,7 @@ def new_function(
     mode: Optional[str] = None,
     handler: Optional[str] = None,
     source: Optional[str] = None,
-    requirements: Optional[Union[str, list[str]]] = None,
+    requirements: Optional[list[str]] = None,
     kfp: Optional[bool] = None,
     requirements_file: str = "",
 ):
@@ -1015,7 +1015,7 @@ def wait_for_pipeline_completion(
             _wait_for_pipeline_completion,
         )
     else:
-        client = get_client(namespace=namespace)
+        client = mlrun_pipelines.utils.get_client(namespace=namespace)
         resp = client.wait_for_run_completion(run_id, timeout)
         if resp:
             resp = resp.to_dict()
@@ -1076,7 +1076,7 @@ def get_pipeline(
         )
 
     else:
-        client = get_client(namespace=namespace)
+        client = mlrun_pipelines.utils.get_client(namespace=namespace)
         resp = client.get_run(run_id)
         if resp:
             resp = resp.to_dict()

mlrun/runtimes/kubejob.py

@@ -114,9 +114,9 @@ class KubejobRuntime(KubeResource):
             e.g. builder_env={"GIT_TOKEN": token}
         """
         if not overwrite:
-            # TODO: change overwrite default to True in 1.8.0
+            # TODO: change overwrite default to True in 1.9.0
             warnings.warn(
-                "The `overwrite` parameter default will change from 'False' to 'True' in 1.8.0.",
+                "The `overwrite` parameter default will change from 'False' to 'True' in 1.9.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
         image = mlrun.utils.helpers.remove_image_protocol_prefix(image)

mlrun/runtimes/nuclio/application/application.py

@@ -542,7 +542,6 @@ class ApplicationRuntime(RemoteRuntime):
         body: typing.Optional[typing.Union[str, bytes, dict]] = None,
         method: typing.Optional[str] = None,
         headers: typing.Optional[dict] = None,
-        dashboard: str = "",
         force_external_address: bool = False,
         auth_info: schemas.AuthInfo = None,
         mock: typing.Optional[bool] = None,
@@ -569,7 +568,6 @@ class ApplicationRuntime(RemoteRuntime):
                 body,
                 method,
                 headers,
-                dashboard,
                 force_external_address,
                 auth_info,
                 mock,

mlrun/runtimes/nuclio/function.py

@@ -767,45 +767,10 @@ class RemoteRuntime(KubeResource):
 
     def _get_state(
         self,
-        dashboard="",
         last_log_timestamp=0,
         verbose=False,
         raise_on_exception=True,
-        resolve_address=True,
-        auth_info: AuthInfo = None,
     ) -> tuple[str, str, typing.Optional[float]]:
-        if dashboard:
-            (
-                state,
-                address,
-                name,
-                last_log_timestamp,
-                text,
-                function_status,
-            ) = get_nuclio_deploy_status(
-                self.metadata.name,
-                self.metadata.project,
-                self.metadata.tag,
-                dashboard,
-                last_log_timestamp=last_log_timestamp,
-                verbose=verbose,
-                resolve_address=resolve_address,
-                auth_info=auth_info,
-            )
-            self.status.internal_invocation_urls = function_status.get(
-                "internalInvocationUrls", []
-            )
-            self.status.external_invocation_urls = function_status.get(
-                "externalInvocationUrls", []
-            )
-            self.status.state = state
-            self.status.nuclio_name = name
-            self.status.container_image = function_status.get("containerImage", "")
-            if address:
-                self.status.address = address
-                self.spec.command = f"http://{address}"
-            return state, text, last_log_timestamp
-
         try:
             text, last_log_timestamp = self._get_db().get_nuclio_deploy_status(
                 self, last_log_timestamp=last_log_timestamp, verbose=verbose
@@ -916,7 +881,6 @@ class RemoteRuntime(KubeResource):
         body: typing.Optional[typing.Union[str, bytes, dict]] = None,
         method: typing.Optional[str] = None,
         headers: typing.Optional[dict] = None,
-        dashboard: str = "",
         force_external_address: bool = False,
         auth_info: AuthInfo = None,
         mock: typing.Optional[bool] = None,
@@ -932,7 +896,6 @@ class RemoteRuntime(KubeResource):
         :param body:     request body (str, bytes or a dict for json requests)
         :param method:   HTTP method (GET, PUT, ..)
         :param headers:  key/value dict with http headers
-        :param dashboard: nuclio dashboard address (deprecated)
         :param force_external_address:   use the external ingress URL
         :param auth_info: service AuthInfo
         :param mock:     use mock server vs a real Nuclio function (for local simulations)
@@ -940,14 +903,6 @@ class RemoteRuntime(KubeResource):
                                      see this link for more information:
                                      https://requests.readthedocs.io/en/latest/api/#requests.request
         """
-        if dashboard:
-            # TODO: remove in 1.8.0
-            warnings.warn(
-                "'dashboard' parameter is no longer supported on client side, "
-                "it is being configured through the MLRun API. It will be removed in 1.8.0.",
-                FutureWarning,
-            )
-
         if not method:
             method = "POST" if body else "GET"
 
@@ -977,7 +932,7 @@ class RemoteRuntime(KubeResource):
                         "so function can not be invoked via http. Either enable default http trigger creation or "
                         "create custom http trigger"
                     )
-                state, _, _ = self._get_state(dashboard, auth_info=auth_info)
+                state, _, _ = self._get_state()
                 if state not in ["ready", "scaledToZero"]:
                     logger.warning(f"Function is in the {state} state")
                 if not self.status.address: