mlrun 1.8.0rc37__py3-none-any.whl → 1.8.0rc39__py3-none-any.whl

mlrun/model_monitoring/api.py

@@ -118,8 +118,6 @@ def get_or_create_model_endpoint(
             model_endpoint_name=model_endpoint_name,
             function_name=function_name,
             function_tag=function_tag,
-            context=context,
-            sample_set_statistics=sample_set_statistics,
             monitoring_mode=monitoring_mode,
         )
     return model_endpoint
@@ -344,8 +342,6 @@ def _generate_model_endpoint(
     model_endpoint_name: str,
     function_name: str,
     function_tag: str,
-    context: "mlrun.MLClientCtx",
-    sample_set_statistics: dict[str, typing.Any],
     monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
 ) -> ModelEndpoint:
     """
@@ -358,21 +354,10 @@ def _generate_model_endpoint(
     :param model_endpoint_name:      Model endpoint name will be presented under the new model endpoint.
     :param function_name:            If a new model endpoint is created, use this function name.
     :param function_tag:             If a new model endpoint is created, use this function tag.
-    :param context:                  MLRun context. If function_name not provided, use the context to generate the
-                                     full function hash.
-    :param sample_set_statistics:    Dictionary of sample set statistics that will be used as a reference data for
-                                     the current model endpoint. Will be stored under
-                                     `model_endpoint.status.feature_stats`.
+    :param monitoring_mode:          Monitoring mode of the new model endpoint.
 
     :return `mlrun.common.schemas.ModelEndpoint` object.
     """
-    model_obj = None
-    if model_path:
-        model_obj: mlrun.artifacts.ModelArtifact = (
-            mlrun.datastore.store_resources.get_store_resource(
-                model_path, db=db_session
-            )
-        )
     current_time = datetime_now()
     model_endpoint = mlrun.common.schemas.ModelEndpoint(
         metadata=mlrun.common.schemas.ModelEndpointMetadata(
@@ -383,10 +368,7 @@ def _generate_model_endpoint(
         spec=mlrun.common.schemas.ModelEndpointSpec(
             function_name=function_name or "function",
             function_tag=function_tag or "latest",
-            model_name=model_obj.metadata.key if model_obj else None,
-            model_uid=model_obj.metadata.uid if model_obj else None,
-            model_tag=model_obj.metadata.tag if model_obj else None,
-            model_db_key=model_obj.spec.db_key if model_obj else None,
+            model_path=model_path,
             model_class="drift-analysis",
         ),
         status=mlrun.common.schemas.ModelEndpointStatus(

mlrun/model_monitoring/controller.py

@@ -16,6 +16,7 @@ import concurrent.futures
 import datetime
 import json
 import os
+import traceback
 from collections.abc import Iterator
 from contextlib import AbstractContextManager
 from types import TracebackType
@@ -551,14 +552,29 @@ class MonitoringApplicationController:
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=min(len(endpoints), 10)
         ) as pool:
-            for endpoint in endpoints:
+            futures = {
                 pool.submit(
                     MonitoringApplicationController.endpoint_to_regular_event,
                     endpoint,
                     policy,
                     set(applications_names),
                     self.v3io_access_key,
-                )
+                ): endpoint
+                for endpoint in endpoints
+            }
+            for future in concurrent.futures.as_completed(futures):
+                if future.exception():
+                    exception = future.exception()
+                    error = (
+                        f"Failed to push event. Endpoint name: {futures[future].metadata.name}, "
+                        f"endpoint uid: {futures[future].metadata.uid}, traceback:\n"
+                    )
+                    error += "".join(
+                        traceback.format_exception(
+                            None, exception, exception.__traceback__
+                        )
+                    )
+                    logger.error(error)
         logger.info("Finishing monitoring controller chief")
 
     @staticmethod

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -75,13 +75,8 @@ class TDEngineConnector(TSDBConnector):
         """Establish a connection to the TSDB server."""
         logger.debug("Creating a new connection to TDEngine", project=self.project)
         conn = TDEngineConnection(self._tdengine_connection_profile.dsn())
-        conn.run(
-            statements=f"CREATE DATABASE IF NOT EXISTS {self.database}",
-            timeout=self._timeout,
-            retries=self._retries,
-        )
         conn.prefix_statements = [f"USE {self.database}"]
-        logger.debug("Connected to TDEngine", project=self.project)
+
         return conn
 
     def _init_super_tables(self):
@@ -101,8 +96,27 @@ class TDEngineConnector(TSDBConnector):
             ),
         }
 
+    def _create_db_if_not_exists(self):
+        """Create the database if it does not exist."""
+        self.connection.prefix_statements = []
+        self.connection.run(
+            statements=f"CREATE DATABASE IF NOT EXISTS {self.database}",
+            timeout=self._timeout,
+            retries=self._retries,
+        )
+        self.connection.prefix_statements = [f"USE {self.database}"]
+        logger.debug(
+            "The TDEngine database is currently in use",
+            project=self.project,
+            database=self.database,
+        )
+
     def create_tables(self):
         """Create TDEngine supertables."""
+
+        # Create the database if it does not exist
+        self._create_db_if_not_exists()
+
         for table in self.tables:
             create_table_query = self.tables[table]._create_super_table_query()
             conn = self.connection
@@ -344,6 +358,7 @@ class TDEngineConnector(TSDBConnector):
                     project=self.project,
                     database=self.database,
                 )
+
             except Exception as e:
                 logger.warning(
                     "Failed to drop the database. You may need to drop it manually if it is empty.",

mlrun/model_monitoring/helpers.py

@@ -137,30 +137,27 @@ def get_stream_path(
     )
 
     if isinstance(profile, mlrun.datastore.datastore_profile.DatastoreProfileV3io):
-        stream_uri = "v3io"
+        stream_uri = mlrun.mlconf.get_model_monitoring_file_target_path(
+            project=project,
+            kind=mm_constants.FileTargetKind.STREAM,
+            target="online",
+            function_name=function_name,
+        )
+        return stream_uri.replace("v3io://", f"ds://{profile.name}")
+
     elif isinstance(
         profile, mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource
     ):
-        attributes = profile.attributes()
-        stream_uri = f"kafka://{attributes['brokers'][0]}"
+        topic = mlrun.common.model_monitoring.helpers.get_kafka_topic(
+            project=project, function_name=function_name
+        )
+        return f"ds://{profile.name}/{topic}"
     else:
         raise mlrun.errors.MLRunValueError(
             f"Received an unexpected stream profile type: {type(profile)}\n"
             "Expects `DatastoreProfileV3io` or `DatastoreProfileKafkaSource`."
         )
 
-    if not stream_uri or stream_uri == "v3io":
-        stream_uri = mlrun.mlconf.get_model_monitoring_file_target_path(
-            project=project,
-            kind=mm_constants.FileTargetKind.STREAM,
-            target="online",
-            function_name=function_name,
-        )
-
-    return mlrun.common.model_monitoring.helpers.parse_monitoring_stream_path(
-        stream_uri=stream_uri, project=project, function_name=function_name
-    )
-
 
 def get_monitoring_parquet_path(
     project: "MlrunProject",
@@ -314,18 +311,9 @@ def _get_kafka_output_stream(
     topic = mlrun.common.model_monitoring.helpers.get_kafka_topic(
         project=project, function_name=function_name
     )
-    profile_attributes = kafka_profile.attributes()
-    producer_options = profile_attributes.get("producer_options", {})
-    if "sasl" in profile_attributes:
-        sasl = profile_attributes["sasl"]
-        producer_options.update(
-            {
-                "security_protocol": "SASL_PLAINTEXT",
-                "sasl_mechanism": sasl["mechanism"],
-                "sasl_plain_username": sasl["user"],
-                "sasl_plain_password": sasl["password"],
-            },
-        )
+    attributes = kafka_profile.attributes()
+    producer_options = mlrun.datastore.utils.KafkaParameters(attributes).producer()
+
     return mlrun.platforms.iguazio.KafkaOutputStream(
         brokers=kafka_profile.brokers,
         topic=topic,

mlrun/model_monitoring/stream_processing.py

@@ -31,7 +31,6 @@ from mlrun.common.schemas.model_monitoring.constants import (
     FileTargetKind,
     ProjectSecretKeys,
 )
-from mlrun.datastore import parse_kafka_url
 from mlrun.model_monitoring.db import TSDBConnector
 from mlrun.utils import logger
 
@@ -259,39 +258,13 @@ class EventStreamProcessor:
 
         # controller branch
         def apply_push_controller_stream(stream_uri: str):
-            if stream_uri.startswith("v3io://"):
-                graph.add_step(
-                    ">>",
-                    "controller_stream_v3io",
-                    path=stream_uri,
-                    sharding_func=ControllerEvent.ENDPOINT_ID,
-                    access_key=self.v3io_access_key,
-                    after="ForwardNOP",
-                )
-            elif stream_uri.startswith("kafka://"):
-                topic, brokers = parse_kafka_url(stream_uri)
-                logger.info(
-                    "Controller stream uri for kafka",
-                    stream_uri=stream_uri,
-                    topic=topic,
-                    brokers=brokers,
-                )
-                if isinstance(brokers, list):
-                    path = f"kafka://{brokers[0]}/{topic}"
-                elif isinstance(brokers, str):
-                    path = f"kafka://{brokers}/{topic}"
-                else:
-                    raise mlrun.errors.MLRunInvalidArgumentError(
-                        "Brokers must be a list or str check controller stream uri"
-                    )
-                graph.add_step(
-                    ">>",
-                    "controller_stream_kafka",
-                    path=path,
-                    kafka_brokers=brokers,
-                    _sharding_func=ControllerEvent.ENDPOINT_ID,
-                    after="ForwardNOP",
-                )
+            graph.add_step(
+                ">>",
+                "controller_stream",
+                path=stream_uri,
+                sharding_func=ControllerEvent.ENDPOINT_ID,
+                after="ForwardNOP",
+            )
 
         apply_push_controller_stream(controller_stream_uri)
 

mlrun/projects/operations.py

@@ -294,9 +294,9 @@ def build_function(
     :param force_build: Force building the image, even when no changes were made
     """
     if not overwrite_build_params:
-        # TODO: change overwrite_build_params default to True in 1.8.0
+        # TODO: change overwrite_build_params default to True in 1.9.0
         warnings.warn(
-            "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.8.0.",
+            "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
             mlrun.utils.OverwriteBuildParamsWarning,
         )
 
@@ -325,7 +325,7 @@ def build_function(
             skip_deployed=skip_deployed,
         )
     else:
-        # TODO: remove filter once overwrite_build_params default is changed to True in 1.8.0
+        # TODO: remove filter once overwrite_build_params default is changed to True in 1.9.0
         with warnings.catch_warnings():
             warnings.simplefilter(
                 "ignore", category=mlrun.utils.OverwriteBuildParamsWarning

mlrun/projects/pipelines.py

@@ -1139,6 +1139,11 @@ def load_and_run_workflow(
         if "running" in notification.when
     ]
 
+    # Prevent redundant notifications for run completion by ensuring that notifications are only triggered when the run
+    # reaches the "running" state, as the server already handles the completion notifications.
+    for notification in start_notifications:
+        notification.when = ["running"]
+
     workflow_log_message = workflow_name or workflow_path
     context.logger.info(f"Running workflow {workflow_log_message} from remote")
     run = project.run(

mlrun/projects/project.py

@@ -4059,9 +4059,9 @@ class MlrunProject(ModelObj):
             (by default `/home/mlrun_code`)
         """
         if not overwrite_build_params:
-            # TODO: change overwrite_build_params default to True in 1.8.0
+            # TODO: change overwrite_build_params default to True in 1.9.0
             warnings.warn(
-                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.8.0.",
+                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
         default_image_name = mlrun.mlconf.default_project_image_name.format(
@@ -4136,9 +4136,9 @@ class MlrunProject(ModelObj):
             )
 
         if not overwrite_build_params:
-            # TODO: change overwrite_build_params default to True in 1.8.0
+            # TODO: change overwrite_build_params default to True in 1.9.0
             warnings.warn(
-                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.8.0.",
+                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
 

mlrun/run.py

@@ -36,9 +36,9 @@ import mlrun.common.formatters
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.helpers
+import mlrun_pipelines.utils
 from mlrun_pipelines.common.models import RunStatuses
 from mlrun_pipelines.common.ops import format_summary_from_kfp_run, show_kfp_run
-from mlrun_pipelines.utils import get_client
 
 from .common.helpers import parse_versioned_object_uri
 from .config import config as mlconf
@@ -437,7 +437,7 @@ def new_function(
     mode: Optional[str] = None,
     handler: Optional[str] = None,
     source: Optional[str] = None,
-    requirements: Optional[Union[str, list[str]]] = None,
+    requirements: Optional[list[str]] = None,
     kfp: Optional[bool] = None,
     requirements_file: str = "",
 ):
@@ -1015,7 +1015,7 @@ def wait_for_pipeline_completion(
             _wait_for_pipeline_completion,
         )
     else:
-        client = get_client(namespace=namespace)
+        client = mlrun_pipelines.utils.get_client(namespace=namespace)
         resp = client.wait_for_run_completion(run_id, timeout)
         if resp:
             resp = resp.to_dict()
@@ -1076,7 +1076,7 @@ def get_pipeline(
         )
 
     else:
-        client = get_client(namespace=namespace)
+        client = mlrun_pipelines.utils.get_client(namespace=namespace)
         resp = client.get_run(run_id)
         if resp:
             resp = resp.to_dict()

mlrun/runtimes/kubejob.py

@@ -114,9 +114,9 @@ class KubejobRuntime(KubeResource):
             e.g. builder_env={"GIT_TOKEN": token}
         """
         if not overwrite:
-            # TODO: change overwrite default to True in 1.8.0
+            # TODO: change overwrite default to True in 1.9.0
             warnings.warn(
-                "The `overwrite` parameter default will change from 'False' to 'True' in 1.8.0.",
+                "The `overwrite` parameter default will change from 'False' to 'True' in 1.9.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
         image = mlrun.utils.helpers.remove_image_protocol_prefix(image)

mlrun/runtimes/nuclio/application/application.py

@@ -542,7 +542,6 @@ class ApplicationRuntime(RemoteRuntime):
         body: typing.Optional[typing.Union[str, bytes, dict]] = None,
         method: typing.Optional[str] = None,
         headers: typing.Optional[dict] = None,
-        dashboard: str = "",
         force_external_address: bool = False,
         auth_info: schemas.AuthInfo = None,
         mock: typing.Optional[bool] = None,
@@ -569,7 +568,6 @@ class ApplicationRuntime(RemoteRuntime):
                 body,
                 method,
                 headers,
-                dashboard,
                 force_external_address,
                 auth_info,
                 mock,

mlrun/runtimes/nuclio/function.py

@@ -767,45 +767,10 @@ class RemoteRuntime(KubeResource):
 
     def _get_state(
         self,
-        dashboard="",
         last_log_timestamp=0,
         verbose=False,
         raise_on_exception=True,
-        resolve_address=True,
-        auth_info: AuthInfo = None,
     ) -> tuple[str, str, typing.Optional[float]]:
-        if dashboard:
-            (
-                state,
-                address,
-                name,
-                last_log_timestamp,
-                text,
-                function_status,
-            ) = get_nuclio_deploy_status(
-                self.metadata.name,
-                self.metadata.project,
-                self.metadata.tag,
-                dashboard,
-                last_log_timestamp=last_log_timestamp,
-                verbose=verbose,
-                resolve_address=resolve_address,
-                auth_info=auth_info,
-            )
-            self.status.internal_invocation_urls = function_status.get(
-                "internalInvocationUrls", []
-            )
-            self.status.external_invocation_urls = function_status.get(
-                "externalInvocationUrls", []
-            )
-            self.status.state = state
-            self.status.nuclio_name = name
-            self.status.container_image = function_status.get("containerImage", "")
-            if address:
-                self.status.address = address
-                self.spec.command = f"http://{address}"
-            return state, text, last_log_timestamp
-
         try:
             text, last_log_timestamp = self._get_db().get_nuclio_deploy_status(
                 self, last_log_timestamp=last_log_timestamp, verbose=verbose
@@ -916,7 +881,6 @@ class RemoteRuntime(KubeResource):
         body: typing.Optional[typing.Union[str, bytes, dict]] = None,
         method: typing.Optional[str] = None,
         headers: typing.Optional[dict] = None,
-        dashboard: str = "",
         force_external_address: bool = False,
         auth_info: AuthInfo = None,
         mock: typing.Optional[bool] = None,
@@ -932,7 +896,6 @@ class RemoteRuntime(KubeResource):
         :param body:     request body (str, bytes or a dict for json requests)
         :param method:   HTTP method (GET, PUT, ..)
         :param headers:  key/value dict with http headers
-        :param dashboard: nuclio dashboard address (deprecated)
         :param force_external_address:   use the external ingress URL
         :param auth_info: service AuthInfo
         :param mock:     use mock server vs a real Nuclio function (for local simulations)
@@ -940,14 +903,6 @@ class RemoteRuntime(KubeResource):
                                      see this link for more information:
                                      https://requests.readthedocs.io/en/latest/api/#requests.request
         """
-        if dashboard:
-            # TODO: remove in 1.8.0
-            warnings.warn(
-                "'dashboard' parameter is no longer supported on client side, "
-                "it is being configured through the MLRun API. It will be removed in 1.8.0.",
-                FutureWarning,
-            )
-
         if not method:
             method = "POST" if body else "GET"
 
@@ -977,7 +932,7 @@ class RemoteRuntime(KubeResource):
                         "so function can not be invoked via http. Either enable default http trigger creation or "
                         "create custom http trigger"
                     )
-                state, _, _ = self._get_state(dashboard, auth_info=auth_info)
+                state, _, _ = self._get_state()
                 if state not in ["ready", "scaledToZero"]:
                     logger.warning(f"Function is in the {state} state")
                 if not self.status.address: