mlrun 1.7.0rc22__py3-none-any.whl → 1.7.0rc24__py3-none-any.whl

mlrun/model_monitoring/helpers.py

@@ -97,7 +97,7 @@ def get_monitoring_parquet_path(
     return parquet_path
 
 
-def get_connection_string(secret_provider: typing.Callable = None) -> str:
+def get_connection_string(secret_provider: typing.Callable[[str], str] = None) -> str:
     """Get endpoint store connection string from the project secret. If wasn't set, take it from the system
     configurations.
 
@@ -117,7 +117,7 @@ def get_connection_string(secret_provider: typing.Callable = None) -> str:
 
 
 def get_tsdb_connection_string(
-    secret_provider: typing.Optional[typing.Callable] = None,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
 ) -> str:
     """Get TSDB connection string from the project secret. If wasn't set, take it from the system
     configurations.
@@ -278,9 +278,13 @@ def calculate_inputs_statistics(
     return inputs_statistics
 
 
-def get_endpoint_record(project: str, endpoint_id: str):
+def get_endpoint_record(
+    project: str,
+    endpoint_id: str,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
+) -> dict[str, typing.Any]:
     model_endpoint_store = mlrun.model_monitoring.get_store_object(
-        project=project,
+        project=project, secret_provider=secret_provider
     )
     return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
 

mlrun/model_monitoring/stream_processing.py

@@ -66,10 +66,6 @@ class EventStreamProcessor:
         self.parquet_batching_max_events = parquet_batching_max_events
         self.parquet_batching_timeout_secs = parquet_batching_timeout_secs
 
-        self.model_endpoint_store_target = (
-            mlrun.mlconf.model_endpoint_monitoring.store_type
-        )
-
         logger.info(
             "Initializing model monitoring event stream processor",
             parquet_path=self.parquet_path,
@@ -139,7 +135,7 @@ class EventStreamProcessor:
     def apply_monitoring_serving_graph(
         self,
         fn: mlrun.runtimes.ServingRuntime,
-        tsdb_service_provider: typing.Optional[typing.Callable] = None,
+        secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
     ) -> None:
         """
         Apply monitoring serving graph to a given serving function. The following serving graph includes about 4 main
@@ -167,7 +163,8 @@ class EventStreamProcessor:
            using CE, the parquet target path is based on the defined MLRun artifact path.
 
         :param fn: A serving function.
-        :param tsdb_service_provider: An optional callable function that provides the TSDB connection string.
+        :param secret_provider: An optional callable function that provides the connection string from the project
+                                secret.
         """
 
         graph = typing.cast(
@@ -293,7 +290,6 @@ class EventStreamProcessor:
                 name="UpdateEndpoint",
                 after="ProcessBeforeEndpointUpdate",
                 project=self.project,
-                model_endpoint_store_target=self.model_endpoint_store_target,
             )
 
         apply_update_endpoint()
@@ -310,7 +306,10 @@ class EventStreamProcessor:
                 table=self.kv_path,
             )
 
-        if self.model_endpoint_store_target == ModelEndpointTarget.V3IO_NOSQL:
+        store_object = mlrun.model_monitoring.get_store_object(
+            project=self.project, secret_provider=secret_provider
+        )
+        if store_object.type == ModelEndpointTarget.V3IO_NOSQL:
             apply_infer_schema()
 
         # Emits the event in window size of events based on sample_window size (10 by default)
@@ -328,7 +327,7 @@ class EventStreamProcessor:
         # TSDB branch (skip to Prometheus if in CE env)
         if not mlrun.mlconf.is_ce_mode():
             tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-                project=self.project, secret_provider=tsdb_service_provider
+                project=self.project, secret_provider=secret_provider
             )
             tsdb_connector.apply_monitoring_stream_steps(graph=graph)
 
@@ -904,7 +903,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
 
 
 class UpdateEndpoint(mlrun.feature_store.steps.MapClass):
-    def __init__(self, project: str, model_endpoint_store_target: str, **kwargs):
+    def __init__(self, project: str, **kwargs):
         """
         Update the model endpoint record in the DB. Note that the event at this point includes metadata and stats about
         the average latency and the amount of predictions over time. This data will be used in the monitoring dashboards
@@ -914,7 +913,6 @@ class UpdateEndpoint(mlrun.feature_store.steps.MapClass):
         """
         super().__init__(**kwargs)
         self.project = project
-        self.model_endpoint_store_target = model_endpoint_store_target
 
     def do(self, event: dict):
         # Remove labels from the event

mlrun/model_monitoring/writer.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import json
-from typing import Any, NewType
+from typing import Any, Callable, NewType
 
 import mlrun.common.model_monitoring
 import mlrun.common.schemas
@@ -30,7 +30,7 @@ from mlrun.common.schemas.model_monitoring.constants import (
     WriterEventKind,
 )
 from mlrun.common.schemas.notification import NotificationKind, NotificationSeverity
-from mlrun.model_monitoring.helpers import get_endpoint_record, get_result_instance_fqn
+from mlrun.model_monitoring.helpers import get_result_instance_fqn
 from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 from mlrun.utils.notifications.notification_pusher import CustomNotificationPusher
@@ -102,7 +102,11 @@ class ModelMonitoringWriter(StepToDict):
 
     kind = "monitoring_application_stream_pusher"
 
-    def __init__(self, project: str, tsdb_secret_provider=None) -> None:
+    def __init__(
+        self,
+        project: str,
+        secret_provider: Callable = None,
+    ) -> None:
         self.project = project
         self.name = project  # required for the deployment process
 
@@ -111,10 +115,10 @@ class ModelMonitoringWriter(StepToDict):
         )
 
         self._app_result_store = mlrun.model_monitoring.get_store_object(
-            project=self.project
+            project=self.project, secret_provider=secret_provider
         )
         self._tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-            project=self.project, secret_provider=tsdb_secret_provider
+            project=self.project, secret_provider=secret_provider
         )
         self._endpoints_records = {}
 
@@ -223,7 +227,7 @@ class ModelMonitoringWriter(StepToDict):
             endpoint_id = event[WriterEvent.ENDPOINT_ID]
             endpoint_record = self._endpoints_records.setdefault(
                 endpoint_id,
-                get_endpoint_record(project=self.project, endpoint_id=endpoint_id),
+                self._app_result_store.get_model_endpoint(endpoint_id=endpoint_id),
             )
             event_value = {
                 "app_name": event[WriterEvent.APPLICATION_NAME],

mlrun/package/__init__.py

@@ -12,19 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-"""
-MLRun package enables fully-automated experiment and pipeline tracking and reproducibility, and easy passing of
-python objects between remote jobs, while not requiring any form of editing to the actual function original code.
-Simply set the function code in a project and run it, MLRun takes care of the rest.
-
-MLRun uses packagers: classes that perform 2 tasks:
-
-#. **Parsing inputs** - automatically cast the runtime's inputs (user's input passed to the function via
-the ``inputs`` parameter of the ``run`` method) to the relevant hinted type.  (Does not require handling of data items.)
-#. **Logging outputs** - automatically save, log, and upload the function's returned objects by the provided
-log hints (user's input passed to the function via the ``returns`` parameter of the ``run`` method).
-(Does not require handling of files and artifacts.)
-"""
+
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 import functools

mlrun/package/packagers/__init__.py

@@ -12,12 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-r"""
-MLRun comes with the following list of modules, out of the box. All of the packagers listed here
-use the implementation of :ref:`DefaultPackager <mlrun.package.packagers.default\_packager.DefaultPackager>` and are
-available by default at the start of each run.
-"""
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
+# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 from .default_packager import DefaultPackager
 from .numpy_packagers import NumPySupportedFormat

mlrun/projects/project.py

@@ -2116,6 +2116,7 @@ class MlrunProject(ModelObj):
         *,
         deploy_histogram_data_drift_app: bool = True,
         wait_for_deployment: bool = False,
+        rebuild_images: bool = False,
     ) -> None:
         """
         Deploy model monitoring application controller, writer and stream functions.
@@ -2135,6 +2136,7 @@ class MlrunProject(ModelObj):
         :param wait_for_deployment:             If true, return only after the deployment is done on the backend.
                                                 Otherwise, deploy the model monitoring infrastructure on the
                                                 background, including the histogram data drift app if selected.
+        :param rebuild_images:                  If true, force rebuild of model monitoring infrastructure images.
         """
         if default_controller_image != "mlrun/mlrun":
             # TODO: Remove this in 1.9.0
@@ -2150,6 +2152,7 @@ class MlrunProject(ModelObj):
             image=image,
             base_period=base_period,
             deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
+            rebuild_images=rebuild_images,
         )
 
         if wait_for_deployment:
@@ -3192,7 +3195,8 @@ class MlrunProject(ModelObj):
         tsdb_connection: Optional[str] = None,
     ):
         """Set the credentials that will be used by the project's model monitoring
-        infrastructure functions.
+        infrastructure functions. Important to note that you have to set the credentials before deploying any
+        model monitoring or serving function.
 
         :param access_key:                Model Monitoring access key for managing user permissions
         :param endpoint_store_connection: Endpoint store connection string

mlrun/runtimes/nuclio/application/application.py

@@ -263,7 +263,6 @@ class ApplicationRuntime(RemoteRuntime):
         is_kfp=False,
         mlrun_version_specifier=None,
         show_on_failure: bool = False,
-        skip_access_key_auth: bool = False,
         direct_port_access: bool = False,
         authentication_mode: schemas.APIGatewayAuthenticationMode = None,
         authentication_creds: tuple[str] = None,
@@ -283,7 +282,6 @@ class ApplicationRuntime(RemoteRuntime):
         :param is_kfp:                  Deploy as part of a kfp pipeline
         :param mlrun_version_specifier: Which mlrun package version to include (if not current)
         :param show_on_failure:         Show logs only in case of build failure
-        :param skip_access_key_auth:    Skip adding access key auth to the API Gateway
         :param direct_port_access:      Set True to allow direct port access to the application sidecar
         :param authentication_mode:     API Gateway authentication mode
         :param authentication_creds:    API Gateway authentication credentials as a tuple (username, password)

mlrun/runtimes/nuclio/serving.py

@@ -312,15 +312,18 @@ class ServingRuntime(RemoteRuntime):
         sample: Optional[int] = None,
         stream_args: Optional[dict] = None,
         tracking_policy: Optional[Union["TrackingPolicy", dict]] = None,
+        enable_tracking: bool = True,
     ) -> None:
         """apply on your serving function to monitor a deployed model, including real-time dashboards to detect drift
            and analyze performance.
 
-        :param stream_path:     Path/url of the tracking stream e.g. v3io:///users/mike/mystream
-                                you can use the "dummy://" path for test/simulation.
-        :param batch:           Micro batch size (send micro batches of N records at a time).
-        :param sample:          Sample size (send only one of N records).
-        :param stream_args:     Stream initialization parameters, e.g. shards, retention_in_hours, ..
+        :param stream_path:         Path/url of the tracking stream e.g. v3io:///users/mike/mystream
+                                    you can use the "dummy://" path for test/simulation.
+        :param batch:               Micro batch size (send micro batches of N records at a time).
+        :param sample:              Sample size (send only one of N records).
+        :param stream_args:         Stream initialization parameters, e.g. shards, retention_in_hours, ..
+        :param enable_tracking:     Enabled/Disable model-monitoring tracking.
+                                    Default True (tracking enabled).
 
                                 example::
 
@@ -331,7 +334,7 @@ class ServingRuntime(RemoteRuntime):
 
         """
         # Applying model monitoring configurations
-        self.spec.track_models = True
+        self.spec.track_models = enable_tracking
 
         if stream_path:
             self.spec.parameters["log_stream"] = stream_path

mlrun/serving/server.py

@@ -383,6 +383,10 @@ def v2_serving_handler(context, event, get_body=False):
         if event.body == b"":
             event.body = None
 
+    # ML-6065 – workaround for NUC-178
+    if hasattr(event, "trigger") and event.trigger.kind in ("kafka", "kafka-cluster"):
+        event.path = "/"
+
     return context._server.run(event, context, get_body)
 
 

mlrun/serving/v2_serving.py

@@ -542,48 +542,64 @@ def _init_endpoint_record(
         function_uri=graph_server.function_uri, versioned_model=versioned_model_name
     ).uid
 
-    # If model endpoint object was found in DB, skip the creation process.
     try:
-        mlrun.get_run_db().get_model_endpoint(project=project, endpoint_id=uid)
-
+        model_ep = mlrun.get_run_db().get_model_endpoint(
+            project=project, endpoint_id=uid
+        )
     except mlrun.errors.MLRunNotFoundError:
-        logger.info("Creating a new model endpoint record", endpoint_id=uid)
-
-        try:
-            model_endpoint = mlrun.common.schemas.ModelEndpoint(
-                metadata=mlrun.common.schemas.ModelEndpointMetadata(
-                    project=project, labels=model.labels, uid=uid
-                ),
-                spec=mlrun.common.schemas.ModelEndpointSpec(
-                    function_uri=graph_server.function_uri,
-                    model=versioned_model_name,
-                    model_class=model.__class__.__name__,
-                    model_uri=model.model_path,
-                    stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
-                        project=project, kind="stream"
-                    ),
-                    active=True,
-                    monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
-                    if model.context.server.track_models
-                    else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled,
-                ),
-                status=mlrun.common.schemas.ModelEndpointStatus(
-                    endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP
+        model_ep = None
+
+    if model.context.server.track_models and not model_ep:
+        logger.debug("Creating a new model endpoint record", endpoint_id=uid)
+        model_endpoint = mlrun.common.schemas.ModelEndpoint(
+            metadata=mlrun.common.schemas.ModelEndpointMetadata(
+                project=project, labels=model.labels, uid=uid
+            ),
+            spec=mlrun.common.schemas.ModelEndpointSpec(
+                function_uri=graph_server.function_uri,
+                model=versioned_model_name,
+                model_class=model.__class__.__name__,
+                model_uri=model.model_path,
+                stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
+                    project=project, kind="stream"
                 ),
-            )
-
-            db = mlrun.get_run_db()
-
-            db.create_model_endpoint(
-                project=project,
-                endpoint_id=uid,
-                model_endpoint=model_endpoint.dict(),
-            )
+                active=True,
+                monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
+            ),
+            status=mlrun.common.schemas.ModelEndpointStatus(
+                endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP
+            ),
+        )
 
-        except Exception as e:
-            logger.error("Failed to create endpoint record", exc=err_to_str(e))
+        db = mlrun.get_run_db()
+        db.create_model_endpoint(
+            project=project,
+            endpoint_id=uid,
+            model_endpoint=model_endpoint.dict(),
+        )
 
-    except Exception as e:
-        logger.error("Failed to retrieve model endpoint object", exc=err_to_str(e))
+    elif (
+        model_ep
+        and (
+            model_ep.spec.monitoring_mode
+            == mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
+        )
+        != model.context.server.track_models
+    ):
+        monitoring_mode = (
+            mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
+            if model.context.server.track_models
+            else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
+        )
+        db = mlrun.get_run_db()
+        db.patch_model_endpoint(
+            project=project,
+            endpoint_id=uid,
+            attributes={"monitoring_mode": monitoring_mode},
+        )
+        logger.debug(
+            f"Updating model endpoint monitoring_mode to {monitoring_mode}",
+            endpoint_id=uid,
+        )
 
     return uid

mlrun/utils/notifications/notification/base.py

@@ -69,16 +69,27 @@ class NotificationBase:
         if custom_html:
             return custom_html
 
-        if self.name:
-            message = f"{self.name}: {message}"
-
         if alert:
             if not event_data:
                 return f"[{severity}] {message}"
-            return (
-                f"[{severity}] {message} for project {alert.project} "
-                f"UID {event_data.entity.ids[0]}. Values {event_data.value_dict}"
-            )
+
+            html = f"<h3>[{severity}] {message}</h3>"
+            html += f"<br>{alert.name} alert has occurred<br>"
+            html += f"<br><h4>Project:</h4>{alert.project}<br>"
+            html += f"<br><h4>ID:</h4>{event_data.entity.ids[0]}<br>"
+            html += f"<br><h4>Summary:</h4>{mlrun.utils.helpers.format_alert_summary(alert, event_data)}<br>"
+
+            if event_data.value_dict:
+                html += "<br><h4>Event data:</h4>"
+                for key, value in event_data.value_dict.items():
+                    html += f"{key}: {value}<br>"
+
+            overview_type, url = self._get_overview_type_and_url(alert, event_data)
+            html += f"<br><h4>Overview:</h4><a href={url}>{overview_type}</a>"
+            return html
+
+        if self.name:
+            message = f"{self.name}: {message}"
 
         if not runs:
             return f"[{severity}] {message}"
@@ -90,3 +101,24 @@ class NotificationBase:
         html += "<br>click the hyper links below to see detailed results<br>"
         html += runs.show(display=False, short=True)
         return html
+
+    def _get_overview_type_and_url(
+        self,
+        alert: mlrun.common.schemas.AlertConfig,
+        event_data: mlrun.common.schemas.Event,
+    ) -> (str, str):
+        if (
+            event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
+        ):  # JOB entity
+            uid = event_data.value_dict.get("uid")
+            url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
+            overview_type = "Job overview"
+        else:  # MODEL entity
+            model_name = event_data.value_dict.get("model")
+            model_endpoint_id = event_data.value_dict.get("model_endpoint_id")
+            url = mlrun.utils.helpers.get_model_endpoint_url(
+                alert.project, model_name, model_endpoint_id
+            )
+            overview_type = "Model endpoint"
+
+        return overview_type, url

mlrun/utils/notifications/notification/slack.py

@@ -153,20 +153,7 @@ class SlackNotification(NotificationBase):
             data_text = "\n".join(data_lines)
             line.append(self._get_slack_row(f"*Event data:*\n{data_text}"))
 
-        if (
-            event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
-        ):  # JOB entity
-            uid = event_data.value_dict.get("uid")
-            url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
-            overview_type = "Job overview"
-        else:  # MODEL entity
-            model_name = event_data.value_dict.get("model")
-            model_endpoint_id = event_data.value_dict.get("model_endpoint_id")
-            url = mlrun.utils.helpers.get_model_endpoint_url(
-                alert.project, model_name, model_endpoint_id
-            )
-            overview_type = "Model endpoint"
-
+        overview_type, url = self._get_overview_type_and_url(alert, event_data)
         line.append(self._get_slack_row(f"*Overview:*\n<{url}|*{overview_type}*>"))
 
         return line

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "0cbcbb9417af7ac58072ad981672602728fee57e",
-  "version": "1.7.0-rc22"
+  "git_commit": "fb5875ed70c2de21f86d2f112a830946d10bf5e2",
+  "version": "1.7.0-rc24"
 }

{mlrun-1.7.0rc22.dist-info → mlrun-1.7.0rc24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc22
+Version: 1.7.0rc24
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv