mlrun 1.10.0rc40__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import math
+from collections import defaultdict
 from datetime import datetime, timedelta
 from io import StringIO
-from typing import Callable, Literal, Optional, Union
+from typing import Literal, Optional, Union
 
 import pandas as pd
 import v3io_frames
@@ -225,6 +226,24 @@ class V3IOTSDBConnector(TSDBConnector):
         - endpoint_features (Prediction and feature names and values)
         - custom_metrics (user-defined metrics)
         """
+
+        def apply_list_to_single_dict():
+            graph.add_step(
+                "storey.Map",
+                "MapListToSingleDict",
+                after="FilterNOP",
+                _fn="(event[0] if isinstance(event, list) else event)",
+            )
+            graph.add_step(
+                "mlrun.model_monitoring.stream_processing.MapFeatureNames",
+                name="MapFeatureNamesTSDB",
+                infer_columns_from_data=True,
+                project=self.project,
+                after="MapListToSingleDict",
+            )
+
+        apply_list_to_single_dict()
+
         aggregate_windows = aggregate_windows or ["5m", "1h"]
 
         # Calculate number of predictions and average latency
@@ -242,7 +261,7 @@ class V3IOTSDBConnector(TSDBConnector):
                     }
                 ],
                 name=EventFieldType.LATENCY,
-                after="FilterNOP",
+                after="MapFeatureNamesTSDB",
                 step_name="Aggregates",
                 table=".",
                 key_field=EventFieldType.ENDPOINT_ID,
@@ -263,7 +282,7 @@ class V3IOTSDBConnector(TSDBConnector):
         graph.add_step(
             "storey.TSDBTarget",
             name="tsdb_predictions",
-            after="FilterNOP",
+            after="MapFeatureNamesTSDB",
             path=f"{self.container}/{self.tables[mm_schemas.V3IOTSDBTables.PREDICTIONS]}",
             time_col=mm_schemas.EventFieldType.TIMESTAMP,
             container=self.container,
@@ -1230,11 +1249,9 @@ class V3IOTSDBConnector(TSDBConnector):
             )
         return df.reset_index(drop=True)
 
-    async def add_basic_metrics(
+    def add_basic_metrics(
         self,
         model_endpoint_objects: list[mlrun.common.schemas.ModelEndpoint],
-        project: str,
-        run_in_threadpool: Callable,
         metric_list: Optional[list[str]] = None,
     ) -> list[mlrun.common.schemas.ModelEndpoint]:
         """
@@ -1242,8 +1259,6 @@ class V3IOTSDBConnector(TSDBConnector):
 
         :param model_endpoint_objects: A list of `ModelEndpoint` objects that will
                                        be filled with the relevant basic metrics.
-        :param project:                The name of the project.
-        :param run_in_threadpool:      A function that runs another function in a thread pool.
         :param metric_list:            List of metrics to include from the time series DB. Defaults to all metrics.
 
         :return: A list of `ModelEndpointMonitoringMetric` objects.
@@ -1272,8 +1287,7 @@ class V3IOTSDBConnector(TSDBConnector):
             function,
             _,
         ) in metric_name_to_function_and_column_name.items():
-            metric_name_to_result[metric_name] = await run_in_threadpool(
-                function,
+            metric_name_to_result[metric_name] = function(
                 endpoint_ids=uids,
                 get_raw=True,
             )
@@ -1344,7 +1358,7 @@ class V3IOTSDBConnector(TSDBConnector):
             else:
                 filter_query = app_filter_query
 
-        df = self._get_records(
+        raw_frames: list[v3io_frames.client.RawFrame] = self._get_records(
             table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
             start=start,
             end=end,
@@ -1353,39 +1367,33 @@ class V3IOTSDBConnector(TSDBConnector):
                 mm_schemas.ResultData.RESULT_STATUS,
             ],
             filter_query=filter_query,
+            get_raw=True,
         )
 
-        # filter result status
-        if result_status_list and not df.empty:
-            df = df[df[mm_schemas.ResultData.RESULT_STATUS].isin(result_status_list)]
-
-        if df.empty:
+        if not raw_frames:
             return {}
-        else:
-            # convert application name to lower case
-            df[mm_schemas.ApplicationEvent.APPLICATION_NAME] = df[
-                mm_schemas.ApplicationEvent.APPLICATION_NAME
-            ].str.lower()
-
-            df = (
-                df[
-                    [
-                        mm_schemas.ApplicationEvent.APPLICATION_NAME,
-                        mm_schemas.ResultData.RESULT_STATUS,
-                        mm_schemas.ResultData.RESULT_VALUE,
-                    ]
-                ]
-                .groupby(
-                    [
-                        mm_schemas.ApplicationEvent.APPLICATION_NAME,
-                        mm_schemas.ResultData.RESULT_STATUS,
-                    ],
-                    observed=True,
-                )
-                .count()
-            )
 
-            return df[mm_schemas.ResultData.RESULT_VALUE].to_dict()
+        # Count occurrences by (application_name, result_status) from RawFrame objects
+        count_dict = {}
+
+        for frame in raw_frames:
+            # Extract column data from each RawFrame
+            app_name = frame.column_data(mm_schemas.ApplicationEvent.APPLICATION_NAME)[
+                0
+            ]
+            statuses = frame.column_data(mm_schemas.ResultData.RESULT_STATUS)
+
+            for status in statuses:
+                # Filter by result status if specified
+                if result_status_list and status not in result_status_list:
+                    continue
+
+                # Convert application name to lower case
+                key = (app_name.lower(), status)
+
+                # Update the count in the dictionary
+                count_dict[key] = count_dict.get(key, 0) + 1
+        return count_dict
 
     def count_processed_model_endpoints(
         self,
@@ -1543,51 +1551,111 @@ class V3IOTSDBConnector(TSDBConnector):
     ) -> mm_schemas.ModelEndpointDriftValues:
         table = mm_schemas.V3IOTSDBTables.APP_RESULTS
         start, end, interval = self._prepare_aligned_start_end(start, end)
-        df = self._get_records(
+
+        raw_frames: list[v3io_frames.client.RawFrame] = self._get_records(
             table=table,
             start=start,
             end=end,
             columns=[mm_schemas.ResultData.RESULT_STATUS],
+            get_raw=True,
         )
-        df = self._aggregate_raw_drift_data(df, start, end, interval)
-        if df.empty:
+
+        if not raw_frames:
             return mm_schemas.ModelEndpointDriftValues(values=[])
-        df = df[df[f"max({mm_schemas.ResultData.RESULT_STATUS})"] >= 1]
-        return self._df_to_drift_data(df)
+
+        # Combine aggregation, filtering, and conversion in one pass
+        drift_values = self._process_drifted_endpoints_data(
+            raw_frames=raw_frames, start=start, end=end, interval=interval
+        )
+
+        return drift_values
 
     @staticmethod
-    def _aggregate_raw_drift_data(
-        df: pd.DataFrame, start: datetime, end: datetime, interval: str
-    ) -> pd.DataFrame:
-        if df.empty:
-            return df
-        if not isinstance(df.index, pd.DatetimeIndex):
-            raise TypeError("Expected a DatetimeIndex on the DataFrame (time index).")
-        df[EventFieldType.ENDPOINT_ID] = (
-            df[EventFieldType.ENDPOINT_ID].astype("string").str.strip()
-        )  # remove extra data carried by the category dtype
-        window = df.loc[
-            (df.index >= start) & (df.index < end),
-            [mm_schemas.ResultData.RESULT_STATUS, EventFieldType.ENDPOINT_ID],
-        ]
-        out = (
-            window.groupby(
-                [
-                    EventFieldType.ENDPOINT_ID,
-                    pd.Grouper(
-                        freq=interval, origin=start, label="left", closed="left"
-                    ),
-                ]
-                # align to start, [start, end) intervals
-            )[mm_schemas.ResultData.RESULT_STATUS]
-            .max()
-            .reset_index()
-            .rename(
-                columns={
-                    mm_schemas.ResultData.RESULT_STATUS: f"max({mm_schemas.ResultData.RESULT_STATUS})"
-                }
+    def _process_drifted_endpoints_data(
+        raw_frames: list[v3io_frames.client.RawFrame],
+        start: datetime,
+        end: datetime,
+        interval: str,
+    ) -> mm_schemas.ModelEndpointDriftValues:
+        """
+        Optimized single-pass processing of drift data from RawFrame objects.
+        Combines aggregation, filtering, and conversion into one operation.
+
+        :param raw_frames: List of RawFrame objects containing drift data.
+        :param start:      Start datetime for filtering data.
+        :param end:        End datetime for filtering data.
+        :param interval:   Time interval string (e.g., '5min') for aggregation
+
+        :returns: ModelEndpointDriftValues with counts of suspected and detected per timestamp
+        """
+
+        if not raw_frames:
+            return mm_schemas.ModelEndpointDriftValues(values=[])
+
+        # Pre-compute constants
+        interval_td = pd.Timedelta(interval)
+        interval_ns = interval_td.value  # nanoseconds for integer arithmetic
+        start_ns = pd.Timestamp(start).value
+        end_ns = pd.Timestamp(end).value
+
+        suspected_val = mm_schemas.constants.ResultStatusApp.potential_detection.value
+        detected_val = mm_schemas.constants.ResultStatusApp.detected.value
+
+        # Single dictionary to track: bucket_start_ns -> {endpoint_id -> max_status}
+        # This allows us to calculate max per endpoint per bucket in one pass
+        bucket_endpoint_status = defaultdict(dict)
+
+        for frame in raw_frames:
+            endpoint_id = frame.column_data(EventFieldType.ENDPOINT_ID)[0]
+            result_statuses = frame.column_data(mm_schemas.ResultData.RESULT_STATUS)
+            timestamps = frame.indices()[0].times
+
+            for status, timestamp_ns in zip(result_statuses, timestamps):
+                # Early skip: invalid status or outside time range
+                if status is None or math.isnan(status) or status < 1:
+                    continue
+                if not (start_ns <= timestamp_ns < end_ns):
+                    continue
+
+                # Calculate bucket using integer arithmetic
+                bucket_index = (timestamp_ns - start_ns) // interval_ns
+                bucket_start_ns = start_ns + (bucket_index * interval_ns)
+
+                # Initialize bucket if needed
+                bucket = bucket_endpoint_status[bucket_start_ns]
+                bucket[endpoint_id] = max(bucket.get(endpoint_id, status), status)
+
+        if not bucket_endpoint_status:
+            return mm_schemas.ModelEndpointDriftValues(values=[])
+
+        # Second pass: count suspected/detected per timestamp bucket
+        # Structure: bucket_start_ns -> {count_suspected, count_detected}
+        timestamp_counts = {}
+
+        for bucket_start_ns, endpoint_statuses in bucket_endpoint_status.items():
+            count_suspected = 0
+            count_detected = 0
+
+            for status in endpoint_statuses.values():
+                if status == suspected_val:
+                    count_suspected += 1
+                elif status == detected_val:
+                    count_detected += 1
+
+            # Only store if there are counts
+            if count_suspected > 0 or count_detected > 0:
+                timestamp_counts[bucket_start_ns] = (count_suspected, count_detected)
+
+        # Convert to final format (sorted by timestamp)
+        values = [
+            (
+                pd.Timestamp(bucket_ns, unit="ns", tz="UTC").to_pydatetime(),
+                count_suspected,
+                count_detected,
             )
-        )
-        return out.rename(
-            columns={"time": "_wstart"}
-        )  # rename datetime column to _wstart to align with the tdengine result
+            for bucket_ns, (count_suspected, count_detected) in sorted(
+                timestamp_counts.items()
+            )
+        ]
+
+        return mm_schemas.ModelEndpointDriftValues(values=values)

mlrun/model_monitoring/features_drift_table.py

@@ -14,7 +14,8 @@
 
 import functools
 import sys
-from typing import Callable, Union
+from collections.abc import Callable
+from typing import Union
 
 import numpy as np
 import plotly.graph_objects as go

mlrun/model_monitoring/helpers.py

@@ -15,8 +15,9 @@
 import datetime
 import functools
 import os
+from collections.abc import Callable
 from fnmatch import fnmatchcase
-from typing import TYPE_CHECKING, Callable, Optional, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Optional, TypedDict, Union, cast
 
 import numpy as np
 import pandas as pd

mlrun/model_monitoring/stream_processing.py

@@ -200,9 +200,25 @@ class EventStreamProcessor:
                 after="ProcessEndpointEvent",
             )
 
+            # split the graph between NOP event to regular event
+            graph.add_step(
+                "storey.Filter",
+                "FilterNOP",
+                after="filter_none",
+                _fn="(not (isinstance(event, dict) and event.get('kind', "
+                ") == 'nop_event'))",
+            )
+            graph.add_step(
+                "storey.Filter",
+                "ForwardNOP",
+                after="filter_none",
+                _fn="(isinstance(event, dict) and event.get('kind', "
+                ") == 'nop_event')",
+            )
+
             # flatten the events
             graph.add_step(
-                "storey.FlatMap", "flatten_events", _fn="(event)", after="filter_none"
+                "storey.FlatMap", "flatten_events", _fn="(event)", after="FilterNOP"
             )
 
         apply_storey_filter_and_flatmap()
@@ -218,19 +234,6 @@ class EventStreamProcessor:
             )
 
         apply_map_feature_names()
-        # split the graph between event with error vs valid event
-        graph.add_step(
-            "storey.Filter",
-            "FilterNOP",
-            after="MapFeatureNames",
-            _fn="(event.get('kind', " ") != 'nop_event')",
-        )
-        graph.add_step(
-            "storey.Filter",
-            "ForwardNOP",
-            after="MapFeatureNames",
-            _fn="(event.get('kind', " ") == 'nop_event')",
-        )
 
         tsdb_connector.apply_monitoring_stream_steps(
             graph=graph,
@@ -244,7 +247,7 @@ class EventStreamProcessor:
             graph.add_step(
                 "ProcessBeforeParquet",
                 name="ProcessBeforeParquet",
-                after="FilterNOP",
+                after="MapFeatureNames",
                 _fn="(event)",
             )
 
@@ -370,7 +373,6 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
             logger.debug(
                 "Skipped nop event inside of ProcessEndpointEvent", event=event
             )
-            full_event.body = [event]
             return full_event
         # Getting model version and function uri from event
         # and use them for retrieving the endpoint_id

mlrun/model_monitoring/writer.py

@@ -14,8 +14,9 @@
 
 import json
 import typing
-from datetime import datetime, timezone
-from typing import Any, Callable, NewType, Optional
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Any, NewType, Optional
 
 import storey
 
@@ -171,7 +172,7 @@ class ModelMonitoringWriter(StepToDict):
         )
         stat_kind = event.get(StatsData.STATS_NAME)
         data, timestamp_str = event.get(StatsData.STATS), event.get(StatsData.TIMESTAMP)
-        timestamp = datetime.fromisoformat(timestamp_str).astimezone(tz=timezone.utc)
+        timestamp = datetime.fromisoformat(timestamp_str).astimezone(tz=UTC)
         if stat_kind == StatsKind.CURRENT_STATS.value:
             ModelMonitoringCurrentStatsFile(self.project, endpoint_id).write(
                 data, timestamp

mlrun/package/__init__.py

@@ -15,7 +15,8 @@
 import functools
 import inspect
 from collections import OrderedDict
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 from ..config import config
 from .context_handler import ContextHandler

mlrun/platforms/__init__.py

@@ -25,50 +25,6 @@ from .iguazio import (
 )
 
 
-# TODO: Remove in 1.11.0
-class _DeprecationHelper:
-    """A helper class to deprecate old schemas"""
-
-    def __init__(self, new_target: str, version="1.8.0"):
-        self._new_target = new_target
-        self._version = version
-
-    def __call__(self, *args, **kwargs):
-        self._warn()
-        return self._lazy_load()(*args, **kwargs)
-
-    def __getattr__(self, attr):
-        self._warn()
-        return getattr(self._lazy_load(), attr)
-
-    def _lazy_load(self, *args, **kwargs):
-        import mlrun.runtimes.mounts as mlrun_mounts
-
-        return getattr(mlrun_mounts, self._new_target)
-
-    def _warn(self):
-        warnings.warn(
-            f"mlrun.platforms.{self._new_target} is deprecated since version {self._version}, "
-            f"and will be removed in 1.11.0. Use mlrun.runtimes.mounts.{self._new_target} instead.",
-            FutureWarning,
-        )
-
-
-# TODO: Remove in 1.11.0
-# For backwards compatibility
-VolumeMount = _DeprecationHelper("VolumeMount")
-auto_mount = _DeprecationHelper("auto_mount")
-mount_configmap = _DeprecationHelper("mount_configmap")
-mount_hostpath = _DeprecationHelper("mount_hostpath")
-mount_pvc = _DeprecationHelper("mount_pvc")
-mount_s3 = _DeprecationHelper("mount_s3")
-mount_secret = _DeprecationHelper("mount_secret")
-mount_v3io = _DeprecationHelper("mount_v3io")
-set_env_variables = _DeprecationHelper("set_env_variables")
-v3io_cred = _DeprecationHelper("v3io_cred")
-# eof 'For backwards compatibility'
-
-
 def watch_stream(
     url,
     shard_ids: Optional[list] = None,

mlrun/platforms/iguazio.py

@@ -140,7 +140,7 @@ class OutputStream:
         self._lazy_init()
 
         def dump_record(rec):
-            if not isinstance(rec, (str, bytes)):
+            if not isinstance(rec, str | bytes):
                 return dict_to_json(rec)
             return str(rec)
 

mlrun/projects/operations.py

@@ -85,17 +85,17 @@ def run_function(
 ) -> Union[mlrun.model.RunObject, mlrun_pipelines.models.PipelineNodeWrapper]:
     """Run a local or remote task as part of a local/kubeflow pipeline
 
-    run_function() allow you to execute a function locally, on a remote cluster, or as part of an automated workflow
-    function can be specified as an object or by name (str), when the function is specified by name it is looked up
-    in the current project eliminating the need to redefine/edit functions.
+    run_function() allows you to execute a function locally, on a remote cluster, or as part of an automated workflow.
+    The function can be specified as an object or by name (str). When the function is specified by name it is looked up
+    in the current project, eliminating the need to redefine/edit functions.
 
-    when functions run as part of a workflow/pipeline (project.run()) some attributes can be set at the run level,
+    When functions run as part of a workflow/pipeline (project.run()) some attributes can be set at the run level,
     e.g. local=True will run all the functions locally, setting artifact_path will direct all outputs to the same path.
-    project runs provide additional notifications/reporting and exception handling.
-    inside a Kubeflow pipeline (KFP) run_function() generates KFP node (see PipelineNodeWrapper) which forms a DAG
-    some behavior may differ between regular runs and deferred KFP runs.
+    Project runs provide additional notifications/reporting and exception handling.
+    Inside a Kubeflow pipeline (KFP) run_function() generates KFP node (see PipelineNodeWrapper) which forms a DAG.
+    Some behavior may differ between regular runs and deferred KFP runs.
 
-    example (use with function object)::
+    Example (use with function object)::
 
         LABELS = "is_error"
         MODEL_CLASS = "sklearn.ensemble.RandomForestClassifier"
@@ -107,7 +107,7 @@ def run_function(
             inputs={"dataset": DATA_PATH},
         )
 
-    example (use with project)::
+    Example (use with project)::
 
         # create a project with two functions (local and from hub)
         project = mlrun.new_project(project_name, "./proj)
@@ -119,7 +119,7 @@ def run_function(
         run2 = run_function("train", params={"label_columns": LABELS, "model_class": MODEL_CLASS},
                                      inputs={"dataset": run1.outputs["data"]})
 
-    example (use in pipeline)::
+    Example (use in pipeline)::
 
         @dsl.pipeline(name="test pipeline", description="test")
         def my_pipe(url=""):
@@ -405,6 +405,7 @@ def deploy_function(
     :param project_object:  override the project object to use, will default to the project set in the runtime context.
     """
     engine, function = _get_engine_and_function(function, project_object)
+    # TODO in ML-11599 need to handle redeployment with different auth token name
     if function.kind not in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
         raise mlrun.errors.MLRunInvalidArgumentError(
             "deploy is used with real-time functions, for other kinds use build_function()"