mlrun 1.10.0rc39__tar.gz → 1.10.0rc41__tar.gz

{mlrun-1.10.0rc39/mlrun.egg-info → mlrun-1.10.0rc41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc39
+Version: 1.10.0rc41
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/common/model_monitoring/helpers.py

@@ -170,6 +170,6 @@ def log_background_task_state(
             f"Model endpoint creation task is still in progress with the current state: "
             f"{background_task_state}. Events will not be monitored for the next "
             f"{mlrun.mlconf.model_endpoint_monitoring.model_endpoint_creation_check_period} seconds",
-            function_name=server.function.name,
+            function_name=server.function_name,
             background_task_check_timestamp=background_task_check_timestamp.isoformat(),
         )

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/common/secrets.py

@@ -25,6 +25,19 @@ _AUTH_SECRET_NAME_TEMPLATE = re.escape(
 AUTH_SECRET_PATTERN = re.compile(f"^{_AUTH_SECRET_NAME_TEMPLATE}.*")
 
 
+def validate_not_forbidden_secret(secret_name: str) -> None:
+    """
+    Forbid client-supplied references to internal MLRun auth/project secrets.
+    No-op when running inside the API server (API enrichments are allowed).
+    """
+    if not secret_name or mlrun.config.is_running_as_api():
+        return
+    if AUTH_SECRET_PATTERN.match(secret_name):
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            f"Forbidden secret '{secret_name}' matches MLRun auth-secret pattern."
+        )
+
+
 class SecretProviderInterface(ABC):
     @abstractmethod
     def store_auth_secret(

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/model.py

@@ -1619,11 +1619,9 @@ class RunTemplate(ModelObj):
         """
         if kind == "azure_vault" and isinstance(source, dict):
             candidate_secret_name = (source.get("k8s_secret") or "").strip()
-            if candidate_secret_name and mlrun.common.secrets.AUTH_SECRET_PATTERN.match(
-                candidate_secret_name
-            ):
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    f"Forbidden secret '{candidate_secret_name}' matches MLRun auth-secret pattern."
+            if candidate_secret_name:
+                mlrun.common.secrets.validate_not_forbidden_secret(
+                    candidate_secret_name
                 )
         if kind == "vault" and isinstance(source, list):
             source = {"project": self.metadata.project, "secrets": source}

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/model_monitoring/applications/context.py

@@ -249,6 +249,7 @@ class MonitoringApplicationContext:
                 project=self.project_name,
                 endpoint_id=self.endpoint_id,
                 feature_analysis=True,
+                tsdb_metrics=False,
             )
         return self._model_endpoint
 

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/model_monitoring/db/tsdb/base.py

@@ -14,7 +14,7 @@
 
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta
-from typing import Callable, ClassVar, Literal, Optional, Union
+from typing import ClassVar, Literal, Optional, Union
 
 import pandas as pd
 import pydantic.v1
@@ -444,11 +444,9 @@ class TSDBConnector(ABC):
                                    ]
         """
 
-    async def add_basic_metrics(
+    def add_basic_metrics(
         self,
         model_endpoint_objects: list[mlrun.common.schemas.ModelEndpoint],
-        project: str,
-        run_in_threadpool: Callable,
         metric_list: Optional[list[str]] = None,
     ) -> list[mlrun.common.schemas.ModelEndpoint]:
         raise NotImplementedError()

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -14,7 +14,7 @@
 
 import threading
 from datetime import datetime, timedelta
-from typing import Callable, Final, Literal, Optional, Union
+from typing import Final, Literal, Optional, Union
 
 import pandas as pd
 import taosws
@@ -1057,7 +1057,6 @@ class TDEngineConnector(TSDBConnector):
             ]
         ):
             metric_objects = []
-
             if not df_results.empty:
                 df_results.rename(
                     columns={
@@ -1069,7 +1068,9 @@ class TDEngineConnector(TSDBConnector):
                     metric_objects.append(
                         mm_schemas.ApplicationResultRecord(
                             time=datetime.fromisoformat(
-                                row[mm_schemas.WriterEvent.END_INFER_TIME]
+                                row[mm_schemas.WriterEvent.END_INFER_TIME].replace(
+                                    " +", "+"
+                                )
                             ),
                             result_name=row[mm_schemas.ResultData.RESULT_NAME],
                             kind=row[mm_schemas.ResultData.RESULT_KIND],
@@ -1089,7 +1090,9 @@ class TDEngineConnector(TSDBConnector):
                     metric_objects.append(
                         mm_schemas.ApplicationMetricRecord(
                             time=datetime.fromisoformat(
-                                row[mm_schemas.WriterEvent.END_INFER_TIME]
+                                row[mm_schemas.WriterEvent.END_INFER_TIME].replace(
+                                    " +", "+"
+                                )
                             ),
                             metric_name=row[mm_schemas.MetricData.METRIC_NAME],
                             value=row[mm_schemas.MetricData.METRIC_VALUE],
@@ -1248,11 +1251,9 @@ class TDEngineConnector(TSDBConnector):
             df.dropna(inplace=True)
         return df
 
-    async def add_basic_metrics(
+    def add_basic_metrics(
         self,
         model_endpoint_objects: list[mlrun.common.schemas.ModelEndpoint],
-        project: str,
-        run_in_threadpool: Callable,
         metric_list: Optional[list[str]] = None,
     ) -> list[mlrun.common.schemas.ModelEndpoint]:
         """
@@ -1260,8 +1261,6 @@ class TDEngineConnector(TSDBConnector):
 
         :param model_endpoint_objects: A list of `ModelEndpoint` objects that will
                                         be filled with the relevant basic metrics.
-        :param project:                The name of the project.
-        :param run_in_threadpool:      A function that runs another function in a thread pool.
         :param metric_list:            List of metrics to include from the time series DB. Defaults to all metrics.
 
         :return: A list of `ModelEndpointMonitoringMetric` objects.

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import math
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from io import StringIO
-from typing import Callable, Literal, Optional, Union
+from typing import Literal, Optional, Union
 
 import pandas as pd
 import v3io_frames
@@ -1230,11 +1230,9 @@ class V3IOTSDBConnector(TSDBConnector):
             )
         return df.reset_index(drop=True)
 
-    async def add_basic_metrics(
+    def add_basic_metrics(
         self,
         model_endpoint_objects: list[mlrun.common.schemas.ModelEndpoint],
-        project: str,
-        run_in_threadpool: Callable,
         metric_list: Optional[list[str]] = None,
     ) -> list[mlrun.common.schemas.ModelEndpoint]:
         """
@@ -1242,8 +1240,6 @@ class V3IOTSDBConnector(TSDBConnector):
 
         :param model_endpoint_objects: A list of `ModelEndpoint` objects that will
                                        be filled with the relevant basic metrics.
-        :param project:                The name of the project.
-        :param run_in_threadpool:      A function that runs another function in a thread pool.
         :param metric_list:            List of metrics to include from the time series DB. Defaults to all metrics.
 
         :return: A list of `ModelEndpointMonitoringMetric` objects.
@@ -1272,8 +1268,7 @@ class V3IOTSDBConnector(TSDBConnector):
             function,
             _,
         ) in metric_name_to_function_and_column_name.items():
-            metric_name_to_result[metric_name] = await run_in_threadpool(
-                function,
+            metric_name_to_result[metric_name] = function(
                 endpoint_ids=uids,
                 get_raw=True,
             )
@@ -1344,7 +1339,7 @@ class V3IOTSDBConnector(TSDBConnector):
             else:
                 filter_query = app_filter_query
 
-        df = self._get_records(
+        raw_frames: list[v3io_frames.client.RawFrame] = self._get_records(
             table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
             start=start,
             end=end,
@@ -1353,39 +1348,33 @@ class V3IOTSDBConnector(TSDBConnector):
                 mm_schemas.ResultData.RESULT_STATUS,
             ],
             filter_query=filter_query,
+            get_raw=True,
         )
 
-        # filter result status
-        if result_status_list and not df.empty:
-            df = df[df[mm_schemas.ResultData.RESULT_STATUS].isin(result_status_list)]
-
-        if df.empty:
+        if not raw_frames:
             return {}
-        else:
-            # convert application name to lower case
-            df[mm_schemas.ApplicationEvent.APPLICATION_NAME] = df[
-                mm_schemas.ApplicationEvent.APPLICATION_NAME
-            ].str.lower()
-
-            df = (
-                df[
-                    [
-                        mm_schemas.ApplicationEvent.APPLICATION_NAME,
-                        mm_schemas.ResultData.RESULT_STATUS,
-                        mm_schemas.ResultData.RESULT_VALUE,
-                    ]
-                ]
-                .groupby(
-                    [
-                        mm_schemas.ApplicationEvent.APPLICATION_NAME,
-                        mm_schemas.ResultData.RESULT_STATUS,
-                    ],
-                    observed=True,
-                )
-                .count()
-            )
 
-            return df[mm_schemas.ResultData.RESULT_VALUE].to_dict()
+        # Count occurrences by (application_name, result_status) from RawFrame objects
+        count_dict = {}
+
+        for frame in raw_frames:
+            # Extract column data from each RawFrame
+            app_name = frame.column_data(mm_schemas.ApplicationEvent.APPLICATION_NAME)[
+                0
+            ]
+            statuses = frame.column_data(mm_schemas.ResultData.RESULT_STATUS)
+
+            for status in statuses:
+                # Filter by result status if specified
+                if result_status_list and status not in result_status_list:
+                    continue
+
+                # Convert application name to lower case
+                key = (app_name.lower(), status)
+
+                # Update the count in the dictionary
+                count_dict[key] = count_dict.get(key, 0) + 1
+        return count_dict
 
     def count_processed_model_endpoints(
         self,
@@ -1543,51 +1532,140 @@ class V3IOTSDBConnector(TSDBConnector):
     ) -> mm_schemas.ModelEndpointDriftValues:
         table = mm_schemas.V3IOTSDBTables.APP_RESULTS
         start, end, interval = self._prepare_aligned_start_end(start, end)
-        df = self._get_records(
+        raw_frames: list[v3io_frames.client.RawFrame] = self._get_records(
             table=table,
             start=start,
             end=end,
             columns=[mm_schemas.ResultData.RESULT_STATUS],
+            get_raw=True,
         )
-        df = self._aggregate_raw_drift_data(df, start, end, interval)
-        if df.empty:
+
+        if not raw_frames:
+            return mm_schemas.ModelEndpointDriftValues(values=[])
+
+        aggregated_data = self._aggregate_raw_drift_data(
+            raw_frames=raw_frames, start=start, end=end, interval=interval
+        )
+        if not aggregated_data:
+            return mm_schemas.ModelEndpointDriftValues(values=[])
+
+        # Filter to only include entries with max result_status >= 1
+        filtered_data = [
+            (endpoint_id, timestamp, max_status)
+            for endpoint_id, timestamp, max_status in aggregated_data
+            if max_status >= 1
+        ]
+
+        if not filtered_data:
             return mm_schemas.ModelEndpointDriftValues(values=[])
-        df = df[df[f"max({mm_schemas.ResultData.RESULT_STATUS})"] >= 1]
-        return self._df_to_drift_data(df)
+
+        return self._convert_drift_data_to_values(aggregated_data=filtered_data)
 
     @staticmethod
     def _aggregate_raw_drift_data(
-        df: pd.DataFrame, start: datetime, end: datetime, interval: str
-    ) -> pd.DataFrame:
-        if df.empty:
-            return df
-        if not isinstance(df.index, pd.DatetimeIndex):
-            raise TypeError("Expected a DatetimeIndex on the DataFrame (time index).")
-        df[EventFieldType.ENDPOINT_ID] = (
-            df[EventFieldType.ENDPOINT_ID].astype("string").str.strip()
-        )  # remove extra data carried by the category dtype
-        window = df.loc[
-            (df.index >= start) & (df.index < end),
-            [mm_schemas.ResultData.RESULT_STATUS, EventFieldType.ENDPOINT_ID],
+        raw_frames: list[v3io_frames.client.RawFrame],
+        start: datetime,
+        end: datetime,
+        interval: str,
+    ) -> list[tuple[str, datetime, float]]:
+        """
+        Aggregate raw drift data from RawFrame objects.
+
+        :param raw_frames: List of RawFrame objects containing drift data.
+        :param start:      Start datetime for filtering data.
+        :param end:        End datetime for filtering data.
+        :param interval:   Time interval string (e.g., '5min') for aggregation
+
+        :returns: list of tuples: (endpoint_id, timestamp, max_result_status)
+        """
+        if not raw_frames:
+            return []
+
+        # Parse interval to get timedelta
+        interval_td = pd.Timedelta(interval)
+
+        # Collect all data points from RawFrame objects
+        data_points = []
+        for frame in raw_frames:
+            endpoint_id = frame.column_data(EventFieldType.ENDPOINT_ID)[0]
+            result_statuses = frame.column_data(mm_schemas.ResultData.RESULT_STATUS)
+            timestamps = frame.indices()[0].times
+
+            # Combine data from this frame
+            for i, (status, timestamp) in enumerate(zip(result_statuses, timestamps)):
+                # V3IO TSDB returns timestamps in nanoseconds
+                timestamp_dt = pd.Timestamp(
+                    timestamp, unit="ns", tzinfo=timezone.utc
+                ).to_pydatetime()
+
+                # Filter by time window
+                if start <= timestamp_dt < end:
+                    data_points.append((endpoint_id, timestamp_dt, status))
+
+        if not data_points:
+            return []
+
+        # Group by endpoint_id and time intervals, then find max status
+        # Create time buckets aligned to start
+        grouped_data = {}
+        for endpoint_id, timestamp, status in data_points:
+            # Calculate which interval bucket this timestamp falls into
+            time_diff = timestamp - start
+            bucket_index = int(time_diff / interval_td)
+            bucket_start = start + (bucket_index * interval_td)
+
+            key = (endpoint_id, bucket_start)
+            if key not in grouped_data:
+                grouped_data[key] = status
+            else:
+                # Keep the maximum status value
+                grouped_data[key] = max(grouped_data[key], status)
+
+        # Convert to list of tuples
+        result = [
+            (endpoint_id, timestamp, max_status)
+            for (endpoint_id, timestamp), max_status in grouped_data.items()
         ]
-        out = (
-            window.groupby(
-                [
-                    EventFieldType.ENDPOINT_ID,
-                    pd.Grouper(
-                        freq=interval, origin=start, label="left", closed="left"
-                    ),
-                ]
-                # align to start, [start, end) intervals
-            )[mm_schemas.ResultData.RESULT_STATUS]
-            .max()
-            .reset_index()
-            .rename(
-                columns={
-                    mm_schemas.ResultData.RESULT_STATUS: f"max({mm_schemas.ResultData.RESULT_STATUS})"
+
+        return result
+
+    @staticmethod
+    def _convert_drift_data_to_values(
+        aggregated_data: list[tuple[str, datetime, float]],
+    ) -> mm_schemas.ModelEndpointDriftValues:
+        """
+        Convert aggregated drift data to ModelEndpointDriftValues format.
+
+        :param aggregated_data: List of tuples (endpoint_id, timestamp, max_result_status)
+        :return: ModelEndpointDriftValues with counts of suspected and detected per timestamp
+        """
+        suspected_val = mm_schemas.constants.ResultStatusApp.potential_detection.value
+        detected_val = mm_schemas.constants.ResultStatusApp.detected.value
+
+        # Group by timestamp and result status, then count occurrences
+        timestamp_status_counts = {}
+        for _, timestamp, max_status in aggregated_data:
+            key = (timestamp, max_status)
+            timestamp_status_counts[key] = timestamp_status_counts.get(key, 0) + 1
+
+        # Organize by timestamp with counts for suspected and detected
+        timestamp_counts = {}
+        for (timestamp, status), count in timestamp_status_counts.items():
+            if timestamp not in timestamp_counts:
+                timestamp_counts[timestamp] = {
+                    "count_suspected": 0,
+                    "count_detected": 0,
                 }
-            )
-        )
-        return out.rename(
-            columns={"time": "_wstart"}
-        )  # rename datetime column to _wstart to align with the tdengine result
+
+            if status == suspected_val:
+                timestamp_counts[timestamp]["count_suspected"] = count
+            elif status == detected_val:
+                timestamp_counts[timestamp]["count_detected"] = count
+
+        # Convert to the expected format: list of (timestamp, count_suspected, count_detected)
+        values = [
+            (timestamp, counts["count_suspected"], counts["count_detected"])
+            for timestamp, counts in sorted(timestamp_counts.items())
+        ]
+
+        return mm_schemas.ModelEndpointDriftValues(values=values)

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/projects/project.py

@@ -3455,11 +3455,9 @@ class MlrunProject(ModelObj):
         # Block using mlrun-auth-secrets.* via azure_vault's k8s_secret param (client-side only)
         if kind == "azure_vault" and isinstance(source, dict):
             candidate_secret_name = (source.get("k8s_secret") or "").strip()
-            if candidate_secret_name and mlrun.common.secrets.AUTH_SECRET_PATTERN.match(
-                candidate_secret_name
-            ):
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    f"Forbidden secret '{candidate_secret_name}' matches MLRun auth-secret pattern."
+            if candidate_secret_name:
+                mlrun.common.secrets.validate_not_forbidden_secret(
+                    candidate_secret_name
                 )
 
         if kind == "vault" and isinstance(source, list):

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/runtimes/mounts.py

@@ -414,12 +414,8 @@ def mount_secret(
                          the specified paths, and unlisted keys will not be
                          present."""
 
-    if secret_name and mlrun.common.secrets.AUTH_SECRET_PATTERN.match(
-        secret_name.strip()
-    ):
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"Forbidden secret '{secret_name}' matches MLRun auth-secret pattern."
-        )
+    if secret_name:
+        mlrun.common.secrets.validate_not_forbidden_secret(secret_name.strip())
 
     def _mount_secret(runtime: "KubeResource"):
         # Define the secret volume source

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/runtimes/nuclio/serving.py

@@ -23,6 +23,7 @@ from nuclio import KafkaTrigger
 
 import mlrun
 import mlrun.common.schemas as schemas
+import mlrun.common.secrets
 import mlrun.datastore.datastore_profile as ds_profile
 from mlrun.datastore import get_kafka_brokers_from_dict, parse_kafka_url
 from mlrun.model import ObjectList
@@ -44,7 +45,6 @@ from mlrun.serving.states import (
 )
 from mlrun.utils import get_caller_globals, logger, set_paths
 
-from ...common.secrets import AUTH_SECRET_PATTERN
 from .. import KubejobRuntime
 from ..pod import KubeResourceSpec
 from .function import NuclioSpec, RemoteRuntime, min_nuclio_versions
@@ -638,11 +638,9 @@ class ServingRuntime(RemoteRuntime):
         """
         if kind == "azure_vault" and isinstance(source, dict):
             candidate_secret_name = (source.get("k8s_secret") or "").strip()
-            if candidate_secret_name and AUTH_SECRET_PATTERN.match(
-                candidate_secret_name
-            ):
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    f"Forbidden secret '{candidate_secret_name}' matches MLRun auth-secret pattern."
+            if candidate_secret_name:
+                mlrun.common.secrets.validate_not_forbidden_secret(
+                    candidate_secret_name
                 )
         if kind == "vault" and isinstance(source, list):
             source = {"project": self.metadata.project, "secrets": source}

{mlrun-1.10.0rc39 → mlrun-1.10.0rc41}/mlrun/runtimes/pod.py

@@ -27,6 +27,7 @@ import kubernetes.client as k8s_client
 from kubernetes.client import V1Volume, V1VolumeMount
 
 import mlrun.common.constants
+import mlrun.common.secrets
 import mlrun.errors
 import mlrun.runtimes.mounts
 import mlrun.utils.regex
@@ -35,7 +36,6 @@ from mlrun.common.schemas import (
     SecurityContextEnrichmentModes,
 )
 
-from ..common.secrets import AUTH_SECRET_PATTERN
 from ..config import config as mlconf
 from ..k8s_utils import (
     generate_preemptible_nodes_affinity_terms,
@@ -716,21 +716,16 @@ class KubeResource(BaseRuntime):
         secret: Optional[str] = None,
         secret_key: Optional[str] = None,
     ):
-        """Set env var from secret; block auth-secret usage on client side."""
-        self._validate_no_auth_secret(
-            secret_name=secret,
-        )
-        secret_key = secret_key or name
+        """
+        Set an environment variable from a Kubernetes Secret.
+        Client-side guard forbids MLRun internal auth/project secrets; no-op on API.
+        """
+        mlrun.common.secrets.validate_not_forbidden_secret(secret)
+        key = secret_key or name
         value_from = k8s_client.V1EnvVarSource(
-            secret_key_ref=k8s_client.V1SecretKeySelector(
-                name=secret,
-                key=secret_key,
-            )
-        )
-        return self._set_env(
-            name=name,
-            value_from=value_from,
+            secret_key_ref=k8s_client.V1SecretKeySelector(name=secret, key=key)
         )
+        return self._set_env(name=name, value_from=value_from)
 
     def set_env(
         self,
@@ -738,16 +733,22 @@ class KubeResource(BaseRuntime):
         value: Optional[str] = None,
         value_from: Optional[typing.Any] = None,
     ):
-        """Set env var; block auth-secret usage when coming from a secret."""
+        """
+        Set an environment variable.
+        If value comes from a Secret, validate on client-side only.
+        """
         if value_from is not None:
             secret_name = self._extract_secret_name_from_value_from(
-                value_from=value_from,
-            )
-            self._validate_no_auth_secret(
-                secret_name=secret_name,
+                value_from=value_from
             )
-            return self._set_env(name, value_from=value_from)
-        return self._set_env(name, value=str(value) if value is not None else None)
+            if secret_name:
+                mlrun.common.secrets.validate_not_forbidden_secret(secret_name)
+            return self._set_env(name=name, value_from=value_from)
+
+        # Plain literal value path
+        return self._set_env(
+            name=name, value=(str(value) if value is not None else None)
+        )
 
     def with_annotations(self, annotations: dict):
         """set a key/value annotations in the metadata of the pod"""
@@ -1393,16 +1394,6 @@ class KubeResource(BaseRuntime):
 
         return self.status.state
 
-    @staticmethod
-    def _validate_no_auth_secret(
-        secret_name: Optional[str],
-    ):
-        """Raise if secret name matches MLRun auth-secret pattern."""
-        if secret_name and AUTH_SECRET_PATTERN.match(secret_name):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                f"Forbidden secret '{secret_name}' matches MLRun auth-secret pattern."
-            )
-
     @staticmethod
     def _extract_secret_name_from_value_from(
         value_from: typing.Any,