mlrun 1.7.0rc37__py3-none-any.whl → 1.7.0rc38__py3-none-any.whl

mlrun/alerts/alert.py

@@ -29,6 +29,7 @@ class AlertConfig(ModelObj):
         "reset_policy",
         "state",
         "count",
+        "created",
     ]
     _fields_to_serialize = ModelObj._fields_to_serialize + [
         "entities",
@@ -55,12 +56,12 @@ class AlertConfig(ModelObj):
         created: str = None,
         count: int = None,
     ):
-        """
-        Alert config object
+        """Alert config object
 
         Example::
+
             # create an alert on endpoint_id, which will be triggered to slack if there is a "data_drift_detected" event
-            3 times in the next hour.
+            # 3 times in the next hour.
             from mlrun.alerts import AlertConfig
             import mlrun.common.schemas.alert as alert_objects
 

mlrun/common/schemas/model_monitoring/constants.py

@@ -53,9 +53,11 @@ class EventFieldType:
     PREDICTIONS = "predictions"
     NAMED_PREDICTIONS = "named_predictions"
     ERROR_COUNT = "error_count"
+    MODEL_ERROR = "model_error"
     ENTITIES = "entities"
     FIRST_REQUEST = "first_request"
     LAST_REQUEST = "last_request"
+    LAST_REQUEST_TIMESTAMP = "last_request_timestamp"
     METRIC = "metric"
     METRICS = "metrics"
     BATCH_INTERVALS_DICT = "batch_intervals_dict"
@@ -217,6 +219,7 @@ class FileTargetKind:
     APP_METRICS = "app_metrics"
     MONITORING_SCHEDULES = "monitoring_schedules"
     MONITORING_APPLICATION = "monitoring_application"
+    ERRORS = "errors"
 
 
 class ModelMonitoringMode(str, Enum):
@@ -240,6 +243,7 @@ class V3IOTSDBTables(MonitoringStrEnum):
     APP_RESULTS = "app-results"
     METRICS = "metrics"
     EVENTS = "events"
+    ERRORS = "errors"
 
 
 class TDEngineSuperTables(MonitoringStrEnum):

mlrun/common/schemas/notification.py

@@ -71,9 +71,9 @@ class Notification(pydantic.BaseModel):
 
     kind: NotificationKind
     name: str
-    message: str
-    severity: NotificationSeverity
-    when: list[str]
+    message: typing.Optional[str] = None
+    severity: typing.Optional[NotificationSeverity] = None
+    when: typing.Optional[list[str]] = None
     condition: typing.Optional[str] = None
     params: typing.Optional[dict[str, typing.Any]] = None
     status: typing.Optional[NotificationStatus] = None

mlrun/datastore/azure_blob.py

@@ -16,6 +16,7 @@ import time
 from pathlib import Path
 from urllib.parse import urlparse
 
+from azure.storage.blob import BlobServiceClient
 from azure.storage.blob._shared.base_client import parse_connection_str
 from fsspec.registry import get_filesystem_class
 
@@ -29,47 +30,128 @@ from .base import DataStore, FileStats, makeDatastoreSchemaSanitizer
 
 class AzureBlobStore(DataStore):
     using_bucket = True
+    max_concurrency = 100
+    max_blocksize = 1024 * 1024 * 4
+    max_single_put_size = (
+        1024 * 1024 * 8
+    )  # for service_client property only, does not affect filesystem
 
     def __init__(self, parent, schema, name, endpoint="", secrets: dict = None):
         super().__init__(parent, name, schema, endpoint, secrets=secrets)
+        self._service_client = None
+        self._storage_options = None
+
+    @property
+    def storage_options(self):
+        if not self._storage_options:
+            res = dict(
+                account_name=self._get_secret_or_env("account_name")
+                or self._get_secret_or_env("AZURE_STORAGE_ACCOUNT_NAME"),
+                account_key=self._get_secret_or_env("account_key")
+                or self._get_secret_or_env("AZURE_STORAGE_ACCOUNT_KEY"),
+                connection_string=self._get_secret_or_env("connection_string")
+                or self._get_secret_or_env("AZURE_STORAGE_CONNECTION_STRING"),
+                tenant_id=self._get_secret_or_env("tenant_id")
+                or self._get_secret_or_env("AZURE_STORAGE_TENANT_ID"),
+                client_id=self._get_secret_or_env("client_id")
+                or self._get_secret_or_env("AZURE_STORAGE_CLIENT_ID"),
+                client_secret=self._get_secret_or_env("client_secret")
+                or self._get_secret_or_env("AZURE_STORAGE_CLIENT_SECRET"),
+                sas_token=self._get_secret_or_env("sas_token")
+                or self._get_secret_or_env("AZURE_STORAGE_SAS_TOKEN"),
+                credential=self._get_secret_or_env("credential"),
+            )
+            self._storage_options = self._sanitize_storage_options(res)
+        return self._storage_options
 
     @property
     def filesystem(self):
         """return fsspec file system object, if supported"""
-        if self._filesystem:
-            return self._filesystem
         try:
             import adlfs  # noqa
         except ImportError as exc:
             raise ImportError("Azure adlfs not installed") from exc
-        # in order to support az and wasbs kinds.
-        filesystem_class = get_filesystem_class(protocol=self.kind)
-        self._filesystem = makeDatastoreSchemaSanitizer(
-            filesystem_class,
-            using_bucket=self.using_bucket,
-            **self.get_storage_options(),
-        )
+
+        if not self._filesystem:
+            # in order to support az and wasbs kinds
+            filesystem_class = get_filesystem_class(protocol=self.kind)
+            self._filesystem = makeDatastoreSchemaSanitizer(
+                filesystem_class,
+                using_bucket=self.using_bucket,
+                blocksize=self.max_blocksize,
+                **self.storage_options,
+            )
         return self._filesystem
 
-    def get_storage_options(self):
-        res = dict(
-            account_name=self._get_secret_or_env("account_name")
-            or self._get_secret_or_env("AZURE_STORAGE_ACCOUNT_NAME"),
-            account_key=self._get_secret_or_env("account_key")
-            or self._get_secret_or_env("AZURE_STORAGE_KEY"),
-            connection_string=self._get_secret_or_env("connection_string")
-            or self._get_secret_or_env("AZURE_STORAGE_CONNECTION_STRING"),
-            tenant_id=self._get_secret_or_env("tenant_id")
-            or self._get_secret_or_env("AZURE_STORAGE_TENANT_ID"),
-            client_id=self._get_secret_or_env("client_id")
-            or self._get_secret_or_env("AZURE_STORAGE_CLIENT_ID"),
-            client_secret=self._get_secret_or_env("client_secret")
-            or self._get_secret_or_env("AZURE_STORAGE_CLIENT_SECRET"),
-            sas_token=self._get_secret_or_env("sas_token")
-            or self._get_secret_or_env("AZURE_STORAGE_SAS_TOKEN"),
-            credential=self._get_secret_or_env("credential"),
-        )
-        return self._sanitize_storage_options(res)
+    @property
+    def service_client(self):
+        try:
+            import azure  # noqa
+        except ImportError as exc:
+            raise ImportError("Azure not installed") from exc
+
+        if not self._service_client:
+            self._do_connect()
+        return self._service_client
+
+    def _do_connect(self):
+        """
+
+        Creates a client for azure.
+        Raises MLRunInvalidArgumentError if none of the connection details are available
+        based on do_connect in AzureBlobFileSystem:
+        https://github.com/fsspec/adlfs/blob/2023.9.0/adlfs/spec.py#L422
+        """
+        from azure.identity import ClientSecretCredential
+
+        storage_options = self.storage_options
+        connection_string = storage_options.get("connection_string")
+        client_name = storage_options.get("account_name")
+        account_key = storage_options.get("account_key")
+        sas_token = storage_options.get("sas_token")
+        client_id = storage_options.get("client_id")
+        credential = storage_options.get("credential")
+
+        credential_from_client_id = None
+        if (
+            credential is None
+            and account_key is None
+            and sas_token is None
+            and client_id is not None
+        ):
+            credential_from_client_id = ClientSecretCredential(
+                tenant_id=storage_options.get("tenant_id"),
+                client_id=client_id,
+                client_secret=storage_options.get("client_secret"),
+            )
+        try:
+            if connection_string is not None:
+                self._service_client = BlobServiceClient.from_connection_string(
+                    conn_str=connection_string,
+                    max_block_size=self.max_blocksize,
+                    max_single_put_size=self.max_single_put_size,
+                )
+            elif client_name is not None:
+                account_url = f"https://{client_name}.blob.core.windows.net"
+                cred = credential_from_client_id or credential or account_key
+                if not cred and sas_token is not None:
+                    if not sas_token.startswith("?"):
+                        sas_token = f"?{sas_token}"
+                    account_url = account_url + sas_token
+                self._service_client = BlobServiceClient(
+                    account_url=account_url,
+                    credential=cred,
+                    max_block_size=self.max_blocksize,
+                    max_single_put_size=self.max_single_put_size,
+                )
+            else:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Must provide either a connection_string or account_name with credentials"
+                )
+        except Exception as e:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"unable to connect to account for {e}"
+            )
 
     def _convert_key_to_remote_path(self, key):
         key = key.strip("/")
@@ -82,7 +164,15 @@ class AzureBlobStore(DataStore):
 
     def upload(self, key, src_path):
         remote_path = self._convert_key_to_remote_path(key)
-        self.filesystem.put_file(src_path, remote_path, overwrite=True)
+        container, remote_path = remote_path.split("/", 1)
+        container_client = self.service_client.get_container_client(container=container)
+        with open(file=src_path, mode="rb") as data:
+            container_client.upload_blob(
+                name=remote_path,
+                data=data,
+                overwrite=True,
+                max_concurrency=self.max_concurrency,
+            )
 
     def get(self, key, size=None, offset=0):
         remote_path = self._convert_key_to_remote_path(key)
@@ -135,7 +225,7 @@ class AzureBlobStore(DataStore):
 
     def get_spark_options(self):
         res = {}
-        st = self.get_storage_options()
+        st = self.storage_options()
         service = "blob"
         primary_url = None
         if st.get("connection_string"):

mlrun/feature_store/common.py

@@ -37,17 +37,12 @@ def parse_feature_string(feature):
         raise mlrun.errors.MLRunInvalidArgumentError(
             f"feature {feature} must be {expected_message}"
         )
-    splitted = feature.split(feature_separator)
-    if len(splitted) > 2:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"feature {feature} must be {expected_message}, cannot have more than one '.'"
-        )
-    feature_set = splitted[0]
-    feature_name = splitted[1]
-    splitted = feature_name.split(" as ")
-    if len(splitted) > 1:
-        return feature_set.strip(), splitted[0].strip(), splitted[1].strip()
-    return feature_set.strip(), feature_name.strip(), None
+    feature_set, feature_name = feature.rsplit(feature_separator, 1)
+    feature_set = feature_set.strip()
+    split_result = feature_name.split(" as ", 1)
+    feature_name = split_result[0].strip()
+    alias = split_result[1].strip() if len(split_result) > 1 else None
+    return feature_set, feature_name, alias
 
 
 def parse_project_name_from_feature_string(feature):

mlrun/model.py

@@ -1789,6 +1789,11 @@ class RunObject(RunTemplate):
 
         return state
 
+    def abort(self):
+        """abort the run"""
+        db = mlrun.get_run_db()
+        db.abort_run(self.metadata.uid, self.metadata.project)
+
     @staticmethod
     def create_uri(project: str, uid: str, iteration: Union[int, str], tag: str = ""):
         if tag:

mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import http
 import json
 import typing
 from dataclasses import dataclass
@@ -417,11 +417,14 @@ class KVStoreBase(StoreBase):
             )
             return response.output.item[mm_schemas.SchedulingKeys.LAST_ANALYZED]
         except v3io.dataplane.response.HttpResponseError as err:
-            logger.debug("Error while getting last analyzed time", err=err)
-            raise mlrun.errors.MLRunNotFoundError(
-                f"No last analyzed value has been found for {application_name} "
-                f"that processes model endpoint {endpoint_id}",
-            )
+            if err.status_code == http.HTTPStatus.NOT_FOUND:
+                logger.debug("Last analyzed time not found", err=err)
+                raise mlrun.errors.MLRunNotFoundError(
+                    f"No last analyzed value has been found for {application_name} "
+                    f"that processes model endpoint {endpoint_id}",
+                )
+            logger.error("Error while getting last analyzed time", err=err)
+            raise err
 
     def update_last_analyzed(
         self, endpoint_id: str, application_name: str, last_analyzed: int

mlrun/model_monitoring/db/tsdb/base.py

@@ -15,6 +15,7 @@
 import typing
 from abc import ABC, abstractmethod
 from datetime import datetime
+from typing import Union
 
 import pandas as pd
 import pydantic
@@ -47,7 +48,7 @@ class TSDBConnector(ABC):
         self.project = project
 
     @abstractmethod
-    def apply_monitoring_stream_steps(self, graph):
+    def apply_monitoring_stream_steps(self, graph) -> None:
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
         different key metric dictionaries. This data is being used by the monitoring dashboards in
@@ -59,6 +60,14 @@ class TSDBConnector(ABC):
         """
         pass
 
+    @abstractmethod
+    def handle_model_error(self, graph, **kwargs) -> None:
+        """
+        Adds a branch to the stream pod graph to handle events that
+        arrive with errors from the model server and saves them to the error TSDB table.
+        The first step that generates by this method should come after `ForwardError` step.
+        """
+
     @abstractmethod
     def write_application_event(
         self,
@@ -181,6 +190,117 @@ class TSDBConnector(ABC):
         :return:                   Metric values object or no data object.
         """
 
+    @abstractmethod
+    def get_last_request(
+        self,
+        endpoint_ids: Union[str, list[str]],
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        """
+        Fetches data from the predictions TSDB table and returns the most recent request
+        timestamp for each specified endpoint.
+
+        :param endpoint_ids:    A list of model endpoint identifiers.
+        :param start:           The start time for the query.
+        :param end:             The end time for the query.
+
+        :return: A pd.DataFrame containing the columns [endpoint_id, last_request, last_latency].
+        If an endpoint has not been invoked within the specified time range, it will not appear in the result.
+        """
+
+    @abstractmethod
+    def get_drift_status(
+        self,
+        endpoint_ids: Union[str, list[str]],
+        start: Union[datetime, str] = "now-24h",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        """
+        Fetches data from the app-results TSDB table and returns the highest status among all
+        the result in the provided time range, which by default is the last 24 hours, for each specified endpoint.
+
+        :param endpoint_ids:    A list of model endpoint identifiers.
+        :param start:           The start time for the query.
+        :param end:             The end time for the query.
+
+        :return: A pd.DataFrame containing the columns [result_status, endpoint_id].
+        If an endpoint has not been monitored within the specified time range (last 24 hours),
+        it will not appear in the result.
+        """
+
+    @abstractmethod
+    def get_metrics_metadata(
+        self,
+        endpoint_id: str,
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        """
+        Fetches distinct metrics metadata from the metrics TSDB table for a specified model endpoint.
+
+        :param endpoint_id:        The model endpoint identifier.
+        :param start:              The start time of the query.
+        :param end:                The end time of the query.
+
+        :return: A pd.DataFrame containing all distinct metrics for the specified endpoint within the given time range.
+        Containing the columns [application_name, metric_name, endpoint_id]
+        """
+
+    @abstractmethod
+    def get_results_metadata(
+        self,
+        endpoint_id: str,
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        """
+        Fetches distinct results metadata from the app-results TSDB table for a specified model endpoint.
+
+        :param endpoint_id:        The model endpoint identifier.
+        :param start:              The start time of the query.
+        :param end:                The end time of the query.
+
+        :return: A pd.DataFrame containing all distinct results for the specified endpoint within the given time range.
+        Containing the columns [application_name, result_name, result_kind, endpoint_id]
+        """
+
+    @abstractmethod
+    def get_error_count(
+        self,
+        endpoint_ids: Union[str, list[str]],
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        """
+        Fetches data from the error TSDB table and returns the error count for each specified endpoint.
+
+        :param endpoint_ids:    A list of model endpoint identifiers.
+        :param start:           The start time for the query.
+        :param end:             The end time for the query.
+
+        :return: A pd.DataFrame containing the columns [error_count, endpoint_id].
+        If an endpoint have not raised error within the specified time range, it will not appear in the result.
+        """
+
+    @abstractmethod
+    def get_avg_latency(
+        self,
+        endpoint_ids: Union[str, list[str]],
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        """
+        Fetches data from the predictions TSDB table and returns the average latency for each specified endpoint
+
+        :param endpoint_ids:    A list of model endpoint identifiers.
+        :param start:           The start time for the query.
+        :param end:             The end time for the query.
+
+        :return: A pd.DataFrame containing the columns [avg_latency, endpoint_id].
+        If an endpoint has not been invoked within the specified time range, it will not appear in the result.
+        """
+
     @staticmethod
     def df_to_metrics_values(
         *,

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -14,6 +14,7 @@
 
 import typing
 from datetime import datetime
+from typing import Union
 
 import pandas as pd
 import taosws
@@ -156,6 +157,9 @@ class TDEngineConnector(TSDBConnector):
             after="ProcessBeforeTDEngine",
         )
 
+    def handle_model_error(self, graph, **kwargs) -> None:
+        pass
+
     def delete_tsdb_resources(self):
         """
         Delete all project resources in the TSDB connector, such as model endpoints data and drift results.
@@ -246,11 +250,9 @@ class TDEngineConnector(TSDBConnector):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Failed to query table {table} in database {self.database}, {str(e)}"
             )
-        columns = []
-        for column in query_result.fields:
-            columns.append(column.name())
 
-        return pd.DataFrame(query_result, columns=columns)
+        df_columns = [field.name() for field in query_result.fields]
+        return pd.DataFrame(query_result, columns=df_columns)
 
     def read_metrics_data(
         self,
@@ -274,13 +276,22 @@ class TDEngineConnector(TSDBConnector):
             ],
         ],
     ]:
+        timestamp_column = mm_schemas.WriterEvent.END_INFER_TIME
+        columns = [timestamp_column, mm_schemas.WriterEvent.APPLICATION_NAME]
         if type == "metrics":
             table = mm_schemas.TDEngineSuperTables.METRICS
             name = mm_schemas.MetricData.METRIC_NAME
+            columns += [name, mm_schemas.MetricData.METRIC_VALUE]
             df_handler = self.df_to_metrics_values
         elif type == "results":
             table = mm_schemas.TDEngineSuperTables.APP_RESULTS
             name = mm_schemas.ResultData.RESULT_NAME
+            columns += [
+                name,
+                mm_schemas.ResultData.RESULT_VALUE,
+                mm_schemas.ResultData.RESULT_STATUS,
+                mm_schemas.ResultData.RESULT_KIND,
+            ]
             df_handler = self.df_to_results_values
         else:
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -300,7 +311,8 @@ class TDEngineConnector(TSDBConnector):
             start=start,
             end=end,
             filter_query=filter_query,
-            timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
+            timestamp_column=timestamp_column,
+            columns=columns,
         )
 
         df[mm_schemas.WriterEvent.END_INFER_TIME] = pd.to_datetime(
@@ -377,6 +389,54 @@ class TDEngineConnector(TSDBConnector):
             ),  # pyright: ignore[reportArgumentType]
         )
 
+    def get_last_request(
+        self,
+        endpoint_ids: Union[str, list[str]],
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        pass
+
+    def get_drift_status(
+        self,
+        endpoint_ids: Union[str, list[str]],
+        start: Union[datetime, str] = "now-24h",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        pass
+
+    def get_metrics_metadata(
+        self,
+        endpoint_id: str,
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        pass
+
+    def get_results_metadata(
+        self,
+        endpoint_id: str,
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        pass
+
+    def get_error_count(
+        self,
+        endpoint_ids: Union[str, list[str]],
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        pass
+
+    def get_avg_latency(
+        self,
+        endpoint_ids: Union[str, list[str]],
+        start: Union[datetime, str] = "0",
+        end: Union[datetime, str] = "now",
+    ) -> pd.DataFrame:
+        pass
+
     # Note: this function serves as a reference for checking the TSDB for the existence of a metric.
     #
     # def read_prediction_metric_for_endpoint_if_exists(

mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from datetime import datetime
 from typing import Any
 
 import mlrun.feature_store.steps
@@ -20,6 +20,7 @@ from mlrun.common.schemas.model_monitoring import (
     EventKeyMetrics,
     EventLiveStats,
 )
+from mlrun.utils import logger
 
 
 def _normalize_dict_for_v3io_frames(event: dict[str, Any]) -> dict[str, Any]:
@@ -134,3 +135,24 @@ class FilterAndUnpackKeys(mlrun.feature_store.steps.MapClass):
             else:
                 unpacked[key] = new_event[key]
         return unpacked if unpacked else None
+
+
+class ErrorExtractor(mlrun.feature_store.steps.MapClass):
+    def __init__(self, **kwargs):
+        """
+        Prepare the event for insertion into the errors TSDB table.
+        """
+        super().__init__(**kwargs)
+
+    def do(self, event):
+        error = event.get("error")
+        timestamp = datetime.fromisoformat(event.get("when"))
+        endpoint_id = event[EventFieldType.ENDPOINT_ID]
+        event = {
+            EventFieldType.MODEL_ERROR: str(error),
+            EventFieldType.ENDPOINT_ID: endpoint_id,
+            EventFieldType.TIMESTAMP: timestamp,
+            EventFieldType.ERROR_COUNT: 1.0,
+        }
+        logger.info("Write error to errors TSDB table", event=event)
+        return event