mlrun 1.7.0rc36__py3-none-any.whl → 1.7.0rc38__py3-none-any.whl

mlrun/alerts/alert.py

@@ -28,6 +28,8 @@ class AlertConfig(ModelObj):
         "severity",
         "reset_policy",
         "state",
+        "count",
+        "created",
     ]
     _fields_to_serialize = ModelObj._fields_to_serialize + [
         "entities",
@@ -54,6 +56,68 @@ class AlertConfig(ModelObj):
         created: str = None,
         count: int = None,
     ):
+        """Alert config object
+
+        Example::
+
+            # create an alert on endpoint_id, which will be triggered to slack if there is a "data_drift_detected" event
+            # 3 times in the next hour.
+            from mlrun.alerts import AlertConfig
+            import mlrun.common.schemas.alert as alert_objects
+
+            entity_kind = alert_objects.EventEntityKind.MODEL_ENDPOINT_RESULT
+            entity_id = get_default_result_instance_fqn(endpoint_id)
+            event_name = alert_objects.EventKind.DATA_DRIFT_DETECTED
+            notification = mlrun.model.Notification(
+                kind="slack",
+                name="slack_notification",
+                message="drift was detected",
+                severity="warning",
+                when=["now"],
+                condition="failed",
+                secret_params={
+                    "webhook": "https://hooks.slack.com/",
+                },
+            ).to_dict()
+
+            alert_data = AlertConfig(
+                project="my-project",
+                name="drift-alert",
+                summary="a drift was detected",
+                severity=alert_objects.AlertSeverity.LOW,
+                entities=alert_objects.EventEntities(
+                    kind=entity_kind, project="my-project", ids=[entity_id]
+                ),
+                trigger=alert_objects.AlertTrigger(events=[event_name]),
+                criteria=alert_objects.AlertCriteria(count=3, period="1h"),
+                notifications=[alert_objects.AlertNotification(notification=notification)],
+            )
+            project.store_alert_config(alert_data)
+
+        :param project:        name of the project to associate the alert with
+        :param name:           name of the alert
+        :param template:       optional parameter that allows to create an alert based on a predefined template.
+                               you can pass either an AlertTemplate object or a string (the template name).
+                               if a template is used, many fields of the alert will be auto-generated based on the
+                               template. however, you still need to provide the following fields:
+                               `name`, `project`, `entity`, `notifications`
+        :param description:    description of the alert
+        :param summary:        summary of the alert, will be sent in the generated notifications
+        :param severity:       severity of the alert
+        :param trigger:        the events that will trigger this alert, may be a simple trigger based on events or
+                               complex trigger which is based on a prometheus alert
+        :param criteria:       when the alert will be triggered based on the specified number of events within the
+                               defined time period.
+        :param reset_policy:   when to clear the alert. May be "manual" for manual reset of the alert, or
+                               "auto" if the criteria contains a time period
+        :param notifications:  list of notifications to invoke once the alert is triggered
+        :param entities:       entities that the event relates to. The entity object will contain fields that uniquely
+                               identify a given entity in the system
+        :param id:             internal id of the alert (user should not supply it)
+        :param state:          state of the alert, may be active/inactive (user should not supply it)
+        :param created:        when the alert is created (user should not supply it)
+        :param count:          internal counter of the alert (user should not supply it)
+        """
         self.project = project
         self.name = name
         self.description = description

mlrun/common/schemas/alert.py

@@ -149,7 +149,7 @@ class AlertConfig(pydantic.BaseModel):
     entities: EventEntities
     trigger: AlertTrigger
     criteria: Optional[AlertCriteria]
-    reset_policy: ResetPolicy = ResetPolicy.MANUAL
+    reset_policy: ResetPolicy = ResetPolicy.AUTO
     notifications: pydantic.conlist(AlertNotification, min_items=1)
     state: AlertActiveState = AlertActiveState.INACTIVE
     count: Optional[int] = 0
@@ -185,7 +185,7 @@ class AlertTemplate(
     severity: AlertSeverity
     trigger: AlertTrigger
     criteria: Optional[AlertCriteria]
-    reset_policy: ResetPolicy = ResetPolicy.MANUAL
+    reset_policy: ResetPolicy = ResetPolicy.AUTO
 
     # This is slightly different than __eq__ as it doesn't compare everything
     def templates_differ(self, other):

mlrun/common/schemas/model_monitoring/constants.py

@@ -53,9 +53,11 @@ class EventFieldType:
     PREDICTIONS = "predictions"
     NAMED_PREDICTIONS = "named_predictions"
     ERROR_COUNT = "error_count"
+    MODEL_ERROR = "model_error"
     ENTITIES = "entities"
     FIRST_REQUEST = "first_request"
     LAST_REQUEST = "last_request"
+    LAST_REQUEST_TIMESTAMP = "last_request_timestamp"
     METRIC = "metric"
     METRICS = "metrics"
     BATCH_INTERVALS_DICT = "batch_intervals_dict"
@@ -217,6 +219,7 @@ class FileTargetKind:
     APP_METRICS = "app_metrics"
     MONITORING_SCHEDULES = "monitoring_schedules"
     MONITORING_APPLICATION = "monitoring_application"
+    ERRORS = "errors"
 
 
 class ModelMonitoringMode(str, Enum):
@@ -240,6 +243,7 @@ class V3IOTSDBTables(MonitoringStrEnum):
     APP_RESULTS = "app-results"
     METRICS = "metrics"
     EVENTS = "events"
+    ERRORS = "errors"
 
 
 class TDEngineSuperTables(MonitoringStrEnum):

mlrun/common/schemas/notification.py

@@ -50,15 +50,34 @@ class NotificationLimits(enum.Enum):
 
 
 class Notification(pydantic.BaseModel):
+    """
+    Notification object schema
+    :param kind: notification implementation kind - slack, webhook, etc.
+    :param name: for logging and identification
+    :param message: message content in the notification
+    :param severity: severity to display in the notification
+    :param when: list of statuses to trigger the notification: 'running', 'completed', 'error'
+    :param condition: optional condition to trigger the notification, a jinja2 expression that can use run data
+                      to evaluate if the notification should be sent in addition to the 'when' statuses.
+                      e.g.: '{{ run["status"]["results"]["accuracy"] < 0.9}}'
+    :param params: Implementation specific parameters for the notification implementation (e.g. slack webhook url,
+                   git repository details, etc.)
+    :param secret_params: secret parameters for the notification implementation, same as params but will be stored
+                          in a k8s secret and passed as a secret reference to the implementation.
+    :param status: notification status - pending, sent, error
+    :param sent_time: time the notification was sent
+    :param reason: failure reason if the notification failed to send
+    """
+
     kind: NotificationKind
     name: str
-    message: str
-    severity: NotificationSeverity
-    when: list[str]
-    condition: str = None
-    params: dict[str, typing.Any] = None
-    status: NotificationStatus = None
-    sent_time: typing.Union[str, datetime.datetime] = None
+    message: typing.Optional[str] = None
+    severity: typing.Optional[NotificationSeverity] = None
+    when: typing.Optional[list[str]] = None
+    condition: typing.Optional[str] = None
+    params: typing.Optional[dict[str, typing.Any]] = None
+    status: typing.Optional[NotificationStatus] = None
+    sent_time: typing.Optional[typing.Union[str, datetime.datetime]] = None
     secret_params: typing.Optional[dict[str, typing.Any]] = None
     reason: typing.Optional[str] = None
 

mlrun/datastore/azure_blob.py

@@ -16,6 +16,7 @@ import time
 from pathlib import Path
 from urllib.parse import urlparse
 
+from azure.storage.blob import BlobServiceClient
 from azure.storage.blob._shared.base_client import parse_connection_str
 from fsspec.registry import get_filesystem_class
 
@@ -29,47 +30,128 @@ from .base import DataStore, FileStats, makeDatastoreSchemaSanitizer
 
 class AzureBlobStore(DataStore):
     using_bucket = True
+    max_concurrency = 100
+    max_blocksize = 1024 * 1024 * 4
+    max_single_put_size = (
+        1024 * 1024 * 8
+    )  # for service_client property only, does not affect filesystem
 
     def __init__(self, parent, schema, name, endpoint="", secrets: dict = None):
         super().__init__(parent, name, schema, endpoint, secrets=secrets)
+        self._service_client = None
+        self._storage_options = None
+
+    @property
+    def storage_options(self):
+        if not self._storage_options:
+            res = dict(
+                account_name=self._get_secret_or_env("account_name")
+                or self._get_secret_or_env("AZURE_STORAGE_ACCOUNT_NAME"),
+                account_key=self._get_secret_or_env("account_key")
+                or self._get_secret_or_env("AZURE_STORAGE_ACCOUNT_KEY"),
+                connection_string=self._get_secret_or_env("connection_string")
+                or self._get_secret_or_env("AZURE_STORAGE_CONNECTION_STRING"),
+                tenant_id=self._get_secret_or_env("tenant_id")
+                or self._get_secret_or_env("AZURE_STORAGE_TENANT_ID"),
+                client_id=self._get_secret_or_env("client_id")
+                or self._get_secret_or_env("AZURE_STORAGE_CLIENT_ID"),
+                client_secret=self._get_secret_or_env("client_secret")
+                or self._get_secret_or_env("AZURE_STORAGE_CLIENT_SECRET"),
+                sas_token=self._get_secret_or_env("sas_token")
+                or self._get_secret_or_env("AZURE_STORAGE_SAS_TOKEN"),
+                credential=self._get_secret_or_env("credential"),
+            )
+            self._storage_options = self._sanitize_storage_options(res)
+        return self._storage_options
 
     @property
     def filesystem(self):
         """return fsspec file system object, if supported"""
-        if self._filesystem:
-            return self._filesystem
         try:
             import adlfs  # noqa
         except ImportError as exc:
             raise ImportError("Azure adlfs not installed") from exc
-        # in order to support az and wasbs kinds.
-        filesystem_class = get_filesystem_class(protocol=self.kind)
-        self._filesystem = makeDatastoreSchemaSanitizer(
-            filesystem_class,
-            using_bucket=self.using_bucket,
-            **self.get_storage_options(),
-        )
+
+        if not self._filesystem:
+            # in order to support az and wasbs kinds
+            filesystem_class = get_filesystem_class(protocol=self.kind)
+            self._filesystem = makeDatastoreSchemaSanitizer(
+                filesystem_class,
+                using_bucket=self.using_bucket,
+                blocksize=self.max_blocksize,
+                **self.storage_options,
+            )
         return self._filesystem
 
-    def get_storage_options(self):
-        res = dict(
-            account_name=self._get_secret_or_env("account_name")
-            or self._get_secret_or_env("AZURE_STORAGE_ACCOUNT_NAME"),
-            account_key=self._get_secret_or_env("account_key")
-            or self._get_secret_or_env("AZURE_STORAGE_KEY"),
-            connection_string=self._get_secret_or_env("connection_string")
-            or self._get_secret_or_env("AZURE_STORAGE_CONNECTION_STRING"),
-            tenant_id=self._get_secret_or_env("tenant_id")
-            or self._get_secret_or_env("AZURE_STORAGE_TENANT_ID"),
-            client_id=self._get_secret_or_env("client_id")
-            or self._get_secret_or_env("AZURE_STORAGE_CLIENT_ID"),
-            client_secret=self._get_secret_or_env("client_secret")
-            or self._get_secret_or_env("AZURE_STORAGE_CLIENT_SECRET"),
-            sas_token=self._get_secret_or_env("sas_token")
-            or self._get_secret_or_env("AZURE_STORAGE_SAS_TOKEN"),
-            credential=self._get_secret_or_env("credential"),
-        )
-        return self._sanitize_storage_options(res)
+    @property
+    def service_client(self):
+        try:
+            import azure  # noqa
+        except ImportError as exc:
+            raise ImportError("Azure not installed") from exc
+
+        if not self._service_client:
+            self._do_connect()
+        return self._service_client
+
+    def _do_connect(self):
+        """
+
+        Creates a client for azure.
+        Raises MLRunInvalidArgumentError if none of the connection details are available
+        based on do_connect in AzureBlobFileSystem:
+        https://github.com/fsspec/adlfs/blob/2023.9.0/adlfs/spec.py#L422
+        """
+        from azure.identity import ClientSecretCredential
+
+        storage_options = self.storage_options
+        connection_string = storage_options.get("connection_string")
+        client_name = storage_options.get("account_name")
+        account_key = storage_options.get("account_key")
+        sas_token = storage_options.get("sas_token")
+        client_id = storage_options.get("client_id")
+        credential = storage_options.get("credential")
+
+        credential_from_client_id = None
+        if (
+            credential is None
+            and account_key is None
+            and sas_token is None
+            and client_id is not None
+        ):
+            credential_from_client_id = ClientSecretCredential(
+                tenant_id=storage_options.get("tenant_id"),
+                client_id=client_id,
+                client_secret=storage_options.get("client_secret"),
+            )
+        try:
+            if connection_string is not None:
+                self._service_client = BlobServiceClient.from_connection_string(
+                    conn_str=connection_string,
+                    max_block_size=self.max_blocksize,
+                    max_single_put_size=self.max_single_put_size,
+                )
+            elif client_name is not None:
+                account_url = f"https://{client_name}.blob.core.windows.net"
+                cred = credential_from_client_id or credential or account_key
+                if not cred and sas_token is not None:
+                    if not sas_token.startswith("?"):
+                        sas_token = f"?{sas_token}"
+                    account_url = account_url + sas_token
+                self._service_client = BlobServiceClient(
+                    account_url=account_url,
+                    credential=cred,
+                    max_block_size=self.max_blocksize,
+                    max_single_put_size=self.max_single_put_size,
+                )
+            else:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Must provide either a connection_string or account_name with credentials"
+                )
+        except Exception as e:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"unable to connect to account for {e}"
+            )
 
     def _convert_key_to_remote_path(self, key):
         key = key.strip("/")
@@ -82,7 +164,15 @@ class AzureBlobStore(DataStore):
 
     def upload(self, key, src_path):
         remote_path = self._convert_key_to_remote_path(key)
-        self.filesystem.put_file(src_path, remote_path, overwrite=True)
+        container, remote_path = remote_path.split("/", 1)
+        container_client = self.service_client.get_container_client(container=container)
+        with open(file=src_path, mode="rb") as data:
+            container_client.upload_blob(
+                name=remote_path,
+                data=data,
+                overwrite=True,
+                max_concurrency=self.max_concurrency,
+            )
 
     def get(self, key, size=None, offset=0):
         remote_path = self._convert_key_to_remote_path(key)
@@ -135,7 +225,7 @@ class AzureBlobStore(DataStore):
 
     def get_spark_options(self):
         res = {}
-        st = self.get_storage_options()
+        st = self.storage_options()
         service = "blob"
         primary_url = None
         if st.get("connection_string"):

mlrun/datastore/s3.py

@@ -15,6 +15,7 @@
 import time
 
 import boto3
+from boto3.s3.transfer import TransferConfig
 from fsspec.registry import get_filesystem_class
 
 import mlrun.errors
@@ -40,6 +41,12 @@ class S3Store(DataStore):
         profile_name = self._get_secret_or_env("AWS_PROFILE")
         assume_role_arn = self._get_secret_or_env("MLRUN_AWS_ROLE_ARN")
 
+        self.config = TransferConfig(
+            multipart_threshold=1024 * 1024 * 25,
+            max_concurrency=10,
+            multipart_chunksize=1024 * 1024 * 25,
+        )
+
         # If user asks to assume a role, this needs to go through the STS client and retrieve temporary creds
         if assume_role_arn:
             client = boto3.client(
@@ -166,7 +173,7 @@ class S3Store(DataStore):
 
     def upload(self, key, src_path):
         bucket, key = self.get_bucket_and_key(key)
-        self.s3.Object(bucket, key).put(Body=open(src_path, "rb"))
+        self.s3.Bucket(bucket).upload_file(src_path, key, Config=self.config)
 
     def get(self, key, size=None, offset=0):
         bucket, key = self.get_bucket_and_key(key)

mlrun/feature_store/common.py

@@ -37,17 +37,12 @@ def parse_feature_string(feature):
         raise mlrun.errors.MLRunInvalidArgumentError(
             f"feature {feature} must be {expected_message}"
         )
-    splitted = feature.split(feature_separator)
-    if len(splitted) > 2:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"feature {feature} must be {expected_message}, cannot have more than one '.'"
-        )
-    feature_set = splitted[0]
-    feature_name = splitted[1]
-    splitted = feature_name.split(" as ")
-    if len(splitted) > 1:
-        return feature_set.strip(), splitted[0].strip(), splitted[1].strip()
-    return feature_set.strip(), feature_name.strip(), None
+    feature_set, feature_name = feature.rsplit(feature_separator, 1)
+    feature_set = feature_set.strip()
+    split_result = feature_name.split(" as ", 1)
+    feature_name = split_result[0].strip()
+    alias = split_result[1].strip() if len(split_result) > 1 else None
+    return feature_set, feature_name, alias
 
 
 def parse_project_name_from_feature_string(feature):

mlrun/model.py

@@ -1789,6 +1789,11 @@ class RunObject(RunTemplate):
 
         return state
 
+    def abort(self):
+        """abort the run"""
+        db = mlrun.get_run_db()
+        db.abort_run(self.metadata.uid, self.metadata.project)
+
     @staticmethod
     def create_uri(project: str, uid: str, iteration: Union[int, str], tag: str = ""):
         if tag:

mlrun/model_monitoring/api.py

@@ -622,5 +622,5 @@ def _create_model_monitoring_function_base(
         name="PushToMonitoringWriter",
         project=project,
         writer_application_name=mm_constants.MonitoringFunctionNames.WRITER,
-    ).respond()
+    )
     return func_obj

mlrun/model_monitoring/applications/_application_steps.py

@@ -20,6 +20,7 @@ import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constant
 import mlrun.datastore
 import mlrun.serving
+import mlrun.utils.helpers
 import mlrun.utils.v3io_clients
 from mlrun.model_monitoring.helpers import get_stream_path
 from mlrun.serving.utils import StepToDict
@@ -34,8 +35,8 @@ class _PushToMonitoringWriter(StepToDict):
 
     def __init__(
         self,
-        project: Optional[str] = None,
-        writer_application_name: Optional[str] = None,
+        project: str,
+        writer_application_name: str,
         stream_uri: Optional[str] = None,
         name: Optional[str] = None,
     ):
@@ -109,6 +110,7 @@ class _PushToMonitoringWriter(StepToDict):
                 f"Pushing data = {writer_event} \n to stream = {self.stream_uri}"
             )
             self.output_stream.push([writer_event])
+            logger.info(f"Pushed data to {self.stream_uri} successfully")
 
     def _lazy_init(self):
         if self.output_stream is None:
@@ -150,12 +152,15 @@ class _PrepareMonitoringEvent(StepToDict):
 
     @staticmethod
     def _create_mlrun_context(app_name: str):
+        artifact_path = mlrun.utils.helpers.template_artifact_path(
+            mlrun.mlconf.artifact_path, mlrun.mlconf.default_project
+        )
         context = mlrun.get_or_create_ctx(
             f"{app_name}-logger",
             spec={
-                "metadata": {"labels": {"kind": mlrun.runtimes.RuntimeKinds.serving}}
+                "metadata": {"labels": {"kind": mlrun.runtimes.RuntimeKinds.serving}},
+                "spec": {mlrun.utils.helpers.RunKeys.output_path: artifact_path},
             },
-            upload_artifacts=True,
         )
         context.__class__ = MonitoringApplicationContext
         return context

mlrun/model_monitoring/db/stores/sqldb/models/mysql.py

@@ -18,6 +18,7 @@ from sqlalchemy.ext.declarative import declarative_base, declared_attr
 
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
+    ResultData,
     WriterEvent,
 )
 
@@ -32,6 +33,13 @@ Base = declarative_base()
 
 
 class ModelEndpointsTable(Base, ModelEndpointsBaseTable):
+    feature_stats = Column(
+        EventFieldType.FEATURE_STATS, sqlalchemy.dialects.mysql.MEDIUMTEXT
+    )
+    current_stats = Column(
+        EventFieldType.CURRENT_STATS, sqlalchemy.dialects.mysql.MEDIUMTEXT
+    )
+    metrics = Column(EventFieldType.METRICS, sqlalchemy.dialects.mysql.MEDIUMTEXT)
     first_request = Column(
         EventFieldType.FIRST_REQUEST,
         # TODO: migrate to DATETIME, see ML-6921
@@ -72,7 +80,12 @@ class _ApplicationResultOrMetric:
 class ApplicationResultTable(
     Base, _ApplicationResultOrMetric, ApplicationResultBaseTable
 ):
-    pass
+    result_extra_data = Column(
+        ResultData.RESULT_EXTRA_DATA, sqlalchemy.dialects.mysql.MEDIUMTEXT
+    )
+    current_stats = Column(
+        ResultData.CURRENT_STATS, sqlalchemy.dialects.mysql.MEDIUMTEXT
+    )
 
 
 class ApplicationMetricsTable(

mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import http
 import json
 import typing
 from dataclasses import dataclass
@@ -350,7 +350,7 @@ class KVStoreBase(StoreBase):
             table_path = self._get_results_table_path(endpoint_id)
             key = event.pop(mm_schemas.WriterEvent.APPLICATION_NAME)
             metric_name = event.pop(mm_schemas.ResultData.RESULT_NAME)
-            attributes = {metric_name: json.dumps(event)}
+            attributes = {metric_name: self._encode_field(json.dumps(event))}
         else:
             raise ValueError(f"Invalid {kind = }")
 
@@ -417,11 +417,14 @@ class KVStoreBase(StoreBase):
             )
             return response.output.item[mm_schemas.SchedulingKeys.LAST_ANALYZED]
         except v3io.dataplane.response.HttpResponseError as err:
-            logger.debug("Error while getting last analyzed time", err=err)
-            raise mlrun.errors.MLRunNotFoundError(
-                f"No last analyzed value has been found for {application_name} "
-                f"that processes model endpoint {endpoint_id}",
-            )
+            if err.status_code == http.HTTPStatus.NOT_FOUND:
+                logger.debug("Last analyzed time not found", err=err)
+                raise mlrun.errors.MLRunNotFoundError(
+                    f"No last analyzed value has been found for {application_name} "
+                    f"that processes model endpoint {endpoint_id}",
+                )
+            logger.error("Error while getting last analyzed time", err=err)
+            raise err
 
     def update_last_analyzed(
         self, endpoint_id: str, application_name: str, last_analyzed: int