mlrun 1.7.0rc15__py3-none-any.whl → 1.7.0rc16__py3-none-any.whl

mlrun/__init__.py

@@ -22,11 +22,16 @@ __all__ = [
     "handler",
     "ArtifactType",
     "get_secret_or_env",
+    "mount_v3io",
+    "v3io_cred",
+    "auto_mount",
+    "VolumeMount",
 ]
 
 from os import environ, path
 
 import dotenv
+import mlrun_pipelines
 
 from .config import config as mlconf
 from .datastore import DataItem, store_manager
@@ -35,7 +40,6 @@ from .errors import MLRunInvalidArgumentError, MLRunNotFoundError
 from .execution import MLClientCtx
 from .model import RunObject, RunTemplate, new_task
 from .package import ArtifactType, DefaultPackager, Packager, handler
-from .platforms import VolumeMount, auto_mount, mount_v3io, v3io_cred
 from .projects import (
     ProjectMetadata,
     build_function,
@@ -65,6 +69,11 @@ from .utils.version import Version
 
 __version__ = Version().get()["version"]
 
+VolumeMount = mlrun_pipelines.common.mounts.VolumeMount
+mount_v3io = mlrun_pipelines.mounts.mount_v3io
+v3io_cred = mlrun_pipelines.mounts.v3io_cred
+auto_mount = mlrun_pipelines.mounts.auto_mount
+
 
 def get_version():
     """get current mlrun version"""

mlrun/__main__.py

@@ -27,6 +27,7 @@ import click
 import dotenv
 import pandas as pd
 import yaml
+from mlrun_pipelines.mounts import auto_mount as auto_mount_modifier
 from tabulate import tabulate
 
 import mlrun
@@ -37,7 +38,6 @@ from .config import config as mlconf
 from .db import get_run_db
 from .errors import err_to_str
 from .model import RunTemplate
-from .platforms import auto_mount as auto_mount_modifier
 from .projects import load_project
 from .run import (
     get_object,
@@ -466,6 +466,17 @@ def run(
     is_flag=True,
     help="ensure the project exists, if not, create project",
 )
+@click.option(
+    "--state-file-path", default="/tmp/state", help="path to file with state data"
+)
+@click.option(
+    "--image-file-path", default="/tmp/image", help="path to file with image data"
+)
+@click.option(
+    "--full-image-file-path",
+    default="/tmp/fullimage",
+    help="path to file with full image data",
+)
 def build(
     func_url,
     name,
@@ -485,6 +496,9 @@ def build(
     skip,
     env_file,
     ensure_project,
+    state_file_path,
+    image_file_path,
+    full_image_file_path,
 ):
     """Build a container image from code and requirements."""
 
@@ -574,12 +588,12 @@ def build(
         state = func.status.state
         image = func.spec.image
         if kfp:
-            with open("/tmp/state", "w") as fp:
+            with open(state_file_path, "w") as fp:
                 fp.write(state or "none")
             full_image = func.full_image_path(image) or ""
-            with open("/tmp/image", "w") as fp:
+            with open(image_file_path, "w") as fp:
                 fp.write(image)
-            with open("/tmp/fullimage", "w") as fp:
+            with open(full_image_file_path, "w") as fp:
                 fp.write(full_image)
             print("Full image path = ", full_image)
 

mlrun/alerts/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .alert import AlertConfig

mlrun/alerts/alert.py

@@ -0,0 +1,141 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Union
+
+import mlrun
+import mlrun.common.schemas.alert as alert_constants
+from mlrun.common.schemas.notification import Notification
+from mlrun.model import ModelObj
+
+
+class AlertConfig(ModelObj):
+    _dict_fields = [
+        "project",
+        "name",
+        "description",
+        "summary",
+        "severity",
+        "criteria",
+        "reset_policy",
+        "state",
+    ]
+
+    def __init__(
+        self,
+        project: str = None,
+        name: str = None,
+        template: Union[alert_constants.AlertTemplate, str] = None,
+        description: str = None,
+        summary: str = None,
+        severity: alert_constants.AlertSeverity = None,
+        trigger: alert_constants.AlertTrigger = None,
+        criteria: alert_constants.AlertCriteria = None,
+        reset_policy: alert_constants.ResetPolicy = None,
+        notifications: list[Notification] = None,
+        entities: alert_constants.EventEntities = None,
+        id: int = None,
+        state: alert_constants.AlertActiveState = None,
+        created: str = None,
+        count: int = None,
+    ):
+        self.project = project
+        self.name = name
+        self.description = description
+        self.summary = summary
+        self.severity = severity
+        self.trigger = trigger
+        self.criteria = criteria
+        self.reset_policy = reset_policy
+        self.notifications = notifications or []
+        self.entities = entities
+        self.id = id
+        self.state = state
+        self.created = created
+        self.count = count
+
+        if template:
+            self._apply_template(template)
+
+    def validate_required_fields(self):
+        if not self.project or not self.name:
+            raise mlrun.errors.MLRunBadRequestError("Project and name must be provided")
+
+    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+        data = super().to_dict(self._dict_fields)
+
+        data["entities"] = (
+            self.entities.dict()
+            if not isinstance(self.entities, dict)
+            else self.entities
+        )
+        data["notifications"] = [
+            notification.dict() if not isinstance(notification, dict) else notification
+            for notification in self.notifications
+        ]
+        data["trigger"] = (
+            self.trigger.dict() if not isinstance(self.trigger, dict) else self.trigger
+        )
+        return data
+
+    @classmethod
+    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+        new_obj = super().from_dict(struct, fields=fields)
+
+        entity_data = struct.get("entities")
+        if entity_data:
+            entity_obj = alert_constants.EventEntities.parse_obj(entity_data)
+            new_obj.entities = entity_obj
+
+        notifications_data = struct.get("notifications")
+        if notifications_data:
+            notifications_objs = [
+                Notification.parse_obj(notification_data)
+                for notification_data in notifications_data
+            ]
+            new_obj.notifications = notifications_objs
+
+        trigger_data = struct.get("trigger")
+        if trigger_data:
+            trigger_obj = alert_constants.AlertTrigger.parse_obj(trigger_data)
+            new_obj.trigger = trigger_obj
+
+        return new_obj
+
+    def with_notifications(self, notifications: list[Notification]):
+        if not isinstance(notifications, list) or not all(
+            isinstance(item, Notification) for item in notifications
+        ):
+            raise ValueError("Notifications parameter must be a list of notifications")
+        for notification in notifications:
+            self.notifications.append(notification)
+        return self
+
+    def with_entities(self, entities: alert_constants.EventEntities):
+        if not isinstance(entities, alert_constants.EventEntities):
+            raise ValueError("Entities parameter must be of type: EventEntities")
+        self.entities = entities
+        return self
+
+    def _apply_template(self, template):
+        if isinstance(template, str):
+            db = mlrun.get_run_db()
+            template = db.get_alert_template(template)
+
+        # Extract parameters from the template and apply them to the AlertConfig object
+        self.description = template.description
+        self.severity = template.severity
+        self.criteria = template.criteria
+        self.trigger = template.trigger
+        self.reset_policy = template.reset_policy

mlrun/artifacts/__init__.py

@@ -17,7 +17,13 @@
 # Don't remove this, used by sphinx documentation
 __all__ = ["get_model", "update_model"]
 
-from .base import Artifact, ArtifactMetadata, ArtifactSpec, get_artifact_meta
+from .base import (
+    Artifact,
+    ArtifactMetadata,
+    ArtifactSpec,
+    DirArtifact,
+    get_artifact_meta,
+)
 from .dataset import DatasetArtifact, TableArtifact, update_dataset_meta
 from .manager import (
     ArtifactManager,

mlrun/artifacts/base.py

@@ -191,12 +191,30 @@ class Artifact(ModelObj):
         format=None,
         size=None,
         target_path=None,
-        # All params up until here are legacy params for compatibility with legacy artifacts.
         project=None,
+        src_path: str = None,
+        # All params up until here are legacy params for compatibility with legacy artifacts.
+        # TODO: remove them in 1.9.0.
         metadata: ArtifactMetadata = None,
         spec: ArtifactSpec = None,
-        src_path: str = None,
     ):
+        if (
+            key
+            or body
+            or viewer
+            or is_inline
+            or format
+            or size
+            or target_path
+            or project
+            or src_path
+        ):
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
+
         self._metadata = None
         self.metadata = metadata
         self._spec = None
@@ -698,11 +716,18 @@ class LinkArtifact(Artifact):
         link_iteration=None,
         link_key=None,
         link_tree=None,
-        # All params up until here are legacy params for compatibility with legacy artifacts.
         project=None,
+        # All params up until here are legacy params for compatibility with legacy artifacts.
+        # TODO: remove them in 1.9.0.
         metadata: ArtifactMetadata = None,
         spec: LinkArtifactSpec = None,
     ):
+        if key or target_path or link_iteration or link_key or link_tree or project:
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
         super().__init__(
             key, target_path=target_path, project=project, metadata=metadata, spec=spec
         )

mlrun/artifacts/dataset.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import os
 import pathlib
+import warnings
 from io import StringIO
 from typing import Optional
 
@@ -160,6 +161,13 @@ class DatasetArtifact(Artifact):
         label_column: str = None,
         **kwargs,
     ):
+        if key or format or target_path:
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
+
         format = (format or "").lower()
         super().__init__(key, None, format=format, target_path=target_path)
         if format and format not in self.SUPPORTED_FORMATS:

mlrun/artifacts/manager.py

@@ -16,6 +16,7 @@ import typing
 from os.path import exists, isdir
 from urllib.parse import urlparse
 
+import mlrun.common.schemas.artifact
 import mlrun.config
 from mlrun.utils.helpers import (
     get_local_file_schema,
@@ -343,6 +344,23 @@ class ArtifactManager:
                 project=project,
             )
 
+    def delete_artifact(
+        self,
+        item: Artifact,
+        deletion_strategy: mlrun.common.schemas.artifact.ArtifactsDeletionStrategies = (
+            mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
+        ),
+        secrets: dict = None,
+    ):
+        self.artifact_db.del_artifact(
+            key=item.db_key,
+            project=item.project,
+            tag=item.tag,
+            tree=item.tree,
+            deletion_strategy=deletion_strategy,
+            secrets=secrets,
+        )
+
 
 def extend_artifact_path(artifact_path: str, default_artifact_path: str):
     artifact_path = str(artifact_path or "")

mlrun/artifacts/model.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import tempfile
+import warnings
 from os import path
 from typing import Any, Optional
 
@@ -148,6 +149,12 @@ class ModelArtifact(Artifact):
         model_dir=None,
         **kwargs,
     ):
+        if key or body or format or target_path:
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
         super().__init__(key, body, format=format, target_path=target_path, **kwargs)
         model_file = str(model_file or "")
         if model_file and "/" in model_file:

mlrun/artifacts/plots.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import base64
 import typing
+import warnings
 from io import BytesIO
 
 import mlrun
@@ -34,6 +35,12 @@ class PlotArtifact(Artifact):
     def __init__(
         self, key=None, body=None, is_inline=False, target_path=None, title=None
     ):
+        if key or body or is_inline or target_path:
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
         super().__init__(key, body, format="html", target_path=target_path)
         self.metadata.description = title
 
@@ -87,6 +94,12 @@ class PlotlyArtifact(Artifact):
         :param key:         Key for the artifact to be stored in the database.
         :param target_path: Path to save the artifact.
         """
+        if key or target_path:
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
         # Validate the plotly package:
         try:
             from plotly.graph_objs import Figure

mlrun/common/schemas/__init__.py

@@ -14,7 +14,7 @@
 #
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
-from .alert import AlertActiveState, AlertConfig, Event
+from .alert import AlertActiveState, AlertConfig, AlertTemplate, Event
 from .api_gateway import (
     APIGateway,
     APIGatewayAuthenticationMode,
@@ -142,8 +142,10 @@ from .model_monitoring import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
+    MonitoringTSDBTables,
     PrometheusEndpoints,
-    TimeSeriesTarget,
+    TimeSeriesConnector,
+    TSDBTarget,
 )
 from .notification import (
     Notification,

mlrun/common/schemas/alert.py

@@ -26,10 +26,10 @@ class EventEntityKind(StrEnum):
     JOB = "job"
 
 
-class EventEntity(pydantic.BaseModel):
+class EventEntities(pydantic.BaseModel):
     kind: EventEntityKind
     project: str
-    id: str
+    ids: pydantic.conlist(str, min_items=1, max_items=1)
 
 
 class EventKind(StrEnum):
@@ -48,7 +48,7 @@ _event_kind_entity_map = {
 class Event(pydantic.BaseModel):
     kind: EventKind
     timestamp: Union[str, datetime] = None  # occurrence time
-    entity: EventEntity
+    entity: EventEntities
     value_dict: Optional[dict] = pydantic.Field(default_factory=dict)
 
     def is_valid(self):
@@ -71,6 +71,12 @@ class AlertTrigger(pydantic.BaseModel):
     events: list[EventKind] = []
     prometheus_alert: str = None
 
+    def __eq__(self, other):
+        return (
+            self.prometheus_alert == other.prometheus_alert
+            and self.events == other.events
+        )
+
 
 class AlertCriteria(pydantic.BaseModel):
     count: Annotated[
@@ -86,6 +92,9 @@ class AlertCriteria(pydantic.BaseModel):
         ),
     ] = None
 
+    def __eq__(self, other):
+        return self.count == other.count and self.period == other.period
+
 
 class ResetPolicy(StrEnum):
     MANUAL = "manual"
@@ -108,7 +117,7 @@ class AlertConfig(pydantic.BaseModel):
     ]
     created: Union[str, datetime] = None
     severity: AlertSeverity
-    entity: EventEntity
+    entities: EventEntities
     trigger: AlertTrigger
     criteria: Optional[AlertCriteria]
     reset_policy: ResetPolicy = ResetPolicy.MANUAL
@@ -120,3 +129,36 @@ class AlertConfig(pydantic.BaseModel):
 class AlertsModes(StrEnum):
     enabled = "enabled"
     disabled = "disabled"
+
+
+class AlertTemplate(
+    pydantic.BaseModel
+):  # Template fields that are not shared with created configs
+    template_id: int = None
+    template_name: str
+    template_description: Optional[str] = (
+        "String explaining the purpose of this template"
+    )
+
+    # A property that identifies templates that were created by the system and cannot be modified/deleted by the user
+    system_generated: bool = False
+
+    # AlertConfig fields that are pre-defined
+    description: Optional[str] = (
+        "String to be sent in the generated notifications e.g. 'Model {{ $project }}/{{ $entity }} is drifting.'"
+    )
+    severity: AlertSeverity
+    trigger: AlertTrigger
+    criteria: Optional[AlertCriteria]
+    reset_policy: ResetPolicy = ResetPolicy.MANUAL
+
+    # This is slightly different than __eq__ as it doesn't compare everything
+    def templates_differ(self, other):
+        return (
+            self.template_description != other.template_description
+            or self.description != other.description
+            or self.severity != other.severity
+            or self.trigger != other.trigger
+            or self.reset_policy != other.reset_policy
+            or self.criteria != other.criteria
+        )

mlrun/common/schemas/api_gateway.py

@@ -23,6 +23,7 @@ import mlrun.common.types
 class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
     basic = "basicAuth"
     none = "none"
+    access_key = "accessKey"
 
     @classmethod
     def from_str(cls, authentication_mode: str):
@@ -30,6 +31,8 @@ class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
             return cls.none
         elif authentication_mode == "basicAuth":
             return cls.basic
+        elif authentication_mode == "accessKey":
+            return cls.access_key
         else:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Authentication mode `{authentication_mode}` is not supported",
@@ -63,6 +66,7 @@ class APIGatewayUpstream(_APIGatewayBaseModel):
     kind: Optional[str] = "nucliofunction"
     nucliofunction: dict[str, str]
     percentage: Optional[int] = 0
+    port: Optional[int] = 0
 
 
 class APIGatewaySpec(_APIGatewayBaseModel):

mlrun/common/schemas/artifact.py

@@ -93,3 +93,18 @@ class Artifact(pydantic.BaseModel):
     metadata: ArtifactMetadata
     spec: ArtifactSpec
     status: ObjectStatus
+
+
+class ArtifactsDeletionStrategies(mlrun.common.types.StrEnum):
+    """Artifacts deletion strategies types."""
+
+    metadata_only = "metadata-only"
+    """Only removes the artifact db record, leaving all related artifact data in-place"""
+
+    data_optional = "data-optional"
+    """Delete the artifact data of the artifact as a best-effort.
+    If artifact data deletion fails still try to delete the artifact db record"""
+
+    data_force = "data-force"
+    """Delete the artifact data, and if cannot delete it fail the deletion
+    and don’t delete the artifact db record"""

mlrun/common/schemas/auth.py

@@ -59,6 +59,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
     hub_source = "hub-source"
     workflow = "workflow"
     alert = "alert"
+    alert_templates = "alert-templates"
     event = "event"
     datastore_profile = "datastore-profile"
     api_gateway = "api-gateway"
@@ -87,6 +88,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
             AuthorizationResourceTypes.run: "/projects/{project_name}/runs/{resource_name}",
             AuthorizationResourceTypes.event: "/projects/{project_name}/events/{resource_name}",
             AuthorizationResourceTypes.alert: "/projects/{project_name}/alerts/{resource_name}",
+            AuthorizationResourceTypes.alert_templates: "/alert-templates/{resource_name}",
             # runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
             AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
             AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",

mlrun/common/schemas/model_monitoring/__init__.py

@@ -30,14 +30,17 @@ from .constants import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
+    MonitoringTSDBTables,
     ProjectSecretKeys,
     PrometheusEndpoints,
     PrometheusMetric,
     ResultData,
     SchedulingKeys,
-    TimeSeriesTarget,
+    TimeSeriesConnector,
+    TSDBTarget,
     VersionedModel,
     WriterEvent,
+    WriterEventKind,
 )
 from .grafana import (
     GrafanaColumn,

mlrun/common/schemas/model_monitoring/constants.py

@@ -156,7 +156,7 @@ class EventKeyMetrics:
     REAL_TIME = "real_time"
 
 
-class TimeSeriesTarget:
+class TimeSeriesConnector:
     TSDB = "tsdb"
 
 
@@ -194,6 +194,7 @@ class FileTargetKind:
     LOG_STREAM = "log_stream"
     APP_RESULTS = "app_results"
     MONITORING_SCHEDULES = "monitoring_schedules"
+    MONITORING_APPLICATION = "monitoring_application"
 
 
 class ModelMonitoringMode(str, Enum):
@@ -228,6 +229,12 @@ class MonitoringFunctionNames(MonitoringStrEnum):
     WRITER = "model-monitoring-writer"
 
 
+class MonitoringTSDBTables(MonitoringStrEnum):
+    APP_RESULTS = "app-results"
+    METRICS = "metrics"
+    EVENTS = "events"
+
+
 @dataclass
 class FunctionURI:
     project: str
@@ -329,6 +336,14 @@ class ControllerPolicy:
     BASE_PERIOD = "base_period"
 
 
+class TSDBTarget:
+    V3IO_TSDB = "v3io-tsdb"
+    PROMETHEUS = "prometheus"
+    APP_RESULTS_TABLE = "app-results"
+    V3IO_BE = "tsdb"
+    V3IO_RATE = "1/s"
+
+
 class HistogramDataDriftApplicationConstants:
     NAME = "histogram-data-drift"
     GENERAL_RESULT_NAME = "general_drift"