mlrun 1.7.0rc4__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/common/schemas/alert.py

@@ -0,0 +1,202 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import datetime
+from typing import Annotated, Optional, Union
+
+import pydantic
+
+from mlrun.common.schemas.notification import Notification
+from mlrun.common.types import StrEnum
+
+
+class EventEntityKind(StrEnum):
+    MODEL_ENDPOINT_RESULT = "model-endpoint-result"
+    MODEL_MONITORING_APPLICATION = "model-monitoring-application"
+    JOB = "job"
+
+
+class EventEntities(pydantic.BaseModel):
+    kind: EventEntityKind
+    project: str
+    ids: pydantic.conlist(str, min_items=1, max_items=1)
+
+
+class EventKind(StrEnum):
+    DATA_DRIFT_DETECTED = "data-drift-detected"
+    DATA_DRIFT_SUSPECTED = "data-drift-suspected"
+    CONCEPT_DRIFT_DETECTED = "concept-drift-detected"
+    CONCEPT_DRIFT_SUSPECTED = "concept-drift-suspected"
+    MODEL_PERFORMANCE_DETECTED = "model-performance-detected"
+    MODEL_PERFORMANCE_SUSPECTED = "model-performance-suspected"
+    SYSTEM_PERFORMANCE_DETECTED = "system-performance-detected"
+    SYSTEM_PERFORMANCE_SUSPECTED = "system-performance-suspected"
+    MM_APP_ANOMALY_DETECTED = "mm-app-anomaly-detected"
+    MM_APP_ANOMALY_SUSPECTED = "mm-app-anomaly-suspected"
+    MM_APP_FAILED = "mm-app-failed"
+    FAILED = "failed"
+
+
+_event_kind_entity_map = {
+    EventKind.DATA_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.DATA_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.CONCEPT_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.SYSTEM_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.SYSTEM_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MM_APP_FAILED: [EventEntityKind.MODEL_MONITORING_APPLICATION],
+    EventKind.FAILED: [EventEntityKind.JOB],
+}
+
+
+class Event(pydantic.BaseModel):
+    kind: EventKind
+    timestamp: Union[str, datetime] = None  # occurrence time
+    entity: EventEntities
+    value_dict: Optional[dict] = pydantic.Field(default_factory=dict)
+
+    def is_valid(self):
+        return self.entity.kind in _event_kind_entity_map[self.kind]
+
+
+class AlertActiveState(StrEnum):
+    ACTIVE = "active"
+    INACTIVE = "inactive"
+
+
+class AlertSeverity(StrEnum):
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+
+
+# what should trigger the alert. must be either event (at least 1), or prometheus query
+class AlertTrigger(pydantic.BaseModel):
+    events: list[EventKind] = []
+    prometheus_alert: str = None
+
+    def __eq__(self, other):
+        return (
+            self.prometheus_alert == other.prometheus_alert
+            and self.events == other.events
+        )
+
+
+class AlertCriteria(pydantic.BaseModel):
+    count: Annotated[
+        int,
+        pydantic.Field(
+            description="Number of events to wait until notification is sent"
+        ),
+    ] = 1
+    period: Annotated[
+        str,
+        pydantic.Field(
+            description="Time period during which event occurred. e.g. 1d, 3h, 5m, 15s"
+        ),
+    ] = None
+
+    def __eq__(self, other):
+        return self.count == other.count and self.period == other.period
+
+
+class ResetPolicy(StrEnum):
+    MANUAL = "manual"
+    AUTO = "auto"
+
+
+class AlertNotification(pydantic.BaseModel):
+    notification: Notification
+    cooldown_period: Annotated[
+        str,
+        pydantic.Field(
+            description="Period during which notifications "
+            "will not be sent after initial send. The format of this would be in time."
+            " e.g. 1d, 3h, 5m, 15s"
+        ),
+    ] = None
+
+
+class AlertConfig(pydantic.BaseModel):
+    project: str
+    id: int = None
+    name: str
+    description: Optional[str] = ""
+    summary: Annotated[
+        str,
+        pydantic.Field(
+            description=(
+                "String to be sent in the notifications generated."
+                "e.g. 'Model {{project}}/{{entity}} is drifting.'"
+                "Supported variables: project, entity, name"
+            )
+        ),
+    ]
+    created: Union[str, datetime] = None
+    severity: AlertSeverity
+    entities: EventEntities
+    trigger: AlertTrigger
+    criteria: Optional[AlertCriteria]
+    reset_policy: ResetPolicy = ResetPolicy.AUTO
+    notifications: pydantic.conlist(AlertNotification, min_items=1)
+    state: AlertActiveState = AlertActiveState.INACTIVE
+    count: Optional[int] = 0
+
+    def get_raw_notifications(self) -> list[Notification]:
+        return [
+            alert_notification.notification for alert_notification in self.notifications
+        ]
+
+
+class AlertsModes(StrEnum):
+    enabled = "enabled"
+    disabled = "disabled"
+
+
+class AlertTemplate(
+    pydantic.BaseModel
+):  # Template fields that are not shared with created configs
+    template_id: int = None
+    template_name: str
+    template_description: Optional[str] = (
+        "String explaining the purpose of this template"
+    )
+
+    # A property that identifies templates that were created by the system and cannot be modified/deleted by the user
+    system_generated: bool = False
+
+    # AlertConfig fields that are pre-defined
+    summary: Optional[str] = (
+        "String to be sent in the generated notifications e.g. 'Model {{project}}/{{entity}} is drifting.'"
+        "See AlertConfig.summary description"
+    )
+    severity: AlertSeverity
+    trigger: AlertTrigger
+    criteria: Optional[AlertCriteria]
+    reset_policy: ResetPolicy = ResetPolicy.AUTO
+
+    # This is slightly different than __eq__ as it doesn't compare everything
+    def templates_differ(self, other):
+        return (
+            self.template_description != other.template_description
+            or self.summary != other.summary
+            or self.severity != other.severity
+            or self.trigger != other.trigger
+            or self.reset_policy != other.reset_policy
+            or self.criteria != other.criteria
+        )

mlrun/common/schemas/api_gateway.py

@@ -0,0 +1,196 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import typing
+from typing import Optional
+
+import pydantic
+
+import mlrun.common.constants as mlrun_constants
+import mlrun.common.types
+from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
+from mlrun.common.helpers import generate_api_gateway_name
+
+
+class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
+    basic = "basicAuth"
+    none = "none"
+    access_key = "accessKey"
+
+    @classmethod
+    def from_str(cls, authentication_mode: str):
+        if authentication_mode == "none":
+            return cls.none
+        elif authentication_mode == "basicAuth":
+            return cls.basic
+        elif authentication_mode == "accessKey":
+            return cls.access_key
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Authentication mode `{authentication_mode}` is not supported",
+            )
+
+
+class APIGatewayState(mlrun.common.types.StrEnum):
+    none = ""
+    ready = "ready"
+    error = "error"
+    waiting_for_provisioning = "waitingForProvisioning"
+
+
+class _APIGatewayBaseModel(pydantic.BaseModel):
+    class Config:
+        extra = pydantic.Extra.allow
+
+
+class APIGatewayMetadata(_APIGatewayBaseModel):
+    name: str
+    namespace: Optional[str]
+    labels: Optional[dict] = {}
+    annotations: Optional[dict] = {}
+
+
+class APIGatewayBasicAuth(_APIGatewayBaseModel):
+    username: str
+    password: str
+
+
+class APIGatewayUpstream(_APIGatewayBaseModel):
+    kind: Optional[str] = "nucliofunction"
+    nucliofunction: dict[str, str]
+    percentage: Optional[int] = 0
+    port: Optional[int] = 0
+
+
+class APIGatewaySpec(_APIGatewayBaseModel):
+    name: str
+    description: Optional[str]
+    path: Optional[str] = "/"
+    authenticationMode: Optional[APIGatewayAuthenticationMode] = (  # noqa: N815 - for compatibility with Nuclio https://github.com/nuclio/nuclio/blob/672b8e36f9edd6e42b4685ec1d27cabae3c5f045/pkg/platform/types.go#L476
+        APIGatewayAuthenticationMode.none
+    )
+    upstreams: list[APIGatewayUpstream]
+    authentication: Optional[dict[str, Optional[APIGatewayBasicAuth]]]
+    host: Optional[str]
+
+
+class APIGatewayStatus(_APIGatewayBaseModel):
+    name: Optional[str]
+    state: Optional[APIGatewayState]
+
+
+class APIGateway(_APIGatewayBaseModel):
+    metadata: APIGatewayMetadata
+    spec: APIGatewaySpec
+    status: Optional[APIGatewayStatus]
+
+    def get_function_names(self):
+        return [
+            upstream.nucliofunction.get("name")
+            for upstream in self.spec.upstreams
+            if upstream.nucliofunction.get("name")
+        ]
+
+    def get_invoke_url(self):
+        if self.spec.host and self.spec.path:
+            return f"{self.spec.host.rstrip('/')}/{self.spec.path.lstrip('/')}".rstrip(
+                "/"
+            )
+        return self.spec.host.rstrip("/")
+
+    def enrich_mlrun_names(self):
+        self._enrich_api_gateway_mlrun_name()
+        self._enrich_mlrun_function_names()
+        return self
+
+    def replace_nuclio_names_with_mlrun_names(self):
+        self._replace_nuclio_api_gateway_name_with_mlrun_name()
+        self._replace_nuclio_function_names_with_mlrun_names()
+        return self
+
+    def _replace_nuclio_function_names_with_mlrun_names(self):
+        # replace function names from nuclio names to mlrun names
+        # and adds mlrun function URI's to an api gateway annotations
+        # so when we then get api gateway entity from nuclio, we are able to get mlrun function names
+        mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
+        if mlrun_functions:
+            mlrun_function_uris = (
+                mlrun_functions.split("&")
+                if "&" in mlrun_functions
+                else [mlrun_functions]
+            )
+            if len(mlrun_function_uris) != len(self.spec.upstreams):
+                raise mlrun.errors.MLRunValueError(
+                    "Error when translating nuclio names to mlrun names in api gateway:"
+                    "  number of functions doesn't match the mlrun functions in annotation"
+                )
+            for i in range(len(mlrun_function_uris)):
+                self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
+        return self
+
+    def _replace_nuclio_api_gateway_name_with_mlrun_name(self):
+        # replace api gateway name
+        # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
+        # remove the project prefix from the name if it exists
+        project_name = self.metadata.labels.get(
+            mlrun_constants.MLRunInternalLabels.nuclio_project_name
+        )
+        if project_name and self.spec.name.startswith(f"{project_name}-"):
+            self.spec.name = self.spec.name[len(project_name) + 1 :]
+            self.metadata.name = self.spec.name
+        return self
+
+    def _enrich_mlrun_function_names(self):
+        # enrich mlrun names with nuclio prefixes
+        # and add mlrun function's URIs to Nuclio function annotations
+        upstream_with_nuclio_names = []
+        mlrun_function_uris = []
+        for upstream in self.spec.upstreams:
+            uri = upstream.nucliofunction.get("name")
+            project, function_name, tag, _ = (
+                mlrun.common.helpers.parse_versioned_object_uri(uri)
+            )
+            upstream.nucliofunction["name"] = (
+                mlrun.runtimes.nuclio.function.get_fullname(function_name, project, tag)
+            )
+
+            upstream_with_nuclio_names.append(upstream)
+            mlrun_function_uris.append(uri)
+
+        self.spec.upstreams = upstream_with_nuclio_names
+        if len(mlrun_function_uris) == 1:
+            self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = mlrun_function_uris[
+                0
+            ]
+        elif len(mlrun_function_uris) == 2:
+            self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = "&".join(
+                mlrun_function_uris
+            )
+        return self
+
+    def _enrich_api_gateway_mlrun_name(self):
+        # replace api gateway name
+        # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
+        # add the project prefix to the name
+        project_name = self.metadata.labels.get(
+            mlrun_constants.MLRunInternalLabels.nuclio_project_name
+        )
+        if project_name:
+            self.spec.name = generate_api_gateway_name(project_name, self.spec.name)
+            self.metadata.name = self.spec.name
+        return self
+
+
+class APIGatewaysOutput(_APIGatewayBaseModel):
+    api_gateways: typing.Optional[dict[str, APIGateway]] = {}

mlrun/common/schemas/artifact.py

@@ -15,6 +15,7 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
@@ -46,6 +47,12 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
                 True,
             )
 
+    @classmethod
+    def from_kind(cls, kind: str) -> "ArtifactCategories":
+        if kind in [cls.model.value, cls.dataset.value]:
+            return cls(kind)
+        return cls.other
+
 
 class ArtifactIdentifier(pydantic.BaseModel):
     # artifact kind
@@ -58,8 +65,13 @@ class ArtifactIdentifier(pydantic.BaseModel):
     # hash: typing.Optional[str]
 
 
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.ArtifactFormat instead.",
+    category=FutureWarning,
+)
 class ArtifactsFormat(mlrun.common.types.StrEnum):
-    # TODO: add a format that returns a minimal response
     full = "full"
 
 
@@ -93,3 +105,18 @@ class Artifact(pydantic.BaseModel):
     metadata: ArtifactMetadata
     spec: ArtifactSpec
     status: ObjectStatus
+
+
+class ArtifactsDeletionStrategies(mlrun.common.types.StrEnum):
+    """Artifacts deletion strategies types."""
+
+    metadata_only = "metadata-only"
+    """Only removes the artifact db record, leaving all related artifact data in-place"""
+
+    data_optional = "data-optional"
+    """Delete the artifact data of the artifact as a best-effort.
+    If artifact data deletion fails still try to delete the artifact db record"""
+
+    data_force = "data-force"
+    """Delete the artifact data, and if cannot delete it fail the deletion
+    and don’t delete the artifact db record"""

mlrun/common/schemas/auth.py

@@ -58,8 +58,12 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
     pipeline = "pipeline"
     hub_source = "hub-source"
     workflow = "workflow"
+    alert = "alert"
+    alert_templates = "alert-templates"
+    event = "event"
     datastore_profile = "datastore-profile"
-    api_gateways = "api-gateways"
+    api_gateway = "api-gateway"
+    project_summaries = "project-summaries"
 
     def to_resource_string(
         self,
@@ -69,6 +73,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
         return {
             # project is the resource itself, so no need for both resource_name and project_name
             AuthorizationResourceTypes.project: "/projects/{project_name}",
+            AuthorizationResourceTypes.project_summaries: "/projects/{project_name}/project-summaries/{resource_name}",
             AuthorizationResourceTypes.function: "/projects/{project_name}/functions/{resource_name}",
             AuthorizationResourceTypes.artifact: "/projects/{project_name}/artifacts/{resource_name}",
             AuthorizationResourceTypes.project_background_task: (
@@ -83,6 +88,9 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
             AuthorizationResourceTypes.schedule: "/projects/{project_name}/schedules/{resource_name}",
             AuthorizationResourceTypes.secret: "/projects/{project_name}/secrets/{resource_name}",
             AuthorizationResourceTypes.run: "/projects/{project_name}/runs/{resource_name}",
+            AuthorizationResourceTypes.event: "/projects/{project_name}/events/{resource_name}",
+            AuthorizationResourceTypes.alert: "/projects/{project_name}/alerts/{resource_name}",
+            AuthorizationResourceTypes.alert_templates: "/alert-templates/{resource_name}",
             # runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
             AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
             AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",
@@ -94,7 +102,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
             AuthorizationResourceTypes.hub_source: "/marketplace/sources",
             # workflow define how to run a pipeline and can be considered as the specification of a pipeline.
             AuthorizationResourceTypes.workflow: "/projects/{project_name}/workflows/{resource_name}",
-            AuthorizationResourceTypes.api_gateways: "/projects/{project_name}/api-gateways",
+            AuthorizationResourceTypes.api_gateway: "/projects/{project_name}/api-gateways/{resource_name}",
         }[self].format(project_name=project_name, resource_name=resource_name)
 
 
@@ -133,6 +141,9 @@ class AuthInfo(pydantic.BaseModel):
             member_ids.extend(self.user_group_ids)
         return member_ids
 
+    def get_session(self) -> str:
+        return self.data_session or self.session
+
 
 class Credentials(pydantic.BaseModel):
     access_key: typing.Optional[str]

mlrun/common/schemas/client_spec.py

@@ -57,8 +57,8 @@ class ClientSpec(pydantic.BaseModel):
     redis_url: typing.Optional[str]
     redis_type: typing.Optional[str]
     sql_url: typing.Optional[str]
-    model_endpoint_monitoring_store_type: typing.Optional[str]
     model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
+    model_monitoring_tsdb_connection: typing.Optional[str]
     ce: typing.Optional[dict]
     # not passing them as one object as it possible client user would like to override only one of the params
     calculate_artifact_hash: typing.Optional[str]
@@ -66,3 +66,4 @@ class ClientSpec(pydantic.BaseModel):
     logs: typing.Optional[dict]
     packagers: typing.Optional[dict]
     external_platform_tracking: typing.Optional[dict]
+    alerts_mode: typing.Optional[str]

mlrun/common/schemas/common.py

@@ -11,16 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
 import typing
 
 import pydantic
 
 
 class ImageBuilder(pydantic.BaseModel):
-    functionSourceCode: typing.Optional[str] = None
-    codeEntryType: typing.Optional[str] = None
-    codeEntryAttributes: typing.Optional[str] = None
+    functionSourceCode: typing.Optional[str] = None  # noqa: N815
+    codeEntryType: typing.Optional[str] = None  # noqa: N815
+    codeEntryAttributes: typing.Optional[str] = None  # noqa: N815
     source: typing.Optional[str] = None
     code_origin: typing.Optional[str] = None
     origin_filename: typing.Optional[str] = None
@@ -38,3 +38,6 @@ class ImageBuilder(pydantic.BaseModel):
     build_pod: typing.Optional[str] = None
     requirements: typing.Optional[list] = None
     source_code_target_dir: typing.Optional[str] = None
+
+    class Config:
+        extra = pydantic.Extra.allow

mlrun/common/schemas/constants.py

@@ -120,10 +120,13 @@ class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
 
 class RunPartitionByField(mlrun.common.types.StrEnum):
     name = "name"  # Supported for runs objects
+    project_and_name = "project_and_name"  # Supported for runs objects
 
     def to_partition_by_db_field(self, db_cls):
         if self.value == RunPartitionByField.name:
             return db_cls.name
+        elif self.value == RunPartitionByField.project_and_name:
+            return db_cls.project, db_cls.name
         else:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Unknown group by field: {self.value}"