mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/common/formatters/function.py

@@ -0,0 +1,41 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class FunctionFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    minimal = "minimal"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            FunctionFormat.full: None,
+            FunctionFormat.minimal: FunctionFormat.filter_obj_method(
+                [
+                    ["kind"],
+                    ["metadata"],
+                    ["status"],
+                    ["spec", "description"],
+                    ["spec", "image"],
+                    ["spec", "default_handler"],
+                    ["spec", "entry_points"],
+                ]
+            ),
+        }[_format]

mlrun/common/formatters/pipeline.py

@@ -0,0 +1,53 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun_pipelines.common.ops
+import mlrun_pipelines.models
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class PipelineFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+    metadata_only = "metadata_only"
+    name_only = "name_only"
+    summary = "summary"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        def _full(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return run.to_dict()
+
+        def _metadata_only(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return mlrun.utils.helpers.format_run(run, with_project=True)
+
+        def _name_only(run: mlrun_pipelines.models.PipelineRun) -> str:
+            return run.get("name")
+
+        def _summary(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return mlrun_pipelines.common.ops.format_summary_from_kfp_run(
+                run, run["project"]
+            )
+
+        return {
+            PipelineFormat.full: _full,
+            PipelineFormat.metadata_only: _metadata_only,
+            PipelineFormat.name_only: _name_only,
+            PipelineFormat.summary: _summary,
+        }[_format]

mlrun/common/formatters/project.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.schemas
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class ProjectFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+    name_only = "name_only"
+    # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
+    # and is used for faster response times (in the UI)
+    minimal = "minimal"
+    # internal - allowed only in follower mode, only for the leader for upgrade purposes
+    leader = "leader"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        def _name_only(project: mlrun.common.schemas.Project) -> str:
+            return project.metadata.name
+
+        def _minimal(
+            project: mlrun.common.schemas.Project,
+        ) -> mlrun.common.schemas.Project:
+            project.spec.functions = None
+            project.spec.workflows = None
+            project.spec.artifacts = None
+            return project
+
+        return {
+            ProjectFormat.full: None,
+            ProjectFormat.name_only: _name_only,
+            ProjectFormat.minimal: _minimal,
+            ProjectFormat.leader: None,
+        }[_format]

mlrun/common/helpers.py

@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import typing
 
 
 def parse_versioned_object_uri(
     uri: str, default_project: str = ""
-) -> typing.Tuple[str, str, str, str]:
+) -> tuple[str, str, str, str]:
     project = default_project
     tag = ""
     hash_key = ""

mlrun/common/model_monitoring/helpers.py

@@ -16,6 +16,7 @@ import sys
 import typing
 
 import mlrun.common
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
 from mlrun.common.schemas.model_monitoring import (
     EndpointUID,
     FunctionURI,
@@ -64,7 +65,7 @@ def parse_model_endpoint_store_prefix(store_prefix: str):
 
 
 def parse_monitoring_stream_path(
-    stream_uri: str, project: str, application_name: str = None
+    stream_uri: str, project: str, function_name: str = None
 ):
     if stream_uri.startswith("kafka://"):
         if "?topic" in stream_uri:
@@ -72,14 +73,17 @@ def parse_monitoring_stream_path(
                 "Custom kafka topic is not allowed"
             )
         # Add topic to stream kafka uri
-        if application_name is None:
+        if (
+            function_name is None
+            or function_name == mm_constants.MonitoringFunctionNames.STREAM
+        ):
             stream_uri += f"?topic=monitoring_stream_{project}"
         else:
-            stream_uri += f"?topic=monitoring_stream_{project}_{application_name}"
+            stream_uri += f"?topic=monitoring_stream_{project}_{function_name}"
 
     elif stream_uri.startswith("v3io://") and mlrun.mlconf.is_ce_mode():
         # V3IO is not supported in CE mode, generating a default http stream path
-        if application_name is None:
+        if function_name is None:
             stream_uri = (
                 mlrun.mlconf.model_endpoint_monitoring.default_http_sink.format(
                     project=project, namespace=mlrun.mlconf.namespace
@@ -89,7 +93,7 @@ def parse_monitoring_stream_path(
             stream_uri = (
                 mlrun.mlconf.model_endpoint_monitoring.default_http_sink_app.format(
                     project=project,
-                    application_name=application_name,
+                    application_name=function_name,
                     namespace=mlrun.mlconf.namespace,
                 )
             )

mlrun/{runtimes → common/runtimes}/constants.py

@@ -15,6 +15,10 @@
 import enum
 import typing
 
+import mlrun_pipelines.common.models
+
+import mlrun.common.constants as mlrun_constants
+
 
 class PodPhases:
     """
@@ -75,7 +79,7 @@ class ThresholdStates:
         ]
 
     @staticmethod
-    def from_pod_phase(pod_phase: str, pod: typing.Dict) -> typing.Optional[str]:
+    def from_pod_phase(pod_phase: str, pod: dict) -> typing.Optional[str]:
         if pod_phase == PodPhases.pending:
             if ThresholdStates.is_pod_in_image_pull_backoff(pod):
                 return ThresholdStates.image_pull_backoff
@@ -90,7 +94,7 @@ class ThresholdStates:
         return None
 
     @staticmethod
-    def is_pod_scheduled(pod: typing.Dict):
+    def is_pod_scheduled(pod: dict):
         conditions = pod["status"].get("conditions", []) or []
         for condition in conditions:
             if condition["type"] == "PodScheduled" and condition["status"] == "True":
@@ -98,7 +102,7 @@ class ThresholdStates:
         return False
 
     @staticmethod
-    def is_pod_in_image_pull_backoff(pod: typing.Dict):
+    def is_pod_in_image_pull_backoff(pod: dict):
         container_statuses = pod.get("status").get("container_statuses", []) or []
         for container_status in container_statuses:
             state_waiting = container_status.get("state", {}).get("waiting", {}) or {}
@@ -107,7 +111,7 @@ class ThresholdStates:
         return False
 
 
-class MPIJobCRDVersions(object):
+class MPIJobCRDVersions:
     v1 = "v1"
     v1alpha1 = "v1alpha1"
 
@@ -122,12 +126,12 @@ class MPIJobCRDVersions(object):
     @staticmethod
     def role_label_by_version(version):
         return {
-            MPIJobCRDVersions.v1alpha1: "mpi_role_type",
-            MPIJobCRDVersions.v1: "mpi-job-role",
+            MPIJobCRDVersions.v1alpha1: mlrun_constants.MLRunInternalLabels.mpi_role_type,
+            MPIJobCRDVersions.v1: mlrun_constants.MLRunInternalLabels.mpi_job_role,
         }[version]
 
 
-class RunStates(object):
+class RunStates:
     completed = "completed"
     error = "error"
     running = "running"
@@ -136,6 +140,7 @@ class RunStates(object):
     unknown = "unknown"
     aborted = "aborted"
     aborting = "aborting"
+    skipped = "skipped"
 
     @staticmethod
     def all():
@@ -148,6 +153,7 @@ class RunStates(object):
             RunStates.unknown,
             RunStates.aborted,
             RunStates.aborting,
+            RunStates.skipped,
         ]
 
     @staticmethod
@@ -156,6 +162,7 @@ class RunStates(object):
             RunStates.completed,
             RunStates.error,
             RunStates.aborted,
+            RunStates.skipped,
         ]
 
     @staticmethod
@@ -188,10 +195,31 @@ class RunStates(object):
             # TODO: add aborting state once we have it
         ]
 
+    @staticmethod
+    def run_state_to_pipeline_run_status(run_state: str):
+        if not run_state:
+            return mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified
 
+        if run_state not in RunStates.all():
+            raise ValueError(f"Invalid run state: {run_state}")
+
+        return {
+            RunStates.completed: mlrun_pipelines.common.models.RunStatuses.succeeded,
+            RunStates.error: mlrun_pipelines.common.models.RunStatuses.failed,
+            RunStates.running: mlrun_pipelines.common.models.RunStatuses.running,
+            RunStates.created: mlrun_pipelines.common.models.RunStatuses.pending,
+            RunStates.pending: mlrun_pipelines.common.models.RunStatuses.pending,
+            RunStates.unknown: mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified,
+            RunStates.aborted: mlrun_pipelines.common.models.RunStatuses.canceled,
+            RunStates.aborting: mlrun_pipelines.common.models.RunStatuses.canceling,
+            RunStates.skipped: mlrun_pipelines.common.models.RunStatuses.skipped,
+        }[run_state]
+
+
+# TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
 class RunLabels(enum.Enum):
-    owner = "owner"
-    v3io_user = "v3io_user"
+    owner = mlrun_constants.MLRunInternalLabels.owner
+    v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user
 
     @staticmethod
     def all():

mlrun/common/schemas/__init__.py

@@ -14,12 +14,29 @@
 #
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
+from .alert import (
+    AlertActiveState,
+    AlertConfig,
+    AlertNotification,
+    AlertTemplate,
+    Event,
+)
+from .api_gateway import (
+    APIGateway,
+    APIGatewayAuthenticationMode,
+    APIGatewayBasicAuth,
+    APIGatewayMetadata,
+    APIGatewaysOutput,
+    APIGatewaySpec,
+    APIGatewayState,
+    APIGatewayStatus,
+    APIGatewayUpstream,
+)
 from .artifact import (
     Artifact,
     ArtifactCategories,
     ArtifactIdentifier,
     ArtifactMetadata,
-    ArtifactsFormat,
     ArtifactSpec,
 )
 from .auth import (
@@ -130,7 +147,9 @@ from .model_monitoring import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
-    TimeSeriesTarget,
+    PrometheusEndpoints,
+    TSDBTarget,
+    V3IOTSDBTables,
 )
 from .notification import (
     Notification,
@@ -140,14 +159,15 @@ from .notification import (
     SetNotificationRequest,
 )
 from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
-from .pipeline import PipelinesFormat, PipelinesOutput, PipelinesPagination
+from .pagination import PaginationInfo
+from .pipeline import PipelinesOutput, PipelinesPagination
 from .project import (
     IguazioProject,
     Project,
     ProjectDesiredState,
     ProjectMetadata,
+    ProjectOutput,
     ProjectOwner,
-    ProjectsFormat,
     ProjectsOutput,
     ProjectSpec,
     ProjectState,

mlrun/common/schemas/alert.py

@@ -0,0 +1,203 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import datetime
+from typing import Annotated, Optional, Union
+
+import pydantic
+
+from mlrun.common.schemas.notification import Notification
+from mlrun.common.types import StrEnum
+
+
+class EventEntityKind(StrEnum):
+    MODEL_ENDPOINT_RESULT = "model-endpoint-result"
+    JOB = "job"
+
+
+class EventEntities(pydantic.BaseModel):
+    kind: EventEntityKind
+    project: str
+    ids: pydantic.conlist(str, min_items=1, max_items=1)
+
+
+class EventKind(StrEnum):
+    DATA_DRIFT_DETECTED = "data_drift_detected"
+    DATA_DRIFT_SUSPECTED = "data_drift_suspected"
+    CONCEPT_DRIFT_DETECTED = "concept_drift_detected"
+    CONCEPT_DRIFT_SUSPECTED = "concept_drift_suspected"
+    MODEL_PERFORMANCE_DETECTED = "model_performance_detected"
+    MODEL_PERFORMANCE_SUSPECTED = "model_performance_suspected"
+    MODEL_SERVING_PERFORMANCE_DETECTED = "model_serving_performance_detected"
+    MODEL_SERVING_PERFORMANCE_SUSPECTED = "model_serving_performance_suspected"
+    MM_APP_ANOMALY_DETECTED = "mm_app_anomaly_detected"
+    MM_APP_ANOMALY_SUSPECTED = "mm_app_anomaly_suspected"
+    FAILED = "failed"
+
+
+_event_kind_entity_map = {
+    EventKind.DATA_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.DATA_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.CONCEPT_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MODEL_SERVING_PERFORMANCE_DETECTED: [
+        EventEntityKind.MODEL_ENDPOINT_RESULT
+    ],
+    EventKind.MODEL_SERVING_PERFORMANCE_SUSPECTED: [
+        EventEntityKind.MODEL_ENDPOINT_RESULT
+    ],
+    EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.FAILED: [EventEntityKind.JOB],
+}
+
+
+class Event(pydantic.BaseModel):
+    kind: EventKind
+    timestamp: Union[str, datetime] = None  # occurrence time
+    entity: EventEntities
+    value_dict: Optional[dict] = pydantic.Field(default_factory=dict)
+
+    def is_valid(self):
+        return self.entity.kind in _event_kind_entity_map[self.kind]
+
+
+class AlertActiveState(StrEnum):
+    ACTIVE = "active"
+    INACTIVE = "inactive"
+
+
+class AlertSeverity(StrEnum):
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+
+
+# what should trigger the alert. must be either event (at least 1), or prometheus query
+class AlertTrigger(pydantic.BaseModel):
+    events: list[EventKind] = []
+    prometheus_alert: str = None
+
+    def __eq__(self, other):
+        return (
+            self.prometheus_alert == other.prometheus_alert
+            and self.events == other.events
+        )
+
+
+class AlertCriteria(pydantic.BaseModel):
+    count: Annotated[
+        int,
+        pydantic.Field(
+            description="Number of events to wait until notification is sent"
+        ),
+    ] = 0
+    period: Annotated[
+        str,
+        pydantic.Field(
+            description="Time period during which event occurred. e.g. 1d, 3h, 5m, 15s"
+        ),
+    ] = None
+
+    def __eq__(self, other):
+        return self.count == other.count and self.period == other.period
+
+
+class ResetPolicy(StrEnum):
+    MANUAL = "manual"
+    AUTO = "auto"
+
+
+class AlertNotification(pydantic.BaseModel):
+    notification: Notification
+    cooldown_period: Annotated[
+        str,
+        pydantic.Field(
+            description="Period during which notifications "
+            "will not be sent after initial send. The format of this would be in time."
+            " e.g. 1d, 3h, 5m, 15s"
+        ),
+    ] = None
+
+
+class AlertConfig(pydantic.BaseModel):
+    project: str
+    id: int = None
+    name: str
+    description: Optional[str] = ""
+    summary: Annotated[
+        str,
+        pydantic.Field(
+            description=(
+                "String to be sent in the notifications generated."
+                "e.g. 'Model {{project}}/{{entity}} is drifting.'"
+                "Supported variables: project, entity, name"
+            )
+        ),
+    ]
+    created: Union[str, datetime] = None
+    severity: AlertSeverity
+    entities: EventEntities
+    trigger: AlertTrigger
+    criteria: Optional[AlertCriteria]
+    reset_policy: ResetPolicy = ResetPolicy.MANUAL
+    notifications: pydantic.conlist(AlertNotification, min_items=1)
+    state: AlertActiveState = AlertActiveState.INACTIVE
+    count: Optional[int] = 0
+
+    def get_raw_notifications(self) -> list[Notification]:
+        return [
+            alert_notification.notification for alert_notification in self.notifications
+        ]
+
+
+class AlertsModes(StrEnum):
+    enabled = "enabled"
+    disabled = "disabled"
+
+
+class AlertTemplate(
+    pydantic.BaseModel
+):  # Template fields that are not shared with created configs
+    template_id: int = None
+    template_name: str
+    template_description: Optional[str] = (
+        "String explaining the purpose of this template"
+    )
+
+    # A property that identifies templates that were created by the system and cannot be modified/deleted by the user
+    system_generated: bool = False
+
+    # AlertConfig fields that are pre-defined
+    summary: Optional[str] = (
+        "String to be sent in the generated notifications e.g. 'Model {{project}}/{{entity}} is drifting.'"
+        "See AlertConfig.summary description"
+    )
+    severity: AlertSeverity
+    trigger: AlertTrigger
+    criteria: Optional[AlertCriteria]
+    reset_policy: ResetPolicy = ResetPolicy.MANUAL
+
+    # This is slightly different than __eq__ as it doesn't compare everything
+    def templates_differ(self, other):
+        return (
+            self.template_description != other.template_description
+            or self.summary != other.summary
+            or self.severity != other.severity
+            or self.trigger != other.trigger
+            or self.reset_policy != other.reset_policy
+            or self.criteria != other.criteria
+        )