mlrun 1.7.0rc4__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -11,26 +11,35 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import enum
 import json
-from typing import Any, Optional
+from datetime import datetime
+from typing import Any, NamedTuple, Optional, TypeVar
 
-from pydantic import BaseModel, Field, validator
-from pydantic.main import Extra
+from pydantic import BaseModel, Extra, Field, constr, validator
 
+# TODO: remove the unused import below after `mlrun.datastore` and `mlrun.utils` usage is removed.
+# At the moment `make lint` fails if this is removed.
 import mlrun.common.model_monitoring
 
 from ..object import ObjectKind, ObjectSpec, ObjectStatus
 from .constants import (
+    FQN_REGEX,
+    MODEL_ENDPOINT_ID_PATTERN,
+    PROJECT_PATTERN,
     EndpointType,
     EventFieldType,
     EventKeyMetrics,
     EventLiveStats,
+    ModelEndpointMonitoringMetricType,
     ModelMonitoringMode,
+    ResultKindApp,
+    ResultStatusApp,
 )
 
+Model = TypeVar("Model", bound=BaseModel)
+
 
 class ModelMonitoringStoreKinds:
     # TODO: do changes in examples & demos In 1.5.0 remove
@@ -39,9 +48,9 @@ class ModelMonitoringStoreKinds:
 
 
 class ModelEndpointMetadata(BaseModel):
-    project: Optional[str] = ""
+    project: constr(regex=PROJECT_PATTERN)
+    uid: constr(regex=MODEL_ENDPOINT_ID_PATTERN)
     labels: Optional[dict] = {}
-    uid: Optional[str] = ""
 
     class Config:
         extra = Extra.allow
@@ -54,12 +63,11 @@ class ModelEndpointMetadata(BaseModel):
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [EventFieldType.LABELS]
 
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
@@ -86,7 +94,6 @@ class ModelEndpointSpec(ObjectSpec):
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [
                 EventFieldType.FEATURE_NAMES,
@@ -94,23 +101,13 @@ class ModelEndpointSpec(ObjectSpec):
                 EventFieldType.MONITOR_CONFIGURATION,
             ]
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
 
-    @validator("monitor_configuration")
-    def set_name(cls, monitor_configuration):
-        return monitor_configuration or {
-            EventFieldType.DRIFT_DETECTED_THRESHOLD: (
-                mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.drift_detected
-            ),
-            EventFieldType.POSSIBLE_DRIFT_THRESHOLD: (
-                mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.possible_drift
-            ),
-        }
-
     @validator("model_uri")
+    @classmethod
     def validate_model_uri(cls, model_uri):
         """Validate that the model uri includes the required prefix"""
         prefix, uri = mlrun.datastore.parse_store_uri(model_uri)
@@ -198,7 +195,6 @@ class ModelEndpointStatus(ObjectStatus):
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [
                 EventFieldType.FEATURE_STATS,
@@ -210,7 +206,7 @@ class ModelEndpointStatus(ObjectStatus):
                 EventFieldType.ENDPOINT_TYPE,
             ]
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
@@ -218,22 +214,13 @@ class ModelEndpointStatus(ObjectStatus):
 
 class ModelEndpoint(BaseModel):
     kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
-    metadata: ModelEndpointMetadata = ModelEndpointMetadata()
+    metadata: ModelEndpointMetadata
     spec: ModelEndpointSpec = ModelEndpointSpec()
     status: ModelEndpointStatus = ModelEndpointStatus()
 
     class Config:
         extra = Extra.allow
 
-    def __init__(self, **data: Any):
-        super().__init__(**data)
-        if self.metadata.uid is None:
-            uid = mlrun.common.model_monitoring.create_model_endpoint_uid(
-                function_uri=self.spec.function_uri,
-                versioned_model=self.spec.model,
-            )
-            self.metadata.uid = str(uid)
-
     def flat_dict(self):
         """Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
         the model endpoint object in the database.
@@ -274,7 +261,7 @@ class ModelEndpoint(BaseModel):
         return flatten_dict
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict):
+    def from_flat_dict(cls, endpoint_dict: dict) -> "ModelEndpoint":
         """Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
         is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
 
@@ -292,21 +279,82 @@ class ModelEndpointList(BaseModel):
     endpoints: list[ModelEndpoint] = []
 
 
+class ModelEndpointMonitoringMetric(BaseModel):
+    project: str
+    app: str
+    type: ModelEndpointMonitoringMetricType
+    name: str
+    full_name: str
+
+
+def _compose_full_name(
+    *,
+    project: str,
+    app: str,
+    name: str,
+    type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.RESULT,
+) -> str:
+    return ".".join([project, app, type, name])
+
+
+def _parse_metric_fqn_to_monitoring_metric(fqn: str) -> ModelEndpointMonitoringMetric:
+    match = FQN_REGEX.fullmatch(fqn)
+    if match is None:
+        raise ValueError("The fully qualified name is not in the expected format")
+    return ModelEndpointMonitoringMetric.parse_obj(
+        match.groupdict() | {"full_name": fqn}
+    )
+
+
+class _MetricPoint(NamedTuple):
+    timestamp: datetime
+    value: float
+
+
+class _ResultPoint(NamedTuple):
+    timestamp: datetime
+    value: float
+    status: ResultStatusApp
+
+
+class _ModelEndpointMonitoringMetricValuesBase(BaseModel):
+    full_name: str
+    type: ModelEndpointMonitoringMetricType
+    data: bool
+
+
+class ModelEndpointMonitoringMetricValues(_ModelEndpointMonitoringMetricValuesBase):
+    type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.METRIC
+    values: list[_MetricPoint]
+    data: bool = True
+
+
+class ModelEndpointMonitoringResultValues(_ModelEndpointMonitoringMetricValuesBase):
+    type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.RESULT
+    result_kind: ResultKindApp
+    values: list[_ResultPoint]
+    data: bool = True
+
+
+class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBase):
+    full_name: str
+    type: ModelEndpointMonitoringMetricType
+    data: bool = False
+
+
 def _mapping_attributes(
-    base_model: BaseModel,
-    flattened_dictionary: dict,
-    json_parse_values: list = None,
-):
+    model_class: type[Model], flattened_dictionary: dict, json_parse_values: list
+) -> Model:
     """Generate a `BaseModel` object with the provided dictionary attributes.
 
-    :param base_model:           `BaseModel` object (e.g. `ModelEndpointMetadata`).
+    :param model_class:          `BaseModel` class (e.g. `ModelEndpointMetadata`).
     :param flattened_dictionary: Flattened dictionary that contains the model endpoint attributes.
     :param json_parse_values:    List of dictionary keys with a JSON string value that will be parsed into a
                                  dictionary using json.loads().
     """
     # Get the fields of the provided base model object. These fields will be used to filter to relevent keys
     # from the flattened dictionary.
-    wanted_keys = base_model.__fields__.keys()
+    wanted_keys = model_class.__fields__.keys()
 
     # Generate a filtered flattened dictionary that will be parsed into the BaseModel object
     dict_to_parse = {}
@@ -320,7 +368,7 @@ def _mapping_attributes(
             else:
                 dict_to_parse[field_key] = flattened_dictionary[field_key]
 
-    return base_model.parse_obj(dict_to_parse)
+    return model_class.parse_obj(dict_to_parse)
 
 
 def _json_loads_if_not_none(field: Any) -> Any:

mlrun/common/schemas/notification.py

@@ -22,11 +22,48 @@ import mlrun.common.types
 
 
 class NotificationKind(mlrun.common.types.StrEnum):
-    console = "console"
-    git = "git"
-    ipython = "ipython"
-    slack = "slack"
-    webhook = "webhook"
+    """Currently, the supported notification kinds and their params are as follows:"""
+
+    console: str = "console"
+    """no params, local only"""
+
+    git: str = "git"
+    """
+    **token** - The git token to use for the git notification.\n
+    **repo** - The git repo to which to send the notification.\n
+    **issue** - The git issue to which to send the notification.\n
+    **merge_request** -
+     In GitLab (as opposed to GitHub), merge requests and issues are separate entities.
+     If using merge request, the issue will be ignored, and vice versa.\n
+    **server** - The git server to which to send the notification.\n
+    **gitlab** - (bool) Whether the git server is GitLab or not.\n
+    """
+
+    ipython: str = "ipython"
+    """no params, local only"""
+
+    slack: str = "slack"
+    """**webhook** - The slack webhook to which to send the notification."""
+
+    webhook: str = "webhook"
+    """
+    **url** - The webhook url to which to send the notification.\n
+    **method** - The http method to use when sending the notification (GET, POST, PUT, etc…).\n
+    **headers** - (dict) The http headers to send with the notification.\n
+    **override_body** -
+     (dict) The body to send with the notification. If not specified, the
+     default body will be a dictionary containing `name`, `message`, `severity`, and a `runs` list of the
+     completed runs. You can also add the run's details.\n
+     Example::
+
+                "override_body": {"message":"Run Completed {{ runs }}"
+                # Results would look like:
+                "message": "Run Completed [{'project': 'my-project', 'name': 'my-function', 'host': <run-host>,
+                         'status': {'state': 'completed', 'results': <run-results>}}]"
+    **verify_ssl** -
+     (bool) Whether SSL certificates are validated during HTTP requests or not.
+     The default is set to True.\n
+    """
 
 
 class NotificationSeverity(mlrun.common.types.StrEnum):
@@ -50,15 +87,35 @@ class NotificationLimits(enum.Enum):
 
 
 class Notification(pydantic.BaseModel):
+    """
+    Notification object schema
+
+    :param kind: notification implementation kind - slack, webhook, etc.
+    :param name: for logging and identification
+    :param message: message content in the notification
+    :param severity: severity to display in the notification
+    :param when: list of statuses to trigger the notification: 'running', 'completed', 'error'
+    :param condition: optional condition to trigger the notification, a jinja2 expression that can use run data
+                      to evaluate if the notification should be sent in addition to the 'when' statuses.
+                      e.g.: '{{ run["status"]["results"]["accuracy"] < 0.9}}'
+    :param params: Implementation specific parameters for the notification implementation (e.g. slack webhook url,
+                   git repository details, etc.)
+    :param secret_params: secret parameters for the notification implementation, same as params but will be stored
+                          in a k8s secret and passed as a secret reference to the implementation.
+    :param status: notification status - pending, sent, error
+    :param sent_time: time the notification was sent
+    :param reason: failure reason if the notification failed to send
+    """
+
     kind: NotificationKind
     name: str
-    message: str
-    severity: NotificationSeverity
-    when: list[str]
-    condition: str
-    params: dict[str, typing.Any] = None
-    status: NotificationStatus = None
-    sent_time: typing.Union[str, datetime.datetime] = None
+    message: typing.Optional[str] = None
+    severity: typing.Optional[NotificationSeverity] = None
+    when: typing.Optional[list[str]] = None
+    condition: typing.Optional[str] = None
+    params: typing.Optional[dict[str, typing.Any]] = None
+    status: typing.Optional[NotificationStatus] = None
+    sent_time: typing.Optional[typing.Union[str, datetime.datetime]] = None
     secret_params: typing.Optional[dict[str, typing.Any]] = None
     reason: typing.Optional[str] = None
 

mlrun/{runtimes/mpijob/v1alpha1.py → common/schemas/pagination.py}

@@ -11,19 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from deprecated import deprecated
 
-from mlrun.runtimes.constants import MPIJobCRDVersions
-from mlrun.runtimes.mpijob.abstract import AbstractMPIJobRuntime
+import typing
 
+import pydantic
 
-# TODO: Remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="v1alpha1 mpi will be removed in 1.7.0, use v1 instead",
-    category=FutureWarning,
-)
-class MpiRuntimeV1Alpha1(AbstractMPIJobRuntime):
-    crd_group = "kubeflow.org"
-    crd_version = MPIJobCRDVersions.v1alpha1
-    crd_plural = "mpijobs"
+
+class PaginationInfo(pydantic.BaseModel):
+    class Config:
+        allow_population_by_field_name = True
+
+    page: typing.Optional[int]
+    page_size: typing.Optional[int] = pydantic.Field(alias="page-size")
+    page_token: typing.Optional[str] = pydantic.Field(alias="page-token")

mlrun/common/schemas/pipeline.py

@@ -15,10 +15,17 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
 
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.PipelineFormat instead.",
+    category=FutureWarning,
+)
 class PipelinesFormat(mlrun.common.types.StrEnum):
     full = "full"
     metadata_only = "metadata_only"

mlrun/common/schemas/project.py

@@ -16,6 +16,7 @@ import datetime
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
@@ -23,6 +24,12 @@ from .common import ImageBuilder
 from .object import ObjectKind, ObjectStatus
 
 
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.ProjectsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.ProjectFormat instead.",
+    category=FutureWarning,
+)
 class ProjectsFormat(mlrun.common.types.StrEnum):
     full = "full"
     name_only = "name_only"
@@ -87,6 +94,30 @@ class ProjectSpec(pydantic.BaseModel):
     custom_packagers: typing.Optional[list[tuple[str, bool]]] = None
     default_image: typing.Optional[str] = None
     build: typing.Optional[ImageBuilder] = None
+    default_function_node_selector: typing.Optional[dict] = {}
+
+    class Config:
+        extra = pydantic.Extra.allow
+
+
+class ProjectSpecOut(pydantic.BaseModel):
+    description: typing.Optional[str] = None
+    owner: typing.Optional[str] = None
+    goals: typing.Optional[str] = None
+    params: typing.Optional[dict] = {}
+    functions: typing.Optional[list] = []
+    workflows: typing.Optional[list] = []
+    artifacts: typing.Optional[list] = []
+    artifact_path: typing.Optional[str] = None
+    conda: typing.Optional[str] = None
+    source: typing.Optional[str] = None
+    subpath: typing.Optional[str] = None
+    origin_url: typing.Optional[str] = None
+    desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
+    custom_packagers: typing.Optional[list[tuple[str, bool]]] = None
+    default_image: typing.Optional[str] = None
+    build: typing.Any = None
+    default_function_node_selector: typing.Optional[dict] = {}
 
     class Config:
         extra = pydantic.Extra.allow
@@ -99,6 +130,15 @@ class Project(pydantic.BaseModel):
     status: ObjectStatus = ObjectStatus()
 
 
+# The reason we have a different schema for the response model is that we don't want to validate project.spec.build in
+# the response as the validation was added late and there may be corrupted values in the DB.
+class ProjectOut(pydantic.BaseModel):
+    kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
+    metadata: ProjectMetadata
+    spec: ProjectSpecOut = ProjectSpecOut()
+    status: ObjectStatus = ObjectStatus()
+
+
 class ProjectOwner(pydantic.BaseModel):
     username: str
     access_key: str
@@ -106,30 +146,41 @@ class ProjectOwner(pydantic.BaseModel):
 
 class ProjectSummary(pydantic.BaseModel):
     name: str
-    files_count: int
-    feature_sets_count: int
-    models_count: int
-    runs_failed_recent_count: int
-    runs_running_count: int
-    schedules_count: int
+    files_count: int = 0
+    feature_sets_count: int = 0
+    models_count: int = 0
+    runs_completed_recent_count: int = 0
+    runs_failed_recent_count: int = 0
+    runs_running_count: int = 0
+    distinct_schedules_count: int = 0
+    distinct_scheduled_jobs_pending_count: int = 0
+    distinct_scheduled_pipelines_pending_count: int = 0
+    pipelines_completed_recent_count: typing.Optional[int] = None
+    pipelines_failed_recent_count: typing.Optional[int] = None
     pipelines_running_count: typing.Optional[int] = None
+    updated: typing.Optional[datetime.datetime] = None
 
 
 class IguazioProject(pydantic.BaseModel):
     data: dict
 
 
+# The format query param controls the project type used:
+# full - ProjectOut
+# name_only - str
+# summary - ProjectSummary
+# leader - currently only IguazioProject supported
+# The way pydantic handles typing.Union is that it takes the object and tries to coerce it to be the types of the
+# union by the definition order. Therefore, we can't currently add generic dict for all leader formats, but we need
+# to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
+# https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
+ProjectOutput = typing.TypeVar(
+    "ProjectOutput", ProjectOut, str, ProjectSummary, IguazioProject
+)
+
+
 class ProjectsOutput(pydantic.BaseModel):
-    # The format query param controls the project type used:
-    # full - Project
-    # name_only - str
-    # summary - ProjectSummary
-    # leader - currently only IguazioProject supported
-    # The way pydantic handles typing.Union is that it takes the object and tries to coerce it to be the types of the
-    # union by the definition order. Therefore we can't currently add generic dict for all leader formats, but we need
-    # to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
-    # https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
-    projects: list[typing.Union[Project, str, ProjectSummary, IguazioProject]]
+    projects: list[ProjectOutput]
 
 
 class ProjectSummariesOutput(pydantic.BaseModel):

mlrun/common/schemas/runs.py

@@ -15,9 +15,26 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
+
+import mlrun.common.types
 
 
 class RunIdentifier(pydantic.BaseModel):
     kind: typing.Literal["run"] = "run"
     uid: typing.Optional[str]
     iter: typing.Optional[int]
+
+
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.RunsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.RunFormat instead.",
+    category=FutureWarning,
+)
+class RunsFormat(mlrun.common.types.StrEnum):
+    # No enrichment, data is pulled as-is from the database.
+    standard = "standard"
+
+    # Performs run enrichment, including the run's artifacts. Only available for the `get` run API.
+    full = "full"

mlrun/common/schemas/schedule.py

@@ -96,7 +96,7 @@ class ScheduleUpdate(BaseModel):
     scheduled_object: Optional[Any]
     cron_trigger: Optional[Union[str, ScheduleCronTrigger]]
     desired_state: Optional[str]
-    labels: Optional[dict] = {}
+    labels: Optional[dict] = None
     concurrency_limit: Optional[int]
     credentials: Credentials = Credentials()
 

mlrun/common/schemas/workflow.py

@@ -16,8 +16,9 @@ import typing
 
 import pydantic
 
-from .notification import Notification
-from .schedule import ScheduleCronTrigger
+from mlrun.common.schemas.notification import Notification
+from mlrun.common.schemas.schedule import ScheduleCronTrigger
+from mlrun.common.types import StrEnum
 
 
 class WorkflowSpec(pydantic.BaseModel):
@@ -32,6 +33,7 @@ class WorkflowSpec(pydantic.BaseModel):
     schedule: typing.Union[str, ScheduleCronTrigger] = None
     run_local: typing.Optional[bool] = None
     image: typing.Optional[str] = None
+    workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
 
 
 class WorkflowRequest(pydantic.BaseModel):
@@ -54,3 +56,9 @@ class WorkflowResponse(pydantic.BaseModel):
 
 class GetWorkflowResponse(pydantic.BaseModel):
     workflow_id: str = None
+
+
+class EngineType(StrEnum):
+    LOCAL = "local"
+    REMOTE = "remote"
+    KFP = "kfp"

mlrun/common/types.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import enum
 
@@ -23,3 +22,17 @@ class StrEnum(str, enum.Enum):
 
     def __repr__(self):
         return self.value
+
+
+# Partial backport from Python 3.11
+# https://docs.python.org/3/library/http.html#http.HTTPMethod
+class HTTPMethod(StrEnum):
+    GET = "GET"
+    POST = "POST"
+    DELETE = "DELETE"
+    PATCH = "PATCH"
+
+
+class Operation(StrEnum):
+    ADD = "add"
+    REMOVE = "remove"