mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -14,27 +14,32 @@
 
 import enum
 import json
-import re
 from datetime import datetime
-from typing import Any, NamedTuple, Optional
+from typing import Any, NamedTuple, Optional, TypeVar
 
-from pydantic import BaseModel, Field, validator
-from pydantic.main import Extra
+from pydantic import BaseModel, Extra, Field, constr, validator
 
+# TODO: remove the unused import below after `mlrun.datastore` and `mlrun.utils` usage is removed.
+# At the moment `make lint` fails if this is removed.
 import mlrun.common.model_monitoring
-import mlrun.common.types
 
 from ..object import ObjectKind, ObjectSpec, ObjectStatus
 from .constants import (
+    FQN_REGEX,
+    MODEL_ENDPOINT_ID_PATTERN,
+    PROJECT_PATTERN,
     EndpointType,
     EventFieldType,
     EventKeyMetrics,
     EventLiveStats,
+    ModelEndpointMonitoringMetricType,
     ModelMonitoringMode,
     ResultKindApp,
     ResultStatusApp,
 )
 
+Model = TypeVar("Model", bound=BaseModel)
+
 
 class ModelMonitoringStoreKinds:
     # TODO: do changes in examples & demos In 1.5.0 remove
@@ -43,9 +48,9 @@ class ModelMonitoringStoreKinds:
 
 
 class ModelEndpointMetadata(BaseModel):
-    project: Optional[str] = ""
+    project: constr(regex=PROJECT_PATTERN)
+    uid: constr(regex=MODEL_ENDPOINT_ID_PATTERN)
     labels: Optional[dict] = {}
-    uid: Optional[str] = ""
 
     class Config:
         extra = Extra.allow
@@ -58,12 +63,11 @@ class ModelEndpointMetadata(BaseModel):
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [EventFieldType.LABELS]
 
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
@@ -90,7 +94,6 @@ class ModelEndpointSpec(ObjectSpec):
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [
                 EventFieldType.FEATURE_NAMES,
@@ -98,7 +101,7 @@ class ModelEndpointSpec(ObjectSpec):
                 EventFieldType.MONITOR_CONFIGURATION,
             ]
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
@@ -192,7 +195,6 @@ class ModelEndpointStatus(ObjectStatus):
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [
                 EventFieldType.FEATURE_STATS,
@@ -204,7 +206,7 @@ class ModelEndpointStatus(ObjectStatus):
                 EventFieldType.ENDPOINT_TYPE,
             ]
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
@@ -212,22 +214,13 @@ class ModelEndpointStatus(ObjectStatus):
 
 class ModelEndpoint(BaseModel):
     kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
-    metadata: ModelEndpointMetadata = ModelEndpointMetadata()
+    metadata: ModelEndpointMetadata
     spec: ModelEndpointSpec = ModelEndpointSpec()
     status: ModelEndpointStatus = ModelEndpointStatus()
 
     class Config:
         extra = Extra.allow
 
-    def __init__(self, **data: Any):
-        super().__init__(**data)
-        if self.metadata.uid is None:
-            uid = mlrun.common.model_monitoring.create_model_endpoint_uid(
-                function_uri=self.spec.function_uri,
-                versioned_model=self.spec.model,
-            )
-            self.metadata.uid = str(uid)
-
     def flat_dict(self):
         """Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
         the model endpoint object in the database.
@@ -268,7 +261,7 @@ class ModelEndpoint(BaseModel):
         return flatten_dict
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict):
+    def from_flat_dict(cls, endpoint_dict: dict) -> "ModelEndpoint":
         """Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
         is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
 
@@ -286,11 +279,6 @@ class ModelEndpointList(BaseModel):
     endpoints: list[ModelEndpoint] = []
 
 
-class ModelEndpointMonitoringMetricType(mlrun.common.types.StrEnum):
-    RESULT = "result"
-    METRIC = "metric"
-
-
 class ModelEndpointMonitoringMetric(BaseModel):
     project: str
     app: str
@@ -309,18 +297,8 @@ def _compose_full_name(
     return ".".join([project, app, type, name])
 
 
-_FQN_PART_PATTERN = r"[a-zA-Z0-9_-]+"
-_FQN_PATTERN = (
-    rf"^(?P<project>{_FQN_PART_PATTERN})\."
-    rf"(?P<app>{_FQN_PART_PATTERN})\."
-    rf"(?P<type>{ModelEndpointMonitoringMetricType.RESULT}|{ModelEndpointMonitoringMetricType.METRIC})\."
-    rf"(?P<name>{_FQN_PART_PATTERN})$"
-)
-_FQN_REGEX = re.compile(_FQN_PATTERN)
-
-
 def _parse_metric_fqn_to_monitoring_metric(fqn: str) -> ModelEndpointMonitoringMetric:
-    match = _FQN_REGEX.fullmatch(fqn)
+    match = FQN_REGEX.fullmatch(fqn)
     if match is None:
         raise ValueError("The fully qualified name is not in the expected format")
     return ModelEndpointMonitoringMetric.parse_obj(
@@ -365,20 +343,18 @@ class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBa
 
 
 def _mapping_attributes(
-    base_model: BaseModel,
-    flattened_dictionary: dict,
-    json_parse_values: list = None,
-):
+    model_class: type[Model], flattened_dictionary: dict, json_parse_values: list
+) -> Model:
     """Generate a `BaseModel` object with the provided dictionary attributes.
 
-    :param base_model:           `BaseModel` object (e.g. `ModelEndpointMetadata`).
+    :param model_class:          `BaseModel` class (e.g. `ModelEndpointMetadata`).
     :param flattened_dictionary: Flattened dictionary that contains the model endpoint attributes.
     :param json_parse_values:    List of dictionary keys with a JSON string value that will be parsed into a
                                  dictionary using json.loads().
     """
     # Get the fields of the provided base model object. These fields will be used to filter to relevent keys
     # from the flattened dictionary.
-    wanted_keys = base_model.__fields__.keys()
+    wanted_keys = model_class.__fields__.keys()
 
     # Generate a filtered flattened dictionary that will be parsed into the BaseModel object
     dict_to_parse = {}
@@ -392,7 +368,7 @@ def _mapping_attributes(
             else:
                 dict_to_parse[field_key] = flattened_dictionary[field_key]
 
-    return base_model.parse_obj(dict_to_parse)
+    return model_class.parse_obj(dict_to_parse)
 
 
 def _json_loads_if_not_none(field: Any) -> Any:

mlrun/common/schemas/notification.py

@@ -22,11 +22,48 @@ import mlrun.common.types
 
 
 class NotificationKind(mlrun.common.types.StrEnum):
-    console = "console"
-    git = "git"
-    ipython = "ipython"
-    slack = "slack"
-    webhook = "webhook"
+    """Currently, the supported notification kinds and their params are as follows:"""
+
+    console: str = "console"
+    """no params, local only"""
+
+    git: str = "git"
+    """
+    **token** - The git token to use for the git notification.\n
+    **repo** - The git repo to which to send the notification.\n
+    **issue** - The git issue to which to send the notification.\n
+    **merge_request** -
+     In GitLab (as opposed to GitHub), merge requests and issues are separate entities.
+     If using merge request, the issue will be ignored, and vice versa.\n
+    **server** - The git server to which to send the notification.\n
+    **gitlab** - (bool) Whether the git server is GitLab or not.\n
+    """
+
+    ipython: str = "ipython"
+    """no params, local only"""
+
+    slack: str = "slack"
+    """**webhook** - The slack webhook to which to send the notification."""
+
+    webhook: str = "webhook"
+    """
+    **url** - The webhook url to which to send the notification.\n
+    **method** - The http method to use when sending the notification (GET, POST, PUT, etc…).\n
+    **headers** - (dict) The http headers to send with the notification.\n
+    **override_body** -
+     (dict) The body to send with the notification. If not specified, the
+     default body will be a dictionary containing `name`, `message`, `severity`, and a `runs` list of the
+     completed runs. You can also add the run's details.\n
+     Example::
+
+                "override_body": {"message":"Run Completed {{ runs }}"
+                # Results would look like:
+                "message": "Run Completed [{'project': 'my-project', 'name': 'my-function', 'host': <run-host>,
+                         'status': {'state': 'completed', 'results': <run-results>}}]"
+    **verify_ssl** -
+     (bool) Whether SSL certificates are validated during HTTP requests or not.
+     The default is set to True.\n
+    """
 
 
 class NotificationSeverity(mlrun.common.types.StrEnum):
@@ -50,15 +87,35 @@ class NotificationLimits(enum.Enum):
 
 
 class Notification(pydantic.BaseModel):
+    """
+    Notification object schema
+
+    :param kind: notification implementation kind - slack, webhook, etc.
+    :param name: for logging and identification
+    :param message: message content in the notification
+    :param severity: severity to display in the notification
+    :param when: list of statuses to trigger the notification: 'running', 'completed', 'error'
+    :param condition: optional condition to trigger the notification, a jinja2 expression that can use run data
+                      to evaluate if the notification should be sent in addition to the 'when' statuses.
+                      e.g.: '{{ run["status"]["results"]["accuracy"] < 0.9}}'
+    :param params: Implementation specific parameters for the notification implementation (e.g. slack webhook url,
+                   git repository details, etc.)
+    :param secret_params: secret parameters for the notification implementation, same as params but will be stored
+                          in a k8s secret and passed as a secret reference to the implementation.
+    :param status: notification status - pending, sent, error
+    :param sent_time: time the notification was sent
+    :param reason: failure reason if the notification failed to send
+    """
+
     kind: NotificationKind
     name: str
-    message: str
-    severity: NotificationSeverity
-    when: list[str]
-    condition: str
-    params: dict[str, typing.Any] = None
-    status: NotificationStatus = None
-    sent_time: typing.Union[str, datetime.datetime] = None
+    message: typing.Optional[str] = None
+    severity: typing.Optional[NotificationSeverity] = None
+    when: typing.Optional[list[str]] = None
+    condition: typing.Optional[str] = None
+    params: typing.Optional[dict[str, typing.Any]] = None
+    status: typing.Optional[NotificationStatus] = None
+    sent_time: typing.Optional[typing.Union[str, datetime.datetime]] = None
     secret_params: typing.Optional[dict[str, typing.Any]] = None
     reason: typing.Optional[str] = None
 

mlrun/common/schemas/project.py

@@ -100,6 +100,29 @@ class ProjectSpec(pydantic.BaseModel):
         extra = pydantic.Extra.allow
 
 
+class ProjectSpecOut(pydantic.BaseModel):
+    description: typing.Optional[str] = None
+    owner: typing.Optional[str] = None
+    goals: typing.Optional[str] = None
+    params: typing.Optional[dict] = {}
+    functions: typing.Optional[list] = []
+    workflows: typing.Optional[list] = []
+    artifacts: typing.Optional[list] = []
+    artifact_path: typing.Optional[str] = None
+    conda: typing.Optional[str] = None
+    source: typing.Optional[str] = None
+    subpath: typing.Optional[str] = None
+    origin_url: typing.Optional[str] = None
+    desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
+    custom_packagers: typing.Optional[list[tuple[str, bool]]] = None
+    default_image: typing.Optional[str] = None
+    build: typing.Any = None
+    default_function_node_selector: typing.Optional[dict] = {}
+
+    class Config:
+        extra = pydantic.Extra.allow
+
+
 class Project(pydantic.BaseModel):
     kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
     metadata: ProjectMetadata
@@ -107,6 +130,15 @@ class Project(pydantic.BaseModel):
     status: ObjectStatus = ObjectStatus()
 
 
+# The reason we have a different schema for the response model is that we don't want to validate project.spec.build in
+# the response as the validation was added late and there may be corrupted values in the DB.
+class ProjectOut(pydantic.BaseModel):
+    kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
+    metadata: ProjectMetadata
+    spec: ProjectSpecOut = ProjectSpecOut()
+    status: ObjectStatus = ObjectStatus()
+
+
 class ProjectOwner(pydantic.BaseModel):
     username: str
     access_key: str
@@ -114,18 +146,19 @@ class ProjectOwner(pydantic.BaseModel):
 
 class ProjectSummary(pydantic.BaseModel):
     name: str
-    files_count: int
-    feature_sets_count: int
-    models_count: int
-    runs_completed_recent_count: int
-    runs_failed_recent_count: int
-    runs_running_count: int
-    distinct_schedules_count: int
-    distinct_scheduled_jobs_pending_count: int
-    distinct_scheduled_pipelines_pending_count: int
+    files_count: int = 0
+    feature_sets_count: int = 0
+    models_count: int = 0
+    runs_completed_recent_count: int = 0
+    runs_failed_recent_count: int = 0
+    runs_running_count: int = 0
+    distinct_schedules_count: int = 0
+    distinct_scheduled_jobs_pending_count: int = 0
+    distinct_scheduled_pipelines_pending_count: int = 0
     pipelines_completed_recent_count: typing.Optional[int] = None
     pipelines_failed_recent_count: typing.Optional[int] = None
     pipelines_running_count: typing.Optional[int] = None
+    updated: typing.Optional[datetime.datetime] = None
 
 
 class IguazioProject(pydantic.BaseModel):
@@ -133,16 +166,16 @@ class IguazioProject(pydantic.BaseModel):
 
 
 # The format query param controls the project type used:
-# full - Project
+# full - ProjectOut
 # name_only - str
 # summary - ProjectSummary
 # leader - currently only IguazioProject supported
 # The way pydantic handles typing.Union is that it takes the object and tries to coerce it to be the types of the
-# union by the definition order. Therefore we can't currently add generic dict for all leader formats, but we need
+# union by the definition order. Therefore, we can't currently add generic dict for all leader formats, but we need
 # to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
 # https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
 ProjectOutput = typing.TypeVar(
-    "ProjectOutput", Project, str, ProjectSummary, IguazioProject
+    "ProjectOutput", ProjectOut, str, ProjectSummary, IguazioProject
 )
 
 

mlrun/common/schemas/workflow.py

@@ -16,8 +16,9 @@ import typing
 
 import pydantic
 
-from .notification import Notification
-from .schedule import ScheduleCronTrigger
+from mlrun.common.schemas.notification import Notification
+from mlrun.common.schemas.schedule import ScheduleCronTrigger
+from mlrun.common.types import StrEnum
 
 
 class WorkflowSpec(pydantic.BaseModel):
@@ -32,6 +33,7 @@ class WorkflowSpec(pydantic.BaseModel):
     schedule: typing.Union[str, ScheduleCronTrigger] = None
     run_local: typing.Optional[bool] = None
     image: typing.Optional[str] = None
+    workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
 
 
 class WorkflowRequest(pydantic.BaseModel):
@@ -54,3 +56,9 @@ class WorkflowResponse(pydantic.BaseModel):
 
 class GetWorkflowResponse(pydantic.BaseModel):
     workflow_id: str = None
+
+
+class EngineType(StrEnum):
+    LOCAL = "local"
+    REMOTE = "remote"
+    KFP = "kfp"

mlrun/common/types.py

@@ -30,6 +30,7 @@ class HTTPMethod(StrEnum):
     GET = "GET"
     POST = "POST"
     DELETE = "DELETE"
+    PATCH = "PATCH"
 
 
 class Operation(StrEnum):