mlrun 1.6.4rc7__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/common/schemas/notification.py

@@ -22,11 +22,48 @@ import mlrun.common.types
 
 
 class NotificationKind(mlrun.common.types.StrEnum):
-    console = "console"
-    git = "git"
-    ipython = "ipython"
-    slack = "slack"
-    webhook = "webhook"
+    """Currently, the supported notification kinds and their params are as follows:"""
+
+    console: str = "console"
+    """no params, local only"""
+
+    git: str = "git"
+    """
+    **token** - The git token to use for the git notification.\n
+    **repo** - The git repo to which to send the notification.\n
+    **issue** - The git issue to which to send the notification.\n
+    **merge_request** -
+     In GitLab (as opposed to GitHub), merge requests and issues are separate entities.
+     If using merge request, the issue will be ignored, and vice versa.\n
+    **server** - The git server to which to send the notification.\n
+    **gitlab** - (bool) Whether the git server is GitLab or not.\n
+    """
+
+    ipython: str = "ipython"
+    """no params, local only"""
+
+    slack: str = "slack"
+    """**webhook** - The slack webhook to which to send the notification."""
+
+    webhook: str = "webhook"
+    """
+    **url** - The webhook url to which to send the notification.\n
+    **method** - The http method to use when sending the notification (GET, POST, PUT, etc…).\n
+    **headers** - (dict) The http headers to send with the notification.\n
+    **override_body** -
+     (dict) The body to send with the notification. If not specified, the
+     default body will be a dictionary containing `name`, `message`, `severity`, and a `runs` list of the
+     completed runs. You can also add the run's details.\n
+     Example::
+
+                "override_body": {"message":"Run Completed {{ runs }}"
+                # Results would look like:
+                "message": "Run Completed [{'project': 'my-project', 'name': 'my-function', 'host': <run-host>,
+                         'status': {'state': 'completed', 'results': <run-results>}}]"
+    **verify_ssl** -
+     (bool) Whether SSL certificates are validated during HTTP requests or not.
+     The default is set to True.\n
+    """
 
 
 class NotificationSeverity(mlrun.common.types.StrEnum):
@@ -50,18 +87,38 @@ class NotificationLimits(enum.Enum):
 
 
 class Notification(pydantic.BaseModel):
+    """
+    Notification object schema
+
+    :param kind: notification implementation kind - slack, webhook, etc.
+    :param name: for logging and identification
+    :param message: message content in the notification
+    :param severity: severity to display in the notification
+    :param when: list of statuses to trigger the notification: 'running', 'completed', 'error'
+    :param condition: optional condition to trigger the notification, a jinja2 expression that can use run data
+                      to evaluate if the notification should be sent in addition to the 'when' statuses.
+                      e.g.: '{{ run["status"]["results"]["accuracy"] < 0.9}}'
+    :param params: Implementation specific parameters for the notification implementation (e.g. slack webhook url,
+                   git repository details, etc.)
+    :param secret_params: secret parameters for the notification implementation, same as params but will be stored
+                          in a k8s secret and passed as a secret reference to the implementation.
+    :param status: notification status - pending, sent, error
+    :param sent_time: time the notification was sent
+    :param reason: failure reason if the notification failed to send
+    """
+
     kind: NotificationKind
     name: str
-    message: str
-    severity: NotificationSeverity
-    when: typing.List[str]
-    condition: str
-    params: typing.Dict[str, typing.Any] = None
-    status: NotificationStatus = None
-    sent_time: typing.Union[str, datetime.datetime] = None
-    secret_params: typing.Optional[typing.Dict[str, typing.Any]] = None
+    message: typing.Optional[str] = None
+    severity: typing.Optional[NotificationSeverity] = None
+    when: typing.Optional[list[str]] = None
+    condition: typing.Optional[str] = None
+    params: typing.Optional[dict[str, typing.Any]] = None
+    status: typing.Optional[NotificationStatus] = None
+    sent_time: typing.Optional[typing.Union[str, datetime.datetime]] = None
+    secret_params: typing.Optional[dict[str, typing.Any]] = None
     reason: typing.Optional[str] = None
 
 
 class SetNotificationRequest(pydantic.BaseModel):
-    notifications: typing.List[Notification] = None
+    notifications: list[Notification] = None

mlrun/common/schemas/object.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 from datetime import datetime
-from typing import List, Optional
+from typing import Optional
 
 from pydantic import BaseModel, Extra
 
@@ -60,7 +60,7 @@ class ObjectRecord(BaseModel):
     project: str
     uid: str
     updated: Optional[datetime] = None
-    labels: List[LabelRecord]
+    labels: list[LabelRecord]
     # state is extracted from the full status dict to enable queries
     state: Optional[str] = None
     full_object: Optional[dict] = None

mlrun/{model_monitoring/controller_handler.py → common/schemas/pagination.py}

@@ -12,18 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mlrun
-from mlrun.model_monitoring.controller import MonitoringApplicationController
+import typing
 
+import pydantic
 
-def handler(context: mlrun.run.MLClientCtx) -> None:
-    """
-    Run model monitoring application processor
 
-    :param context: the MLRun context
-    """
-    monitor_app_controller = MonitoringApplicationController(
-        context=context,
-        project=context.project,
-    )
-    monitor_app_controller.run()
+class PaginationInfo(pydantic.BaseModel):
+    class Config:
+        allow_population_by_field_name = True
+
+    page: typing.Optional[int]
+    page_size: typing.Optional[int] = pydantic.Field(alias="page-size")
+    page_token: typing.Optional[str] = pydantic.Field(alias="page-token")

mlrun/common/schemas/pipeline.py

@@ -15,10 +15,17 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
 
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.PipelineFormat instead.",
+    category=FutureWarning,
+)
 class PipelinesFormat(mlrun.common.types.StrEnum):
     full = "full"
     metadata_only = "metadata_only"
@@ -34,6 +41,6 @@ class PipelinesPagination(str):
 
 class PipelinesOutput(pydantic.BaseModel):
     # use the format query param to control what is returned
-    runs: typing.List[typing.Union[dict, str]]
+    runs: list[typing.Union[dict, str]]
     total_size: int
     next_page_token: typing.Optional[str]

mlrun/common/schemas/project.py

@@ -16,6 +16,7 @@ import datetime
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
@@ -23,6 +24,12 @@ from .common import ImageBuilder
 from .object import ObjectKind, ObjectStatus
 
 
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.ProjectsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.ProjectFormat instead.",
+    category=FutureWarning,
+)
 class ProjectsFormat(mlrun.common.types.StrEnum):
     full = "full"
     name_only = "name_only"
@@ -84,9 +91,33 @@ class ProjectSpec(pydantic.BaseModel):
     subpath: typing.Optional[str] = None
     origin_url: typing.Optional[str] = None
     desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
-    custom_packagers: typing.Optional[typing.List[typing.Tuple[str, bool]]] = None
+    custom_packagers: typing.Optional[list[tuple[str, bool]]] = None
     default_image: typing.Optional[str] = None
     build: typing.Optional[ImageBuilder] = None
+    default_function_node_selector: typing.Optional[dict] = {}
+
+    class Config:
+        extra = pydantic.Extra.allow
+
+
+class ProjectSpecOut(pydantic.BaseModel):
+    description: typing.Optional[str] = None
+    owner: typing.Optional[str] = None
+    goals: typing.Optional[str] = None
+    params: typing.Optional[dict] = {}
+    functions: typing.Optional[list] = []
+    workflows: typing.Optional[list] = []
+    artifacts: typing.Optional[list] = []
+    artifact_path: typing.Optional[str] = None
+    conda: typing.Optional[str] = None
+    source: typing.Optional[str] = None
+    subpath: typing.Optional[str] = None
+    origin_url: typing.Optional[str] = None
+    desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
+    custom_packagers: typing.Optional[list[tuple[str, bool]]] = None
+    default_image: typing.Optional[str] = None
+    build: typing.Any = None
+    default_function_node_selector: typing.Optional[dict] = {}
 
     class Config:
         extra = pydantic.Extra.allow
@@ -99,6 +130,15 @@ class Project(pydantic.BaseModel):
     status: ObjectStatus = ObjectStatus()
 
 
+# The reason we have a different schema for the response model is that we don't want to validate project.spec.build in
+# the response as the validation was added late and there may be corrupted values in the DB.
+class ProjectOut(pydantic.BaseModel):
+    kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
+    metadata: ProjectMetadata
+    spec: ProjectSpecOut = ProjectSpecOut()
+    status: ObjectStatus = ObjectStatus()
+
+
 class ProjectOwner(pydantic.BaseModel):
     username: str
     access_key: str
@@ -106,31 +146,42 @@ class ProjectOwner(pydantic.BaseModel):
 
 class ProjectSummary(pydantic.BaseModel):
     name: str
-    files_count: int
-    feature_sets_count: int
-    models_count: int
-    runs_failed_recent_count: int
-    runs_running_count: int
-    schedules_count: int
+    files_count: int = 0
+    feature_sets_count: int = 0
+    models_count: int = 0
+    runs_completed_recent_count: int = 0
+    runs_failed_recent_count: int = 0
+    runs_running_count: int = 0
+    distinct_schedules_count: int = 0
+    distinct_scheduled_jobs_pending_count: int = 0
+    distinct_scheduled_pipelines_pending_count: int = 0
+    pipelines_completed_recent_count: typing.Optional[int] = None
+    pipelines_failed_recent_count: typing.Optional[int] = None
     pipelines_running_count: typing.Optional[int] = None
+    updated: typing.Optional[datetime.datetime] = None
 
 
 class IguazioProject(pydantic.BaseModel):
     data: dict
 
 
+# The format query param controls the project type used:
+# full - ProjectOut
+# name_only - str
+# summary - ProjectSummary
+# leader - currently only IguazioProject supported
+# The way pydantic handles typing.Union is that it takes the object and tries to coerce it to be the types of the
+# union by the definition order. Therefore, we can't currently add generic dict for all leader formats, but we need
+# to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
+# https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
+ProjectOutput = typing.TypeVar(
+    "ProjectOutput", ProjectOut, str, ProjectSummary, IguazioProject
+)
+
+
 class ProjectsOutput(pydantic.BaseModel):
-    # The format query param controls the project type used:
-    # full - Project
-    # name_only - str
-    # summary - ProjectSummary
-    # leader - currently only IguazioProject supported
-    # The way pydantic handles typing.Union is that it takes the object and tries to coerce it to be the types of the
-    # union by the definition order. Therefore we can't currently add generic dict for all leader formats, but we need
-    # to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
-    # https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
-    projects: typing.List[typing.Union[Project, str, ProjectSummary, IguazioProject]]
+    projects: list[ProjectOutput]
 
 
 class ProjectSummariesOutput(pydantic.BaseModel):
-    project_summaries: typing.List[ProjectSummary]
+    project_summaries: list[ProjectSummary]

mlrun/common/schemas/runs.py

@@ -15,6 +15,7 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
@@ -25,7 +26,12 @@ class RunIdentifier(pydantic.BaseModel):
     iter: typing.Optional[int]
 
 
-# In 1.7 should be moved to mlrun.common.formatters.run.py
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.RunsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.RunFormat instead.",
+    category=FutureWarning,
+)
 class RunsFormat(mlrun.common.types.StrEnum):
     # No enrichment, data is pulled as-is from the database.
     standard = "standard"

mlrun/common/schemas/runtime_resource.py

@@ -26,15 +26,15 @@ class ListRuntimeResourcesGroupByField(mlrun.common.types.StrEnum):
 
 class RuntimeResource(pydantic.BaseModel):
     name: str
-    labels: typing.Dict[str, str] = {}
-    status: typing.Optional[typing.Dict]
+    labels: dict[str, str] = {}
+    status: typing.Optional[dict]
 
 
 class RuntimeResources(pydantic.BaseModel):
-    crd_resources: typing.List[RuntimeResource] = []
-    pod_resources: typing.List[RuntimeResource] = []
+    crd_resources: list[RuntimeResource] = []
+    pod_resources: list[RuntimeResource] = []
     # only for dask runtime
-    service_resources: typing.Optional[typing.List[RuntimeResource]] = None
+    service_resources: typing.Optional[list[RuntimeResource]] = None
 
     class Config:
         extra = pydantic.Extra.allow
@@ -45,14 +45,10 @@ class KindRuntimeResources(pydantic.BaseModel):
     resources: RuntimeResources
 
 
-RuntimeResourcesOutput = typing.List[KindRuntimeResources]
+RuntimeResourcesOutput = list[KindRuntimeResources]
 
 
 # project name -> job uid -> runtime resources
-GroupedByJobRuntimeResourcesOutput = typing.Dict[
-    str, typing.Dict[str, RuntimeResources]
-]
+GroupedByJobRuntimeResourcesOutput = dict[str, dict[str, RuntimeResources]]
 # project name -> kind -> runtime resources
-GroupedByProjectRuntimeResourcesOutput = typing.Dict[
-    str, typing.Dict[str, RuntimeResources]
-]
+GroupedByProjectRuntimeResourcesOutput = dict[str, dict[str, RuntimeResources]]

mlrun/common/schemas/schedule.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 from datetime import datetime
-from typing import Any, List, Literal, Optional, Union
+from typing import Any, Literal, Optional, Union
 
 from pydantic import BaseModel
 
@@ -96,7 +96,7 @@ class ScheduleUpdate(BaseModel):
     scheduled_object: Optional[Any]
     cron_trigger: Optional[Union[str, ScheduleCronTrigger]]
     desired_state: Optional[str]
-    labels: Optional[dict] = {}
+    labels: Optional[dict] = None
     concurrency_limit: Optional[int]
     credentials: Credentials = Credentials()
 
@@ -119,7 +119,7 @@ class ScheduleRecord(ScheduleInput):
     project: str
     last_run_uri: Optional[str]
     state: Optional[str]
-    labels: Optional[List[LabelRecord]]
+    labels: Optional[list[LabelRecord]]
     next_run_time: Optional[datetime]
 
     class Config:
@@ -135,7 +135,7 @@ class ScheduleOutput(ScheduleRecord):
 
 
 class SchedulesOutput(BaseModel):
-    schedules: List[ScheduleOutput]
+    schedules: list[ScheduleOutput]
 
 
 class ScheduleIdentifier(BaseModel):

mlrun/common/schemas/tag.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import typing
 
 import pydantic
 
@@ -29,4 +28,4 @@ class TagObjects(pydantic.BaseModel):
 
     kind: str
     # TODO: Add more types to the list for new supported tagged objects
-    identifiers: typing.List[ArtifactIdentifier]
+    identifiers: list[ArtifactIdentifier]

mlrun/common/schemas/workflow.py

@@ -16,8 +16,9 @@ import typing
 
 import pydantic
 
-from .notification import Notification
-from .schedule import ScheduleCronTrigger
+from mlrun.common.schemas.notification import Notification
+from mlrun.common.schemas.schedule import ScheduleCronTrigger
+from mlrun.common.types import StrEnum
 
 
 class WorkflowSpec(pydantic.BaseModel):
@@ -32,16 +33,17 @@ class WorkflowSpec(pydantic.BaseModel):
     schedule: typing.Union[str, ScheduleCronTrigger] = None
     run_local: typing.Optional[bool] = None
     image: typing.Optional[str] = None
+    workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
 
 
 class WorkflowRequest(pydantic.BaseModel):
     spec: typing.Optional[WorkflowSpec] = None
-    arguments: typing.Optional[typing.Dict] = None
+    arguments: typing.Optional[dict] = None
     artifact_path: typing.Optional[str] = None
     source: typing.Optional[str] = None
     run_name: typing.Optional[str] = None
     namespace: typing.Optional[str] = None
-    notifications: typing.Optional[typing.List[Notification]] = None
+    notifications: typing.Optional[list[Notification]] = None
 
 
 class WorkflowResponse(pydantic.BaseModel):
@@ -54,3 +56,9 @@ class WorkflowResponse(pydantic.BaseModel):
 
 class GetWorkflowResponse(pydantic.BaseModel):
     workflow_id: str = None
+
+
+class EngineType(StrEnum):
+    LOCAL = "local"
+    REMOTE = "remote"
+    KFP = "kfp"

mlrun/common/types.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import enum
 
@@ -23,3 +22,17 @@ class StrEnum(str, enum.Enum):
 
     def __repr__(self):
         return self.value
+
+
+# Partial backport from Python 3.11
+# https://docs.python.org/3/library/http.html#http.HTTPMethod
+class HTTPMethod(StrEnum):
+    GET = "GET"
+    POST = "POST"
+    DELETE = "DELETE"
+    PATCH = "PATCH"
+
+
+class Operation(StrEnum):
+    ADD = "add"
+    REMOVE = "remove"