mlrun 1.8.0rc1__py3-none-any.whl → 1.8.0rc2__py3-none-any.whl

mlrun/common/schemas/memory_reports.py

@@ -15,14 +15,14 @@
 
 import typing
 
-import pydantic
+import pydantic.v1
 
 
-class MostCommonObjectTypesReport(pydantic.BaseModel):
+class MostCommonObjectTypesReport(pydantic.v1.BaseModel):
     object_types: list[tuple[str, int]]
 
 
-class ObjectTypeReport(pydantic.BaseModel):
+class ObjectTypeReport(pydantic.v1.BaseModel):
     object_type: str
     sample_size: int
     start_index: typing.Optional[int]

mlrun/common/schemas/model_monitoring/grafana.py

@@ -14,7 +14,7 @@
 
 from typing import Optional, Union
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 import mlrun.common.types
 

mlrun/common/schemas/model_monitoring/model_endpoint_v2.py

@@ -15,7 +15,7 @@
 import abc
 from typing import Optional
 
-from pydantic import BaseModel, Field, constr
+from pydantic.v1 import BaseModel, Field, constr
 
 from ..object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
 from .constants import (

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -17,7 +17,7 @@ import json
 from datetime import datetime
 from typing import Any, NamedTuple, Optional, TypeVar
 
-from pydantic import BaseModel, Extra, Field, constr, validator
+from pydantic.v1 import BaseModel, Extra, Field, constr, validator
 
 # TODO: remove the unused import below after `mlrun.datastore` and `mlrun.utils` usage is removed.
 # At the moment `make lint` fails if this is removed.

mlrun/common/schemas/notification.py

@@ -15,8 +15,9 @@
 import datetime
 import enum
 import typing
+from typing import Optional
 
-import pydantic
+import pydantic.v1
 
 import mlrun.common.types
 
@@ -45,6 +46,13 @@ class NotificationKind(mlrun.common.types.StrEnum):
     slack: str = "slack"
     """**webhook** - The slack webhook to which to send the notification."""
 
+    mail: str = "mail"
+    """
+    **email_addresses** - The target mails\n
+    **subject** - The subject of the mail\n
+    **body** - The body of the mail\n
+    """
+
     webhook: str = "webhook"
     """
     **url** - The webhook url to which to send the notification.\n
@@ -86,7 +94,7 @@ class NotificationLimits(enum.Enum):
     )  # 900KB (k8s secret size limit is 1MB minus buffer for metadata)
 
 
-class Notification(pydantic.BaseModel):
+class Notification(pydantic.v1.BaseModel):
     """
     Notification object schema
 
@@ -120,5 +128,12 @@ class Notification(pydantic.BaseModel):
     reason: typing.Optional[str] = None
 
 
-class SetNotificationRequest(pydantic.BaseModel):
+class SetNotificationRequest(pydantic.v1.BaseModel):
     notifications: list[Notification] = None
+
+
+class NotificationState(pydantic.v1.BaseModel):
+    kind: str
+    err: Optional[
+        str
+    ]  # empty error means that the notifications were sent successfully

mlrun/common/schemas/object.py

@@ -15,7 +15,7 @@
 from datetime import datetime
 from typing import Optional
 
-from pydantic import BaseModel, Extra
+from pydantic.v1 import BaseModel, Extra
 
 import mlrun.common.types
 

mlrun/common/schemas/pagination.py

@@ -14,13 +14,13 @@
 
 import typing
 
-import pydantic
+import pydantic.v1
 
 
-class PaginationInfo(pydantic.BaseModel):
+class PaginationInfo(pydantic.v1.BaseModel):
     class Config:
         allow_population_by_field_name = True
 
     page: typing.Optional[int]
-    page_size: typing.Optional[int] = pydantic.Field(alias="page-size")
-    page_token: typing.Optional[str] = pydantic.Field(alias="page-token")
+    page_size: typing.Optional[int] = pydantic.v1.Field(alias="page-size")
+    page_token: typing.Optional[str] = pydantic.v1.Field(alias="page-token")

mlrun/common/schemas/partition.py

@@ -30,6 +30,21 @@ class PartitionInterval(StrEnum):
     def valid_intervals(cls) -> list:
         return list(cls._value2member_map_.keys())
 
+    def as_duration(self) -> timedelta:
+        """
+        Convert the partition interval to a duration-like timedelta.
+
+        Returns:
+            timedelta: A duration representing the partition interval.
+        """
+        if self == PartitionInterval.DAY:
+            return timedelta(days=1)
+        elif self == PartitionInterval.MONTH:
+            # Approximate a month as 30 days
+            return timedelta(days=30)
+        elif self == PartitionInterval.YEARWEEK:
+            return timedelta(weeks=1)
+
     @classmethod
     def from_function(cls, partition_function: str):
         """
@@ -48,7 +63,7 @@ class PartitionInterval(StrEnum):
         interval = partition_function_to_partitions_interval.get(partition_function)
         if interval and cls.is_valid(interval):
             return cls[interval]
-        return None
+        raise KeyError(f"Partition function: {partition_function} isn't supported")
 
     def get_partition_info(
         self,

mlrun/common/schemas/pipeline.py

@@ -14,7 +14,7 @@
 #
 import typing
 
-import pydantic
+import pydantic.v1
 from deprecated import deprecated
 
 import mlrun.common.types
@@ -39,7 +39,7 @@ class PipelinesPagination(str):
     max_page_size = 200
 
 
-class PipelinesOutput(pydantic.BaseModel):
+class PipelinesOutput(pydantic.v1.BaseModel):
     # use the format query param to control what is returned
     runs: list[typing.Union[dict, str]]
     total_size: int

mlrun/common/schemas/project.py

@@ -15,7 +15,7 @@
 import datetime
 import typing
 
-import pydantic
+import pydantic.v1
 from deprecated import deprecated
 
 import mlrun.common.types
@@ -40,14 +40,14 @@ class ProjectsFormat(mlrun.common.types.StrEnum):
     leader = "leader"
 
 
-class ProjectMetadata(pydantic.BaseModel):
+class ProjectMetadata(pydantic.v1.BaseModel):
     name: str
     created: typing.Optional[datetime.datetime] = None
     labels: typing.Optional[dict] = {}
     annotations: typing.Optional[dict] = {}
 
     class Config:
-        extra = pydantic.Extra.allow
+        extra = pydantic.v1.Extra.allow
 
 
 class ProjectDesiredState(mlrun.common.types.StrEnum):
@@ -77,7 +77,7 @@ class ProjectStatus(ObjectStatus):
     state: typing.Optional[ProjectState]
 
 
-class ProjectSpec(pydantic.BaseModel):
+class ProjectSpec(pydantic.v1.BaseModel):
     description: typing.Optional[str] = None
     owner: typing.Optional[str] = None
     goals: typing.Optional[str] = None
@@ -97,10 +97,10 @@ class ProjectSpec(pydantic.BaseModel):
     default_function_node_selector: typing.Optional[dict] = {}
 
     class Config:
-        extra = pydantic.Extra.allow
+        extra = pydantic.v1.Extra.allow
 
 
-class ProjectSpecOut(pydantic.BaseModel):
+class ProjectSpecOut(pydantic.v1.BaseModel):
     description: typing.Optional[str] = None
     owner: typing.Optional[str] = None
     goals: typing.Optional[str] = None
@@ -120,11 +120,11 @@ class ProjectSpecOut(pydantic.BaseModel):
     default_function_node_selector: typing.Optional[dict] = {}
 
     class Config:
-        extra = pydantic.Extra.allow
+        extra = pydantic.v1.Extra.allow
 
 
-class Project(pydantic.BaseModel):
-    kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
+class Project(pydantic.v1.BaseModel):
+    kind: ObjectKind = pydantic.v1.Field(ObjectKind.project, const=True)
     metadata: ProjectMetadata
     spec: ProjectSpec = ProjectSpec()
     status: ObjectStatus = ObjectStatus()
@@ -132,19 +132,19 @@ class Project(pydantic.BaseModel):
 
 # The reason we have a different schema for the response model is that we don't want to validate project.spec.build in
 # the response as the validation was added late and there may be corrupted values in the DB.
-class ProjectOut(pydantic.BaseModel):
-    kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
+class ProjectOut(pydantic.v1.BaseModel):
+    kind: ObjectKind = pydantic.v1.Field(ObjectKind.project, const=True)
     metadata: ProjectMetadata
     spec: ProjectSpecOut = ProjectSpecOut()
     status: ObjectStatus = ObjectStatus()
 
 
-class ProjectOwner(pydantic.BaseModel):
+class ProjectOwner(pydantic.v1.BaseModel):
     username: str
     access_key: str
 
 
-class ProjectSummary(pydantic.BaseModel):
+class ProjectSummary(pydantic.v1.BaseModel):
     name: str
     files_count: int = 0
     feature_sets_count: int = 0
@@ -161,7 +161,7 @@ class ProjectSummary(pydantic.BaseModel):
     updated: typing.Optional[datetime.datetime] = None
 
 
-class IguazioProject(pydantic.BaseModel):
+class IguazioProject(pydantic.v1.BaseModel):
     data: dict
 
 
@@ -175,13 +175,18 @@ class IguazioProject(pydantic.BaseModel):
 # to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
 # https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
 ProjectOutput = typing.TypeVar(
-    "ProjectOutput", ProjectOut, str, ProjectSummary, IguazioProject
+    "ProjectOutput",
+    ProjectOut,
+    str,
+    ProjectSummary,
+    IguazioProject,
+    tuple[str, datetime.datetime],
 )
 
 
-class ProjectsOutput(pydantic.BaseModel):
+class ProjectsOutput(pydantic.v1.BaseModel):
     projects: list[ProjectOutput]
 
 
-class ProjectSummariesOutput(pydantic.BaseModel):
+class ProjectSummariesOutput(pydantic.v1.BaseModel):
     project_summaries: list[ProjectSummary]

mlrun/common/schemas/runs.py

@@ -14,13 +14,13 @@
 
 import typing
 
-import pydantic
+import pydantic.v1
 from deprecated import deprecated
 
 import mlrun.common.types
 
 
-class RunIdentifier(pydantic.BaseModel):
+class RunIdentifier(pydantic.v1.BaseModel):
     kind: typing.Literal["run"] = "run"
     uid: typing.Optional[str]
     iter: typing.Optional[int]

mlrun/common/schemas/runtime_resource.py

@@ -14,7 +14,7 @@
 #
 import typing
 
-import pydantic
+import pydantic.v1
 
 import mlrun.common.types
 
@@ -24,23 +24,23 @@ class ListRuntimeResourcesGroupByField(mlrun.common.types.StrEnum):
     project = "project"
 
 
-class RuntimeResource(pydantic.BaseModel):
+class RuntimeResource(pydantic.v1.BaseModel):
     name: str
     labels: dict[str, str] = {}
     status: typing.Optional[dict]
 
 
-class RuntimeResources(pydantic.BaseModel):
+class RuntimeResources(pydantic.v1.BaseModel):
     crd_resources: list[RuntimeResource] = []
     pod_resources: list[RuntimeResource] = []
     # only for dask runtime
     service_resources: typing.Optional[list[RuntimeResource]] = None
 
     class Config:
-        extra = pydantic.Extra.allow
+        extra = pydantic.v1.Extra.allow
 
 
-class KindRuntimeResources(pydantic.BaseModel):
+class KindRuntimeResources(pydantic.v1.BaseModel):
     kind: str
     resources: RuntimeResources
 

mlrun/common/schemas/schedule.py

@@ -15,7 +15,7 @@
 from datetime import datetime
 from typing import Any, Literal, Optional, Union
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 import mlrun.common.types
 from mlrun.common.schemas.auth import Credentials

mlrun/common/schemas/secret.py

@@ -14,7 +14,7 @@
 #
 from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic.v1 import BaseModel, Field
 
 import mlrun.common.types
 

mlrun/common/schemas/tag.py

@@ -13,17 +13,17 @@
 # limitations under the License.
 #
 
-import pydantic
+import pydantic.v1
 
 from .artifact import ArtifactIdentifier
 
 
-class Tag(pydantic.BaseModel):
+class Tag(pydantic.v1.BaseModel):
     name: str
     project: str
 
 
-class TagObjects(pydantic.BaseModel):
+class TagObjects(pydantic.v1.BaseModel):
     """Tag object"""
 
     kind: str

mlrun/common/schemas/workflow.py

@@ -14,14 +14,14 @@
 #
 import typing
 
-import pydantic
+import pydantic.v1
 
 from mlrun.common.schemas.notification import Notification
 from mlrun.common.schemas.schedule import ScheduleCronTrigger
 from mlrun.common.types import StrEnum
 
 
-class WorkflowSpec(pydantic.BaseModel):
+class WorkflowSpec(pydantic.v1.BaseModel):
     name: str
     engine: typing.Optional[str] = None
     code: typing.Optional[str] = None
@@ -36,7 +36,7 @@ class WorkflowSpec(pydantic.BaseModel):
     workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
 
 
-class WorkflowRequest(pydantic.BaseModel):
+class WorkflowRequest(pydantic.v1.BaseModel):
     spec: typing.Optional[WorkflowSpec] = None
     arguments: typing.Optional[dict] = None
     artifact_path: typing.Optional[str] = None
@@ -46,7 +46,7 @@ class WorkflowRequest(pydantic.BaseModel):
     notifications: typing.Optional[list[Notification]] = None
 
 
-class WorkflowResponse(pydantic.BaseModel):
+class WorkflowResponse(pydantic.v1.BaseModel):
     project: str = None
     name: str = None
     status: str = None
@@ -54,7 +54,7 @@ class WorkflowResponse(pydantic.BaseModel):
     schedule: typing.Union[str, ScheduleCronTrigger] = None
 
 
-class GetWorkflowResponse(pydantic.BaseModel):
+class GetWorkflowResponse(pydantic.v1.BaseModel):
     workflow_id: str = None
 
 

mlrun/config.py

@@ -138,6 +138,9 @@ default_config = {
     "object_retentions": {
         "alert_activation": 14 * 7,  # days
     },
+    # A safety margin to account for delays
+    # This ensures that extra partitions are available beyond the specified retention period
+    "partitions_buffer_multiplier": 3,
     # the grace period (in seconds) that will be given to runtime resources (after they're in terminal state)
     # before deleting them (4 hours)
     "runtime_resources_deletion_grace_period": "14400",
@@ -798,11 +801,30 @@ default_config = {
         "max_allowed": 10000,
         # maximum allowed value for count in criteria field inside AlertConfig
         "max_criteria_count": 100,
+        # interval for periodic events generation job
+        "events_generation_interval": "30",
     },
     "auth_with_client_id": {
         "enabled": False,
         "request_timeout": 5,
     },
+    "services": {
+        # The running service name. One of: "api", "alerts"
+        "service_name": "api",
+        "hydra": {
+            # Comma separated list of services to run on the instance.
+            # Currently, this is only considered when the service_name is "api".
+            # "*" starts all services on the same instance,
+            # other options are considered as running only the api service.
+            "services": "*",
+        },
+    },
+    "notifications": {
+        "smtp": {
+            "config_secret_name": "mlrun-smtp-config",
+            "refresh_interval": "30",
+        }
+    },
 }
 _is_running_as_api = None
 

mlrun/datastore/datastore_profile.py

@@ -19,7 +19,7 @@ import typing
 import warnings
 from urllib.parse import ParseResult, urlparse, urlunparse
 
-import pydantic
+import pydantic.v1
 from mergedeep import merge
 
 import mlrun
@@ -28,15 +28,15 @@ import mlrun.errors
 from ..secrets import get_secret_or_env
 
 
-class DatastoreProfile(pydantic.BaseModel):
+class DatastoreProfile(pydantic.v1.BaseModel):
     type: str
     name: str
     _private_attributes: list = ()
 
     class Config:
-        extra = pydantic.Extra.forbid
+        extra = pydantic.v1.Extra.forbid
 
-    @pydantic.validator("name")
+    @pydantic.v1.validator("name")
     @classmethod
     def lower_case(cls, v):
         return v.lower()
@@ -75,14 +75,14 @@ class TemporaryClientDatastoreProfiles(metaclass=mlrun.utils.singleton.Singleton
 
 
 class DatastoreProfileBasic(DatastoreProfile):
-    type: str = pydantic.Field("basic")
+    type: str = pydantic.v1.Field("basic")
     _private_attributes = "private"
     public: str
     private: typing.Optional[str] = None
 
 
 class DatastoreProfileKafkaTarget(DatastoreProfile):
-    type: str = pydantic.Field("kafka_target")
+    type: str = pydantic.v1.Field("kafka_target")
     _private_attributes = "kwargs_private"
     bootstrap_servers: typing.Optional[str] = None
     brokers: typing.Optional[str] = None
@@ -123,7 +123,7 @@ class DatastoreProfileKafkaTarget(DatastoreProfile):
 
 
 class DatastoreProfileKafkaSource(DatastoreProfile):
-    type: str = pydantic.Field("kafka_source")
+    type: str = pydantic.v1.Field("kafka_source")
     _private_attributes = ("kwargs_private", "sasl_user", "sasl_pass")
     brokers: typing.Union[str, list[str]]
     topics: typing.Union[str, list[str]]
@@ -162,7 +162,7 @@ class DatastoreProfileKafkaSource(DatastoreProfile):
 
 
 class DatastoreProfileV3io(DatastoreProfile):
-    type: str = pydantic.Field("v3io")
+    type: str = pydantic.v1.Field("v3io")
     v3io_access_key: typing.Optional[str] = None
     _private_attributes = "v3io_access_key"
 
@@ -178,7 +178,7 @@ class DatastoreProfileV3io(DatastoreProfile):
 
 
 class DatastoreProfileS3(DatastoreProfile):
-    type: str = pydantic.Field("s3")
+    type: str = pydantic.v1.Field("s3")
     _private_attributes = ("access_key_id", "secret_key")
     endpoint_url: typing.Optional[str] = None
     force_non_anonymous: typing.Optional[str] = None
@@ -188,7 +188,7 @@ class DatastoreProfileS3(DatastoreProfile):
     secret_key: typing.Optional[str] = None
     bucket: typing.Optional[str] = None
 
-    @pydantic.validator("bucket")
+    @pydantic.v1.validator("bucket")
     @classmethod
     def check_bucket(cls, v):
         if not v:
@@ -226,7 +226,7 @@ class DatastoreProfileS3(DatastoreProfile):
 
 
 class DatastoreProfileRedis(DatastoreProfile):
-    type: str = pydantic.Field("redis")
+    type: str = pydantic.v1.Field("redis")
     _private_attributes = ("username", "password")
     endpoint_url: str
     username: typing.Optional[str] = None
@@ -269,7 +269,7 @@ class DatastoreProfileRedis(DatastoreProfile):
 
 
 class DatastoreProfileDBFS(DatastoreProfile):
-    type: str = pydantic.Field("dbfs")
+    type: str = pydantic.v1.Field("dbfs")
     _private_attributes = ("token",)
     endpoint_url: typing.Optional[str] = None  # host
     token: typing.Optional[str] = None
@@ -287,13 +287,13 @@ class DatastoreProfileDBFS(DatastoreProfile):
 
 
 class DatastoreProfileGCS(DatastoreProfile):
-    type: str = pydantic.Field("gcs")
+    type: str = pydantic.v1.Field("gcs")
     _private_attributes = ("gcp_credentials",)
     credentials_path: typing.Optional[str] = None  # path to file.
     gcp_credentials: typing.Optional[typing.Union[str, dict]] = None
     bucket: typing.Optional[str] = None
 
-    @pydantic.validator("bucket")
+    @pydantic.v1.validator("bucket")
     @classmethod
     def check_bucket(cls, v):
         if not v:
@@ -304,7 +304,7 @@ class DatastoreProfileGCS(DatastoreProfile):
             )
         return v
 
-    @pydantic.validator("gcp_credentials", pre=True, always=True)
+    @pydantic.v1.validator("gcp_credentials", pre=True, always=True)
     @classmethod
     def convert_dict_to_json(cls, v):
         if isinstance(v, dict):
@@ -332,7 +332,7 @@ class DatastoreProfileGCS(DatastoreProfile):
 
 
 class DatastoreProfileAzureBlob(DatastoreProfile):
-    type: str = pydantic.Field("az")
+    type: str = pydantic.v1.Field("az")
     _private_attributes = (
         "connection_string",
         "account_key",
@@ -350,7 +350,7 @@ class DatastoreProfileAzureBlob(DatastoreProfile):
     credential: typing.Optional[str] = None
     container: typing.Optional[str] = None
 
-    @pydantic.validator("container")
+    @pydantic.v1.validator("container")
     @classmethod
     def check_container(cls, v):
         if not v:
@@ -392,7 +392,7 @@ class DatastoreProfileAzureBlob(DatastoreProfile):
 
 
 class DatastoreProfileHdfs(DatastoreProfile):
-    type: str = pydantic.Field("hdfs")
+    type: str = pydantic.v1.Field("hdfs")
     _private_attributes = "token"
     host: typing.Optional[str] = None
     port: typing.Optional[int] = None
@@ -415,7 +415,7 @@ class DatastoreProfileHdfs(DatastoreProfile):
         return f"webhdfs://{self.host}:{self.http_port}{subpath}"
 
 
-class DatastoreProfile2Json(pydantic.BaseModel):
+class DatastoreProfile2Json(pydantic.v1.BaseModel):
     @staticmethod
     def _to_json(attributes):
         # First, base64 encode the values