mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0rc2__py3-none-any.whl

mlrun/common/schemas/partition.py

@@ -0,0 +1,137 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import datetime, timedelta
+
+from mlrun.common.types import StrEnum
+
+
+class PartitionInterval(StrEnum):
+    DAY = "DAY"
+    MONTH = "MONTH"
+    YEARWEEK = "YEARWEEK"
+
+    @classmethod
+    def is_valid(cls, value: str) -> bool:
+        return value in cls._value2member_map_
+
+    @classmethod
+    def valid_intervals(cls) -> list:
+        return list(cls._value2member_map_.keys())
+
+    def as_duration(self) -> timedelta:
+        """
+        Convert the partition interval to a duration-like timedelta.
+
+        Returns:
+            timedelta: A duration representing the partition interval.
+        """
+        if self == PartitionInterval.DAY:
+            return timedelta(days=1)
+        elif self == PartitionInterval.MONTH:
+            # Approximate a month as 30 days
+            return timedelta(days=30)
+        elif self == PartitionInterval.YEARWEEK:
+            return timedelta(weeks=1)
+
+    @classmethod
+    def from_function(cls, partition_function: str):
+        """
+        Returns the corresponding PartitionInterval for a given partition function,
+        or None if the function is not mapped.
+
+        :param partition_function: The partition function to map to an interval.
+        :return: PartitionInterval corresponding to the function, or None if no match is found.
+        """
+        partition_function_to_partitions_interval = {
+            "DAY": "DAY",
+            "DAYOFMONTH": "DAY",
+            "MONTH": "MONTH",
+            "YEARWEEK": "YEARWEEK",
+        }
+        interval = partition_function_to_partitions_interval.get(partition_function)
+        if interval and cls.is_valid(interval):
+            return cls[interval]
+        raise KeyError(f"Partition function: {partition_function} isn't supported")
+
+    def get_partition_info(
+        self,
+        start_datetime: datetime,
+        partition_number: int = 1,
+    ) -> list[tuple[str, str]]:
+        """
+        Generates partition details for a specified number of partitions starting from a given datetime.
+
+        :param start_datetime: The starting datetime used for generating partition details.
+        :param partition_number: The number of partitions to generate details for.
+
+        :return: A list of tuples:
+            - partition_name: The name for the partition.
+            - partition_value: The "LESS THAN" value for the next partition boundary.
+        """
+        partitioning_information_list = []
+        current_datetime = start_datetime
+
+        for _ in range(partition_number):
+            partition_name = self.get_partition_name(current_datetime)
+            partition_boundary_date = self.get_next_partition_time(current_datetime)
+            partition_value = self.get_partition_name(partition_boundary_date)
+            partitioning_information_list.append((partition_name, partition_value))
+
+            # Move to the next interval
+            current_datetime = partition_boundary_date
+
+        return partitioning_information_list
+
+    def get_next_partition_time(self, current_datetime: datetime) -> datetime:
+        """
+        Calculates the next partition boundary time based on the specified partition interval.
+        :param current_datetime: The current datetime from which the next interval is calculated.
+
+        :return: A datetime object representing the start of the next partition interval.
+            - If the interval is DAY, it advances by one day.
+            - If the interval is MONTH, it advances to the first day of the next month.
+            - If the interval is YEARWEEK, it advances by one week.
+        """
+        if self == PartitionInterval.DAY:
+            return current_datetime + timedelta(days=1)
+        elif self == PartitionInterval.MONTH:
+            return (current_datetime.replace(day=1) + timedelta(days=32)).replace(day=1)
+        elif self == PartitionInterval.YEARWEEK:
+            return current_datetime + timedelta(weeks=1)
+
+    def get_partition_name(self, current_datetime: datetime) -> str:
+        if self == PartitionInterval.DAY:
+            return current_datetime.strftime("%Y%m%d")
+        elif self == PartitionInterval.MONTH:
+            return current_datetime.strftime("%Y%m")
+        elif self == PartitionInterval.YEARWEEK:
+            year, week, _ = current_datetime.isocalendar()
+            return f"{year}{week:02d}"
+
+    def get_partition_expression(self):
+        if self == PartitionInterval.YEARWEEK:
+            return "YEARWEEK(activation_time, 1)"
+        else:
+            return f"{self}(activation_time)"
+
+    def get_number_of_partitions(self, days: int) -> int:
+        # Calculate the number partitions based on given number of days
+        if self == PartitionInterval.DAY:
+            return days
+        elif self == PartitionInterval.MONTH:
+            # Average number days in a month is 30.44
+            return int(days / 30.44)
+        elif self == PartitionInterval.YEARWEEK:
+            return int(days / 7)

mlrun/common/schemas/pipeline.py

@@ -14,7 +14,7 @@
 #
 import typing
 
-import pydantic
+import pydantic.v1
 from deprecated import deprecated
 
 import mlrun.common.types
@@ -39,7 +39,7 @@ class PipelinesPagination(str):
     max_page_size = 200
 
 
-class PipelinesOutput(pydantic.BaseModel):
+class PipelinesOutput(pydantic.v1.BaseModel):
     # use the format query param to control what is returned
     runs: list[typing.Union[dict, str]]
     total_size: int

mlrun/common/schemas/project.py

@@ -15,7 +15,7 @@
 import datetime
 import typing
 
-import pydantic
+import pydantic.v1
 from deprecated import deprecated
 
 import mlrun.common.types
@@ -40,14 +40,14 @@ class ProjectsFormat(mlrun.common.types.StrEnum):
     leader = "leader"
 
 
-class ProjectMetadata(pydantic.BaseModel):
+class ProjectMetadata(pydantic.v1.BaseModel):
     name: str
     created: typing.Optional[datetime.datetime] = None
     labels: typing.Optional[dict] = {}
     annotations: typing.Optional[dict] = {}
 
     class Config:
-        extra = pydantic.Extra.allow
+        extra = pydantic.v1.Extra.allow
 
 
 class ProjectDesiredState(mlrun.common.types.StrEnum):
@@ -77,7 +77,7 @@ class ProjectStatus(ObjectStatus):
     state: typing.Optional[ProjectState]
 
 
-class ProjectSpec(pydantic.BaseModel):
+class ProjectSpec(pydantic.v1.BaseModel):
     description: typing.Optional[str] = None
     owner: typing.Optional[str] = None
     goals: typing.Optional[str] = None
@@ -97,10 +97,10 @@ class ProjectSpec(pydantic.BaseModel):
     default_function_node_selector: typing.Optional[dict] = {}
 
     class Config:
-        extra = pydantic.Extra.allow
+        extra = pydantic.v1.Extra.allow
 
 
-class ProjectSpecOut(pydantic.BaseModel):
+class ProjectSpecOut(pydantic.v1.BaseModel):
     description: typing.Optional[str] = None
     owner: typing.Optional[str] = None
     goals: typing.Optional[str] = None
@@ -120,11 +120,11 @@ class ProjectSpecOut(pydantic.BaseModel):
     default_function_node_selector: typing.Optional[dict] = {}
 
     class Config:
-        extra = pydantic.Extra.allow
+        extra = pydantic.v1.Extra.allow
 
 
-class Project(pydantic.BaseModel):
-    kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
+class Project(pydantic.v1.BaseModel):
+    kind: ObjectKind = pydantic.v1.Field(ObjectKind.project, const=True)
     metadata: ProjectMetadata
     spec: ProjectSpec = ProjectSpec()
     status: ObjectStatus = ObjectStatus()
@@ -132,19 +132,19 @@ class Project(pydantic.BaseModel):
 
 # The reason we have a different schema for the response model is that we don't want to validate project.spec.build in
 # the response as the validation was added late and there may be corrupted values in the DB.
-class ProjectOut(pydantic.BaseModel):
-    kind: ObjectKind = pydantic.Field(ObjectKind.project, const=True)
+class ProjectOut(pydantic.v1.BaseModel):
+    kind: ObjectKind = pydantic.v1.Field(ObjectKind.project, const=True)
     metadata: ProjectMetadata
     spec: ProjectSpecOut = ProjectSpecOut()
     status: ObjectStatus = ObjectStatus()
 
 
-class ProjectOwner(pydantic.BaseModel):
+class ProjectOwner(pydantic.v1.BaseModel):
     username: str
     access_key: str
 
 
-class ProjectSummary(pydantic.BaseModel):
+class ProjectSummary(pydantic.v1.BaseModel):
     name: str
     files_count: int = 0
     feature_sets_count: int = 0
@@ -161,7 +161,7 @@ class ProjectSummary(pydantic.BaseModel):
     updated: typing.Optional[datetime.datetime] = None
 
 
-class IguazioProject(pydantic.BaseModel):
+class IguazioProject(pydantic.v1.BaseModel):
     data: dict
 
 
@@ -175,13 +175,18 @@ class IguazioProject(pydantic.BaseModel):
 # to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
 # https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
 ProjectOutput = typing.TypeVar(
-    "ProjectOutput", ProjectOut, str, ProjectSummary, IguazioProject
+    "ProjectOutput",
+    ProjectOut,
+    str,
+    ProjectSummary,
+    IguazioProject,
+    tuple[str, datetime.datetime],
 )
 
 
-class ProjectsOutput(pydantic.BaseModel):
+class ProjectsOutput(pydantic.v1.BaseModel):
     projects: list[ProjectOutput]
 
 
-class ProjectSummariesOutput(pydantic.BaseModel):
+class ProjectSummariesOutput(pydantic.v1.BaseModel):
     project_summaries: list[ProjectSummary]

mlrun/common/schemas/runs.py

@@ -14,13 +14,13 @@
 
 import typing
 
-import pydantic
+import pydantic.v1
 from deprecated import deprecated
 
 import mlrun.common.types
 
 
-class RunIdentifier(pydantic.BaseModel):
+class RunIdentifier(pydantic.v1.BaseModel):
     kind: typing.Literal["run"] = "run"
     uid: typing.Optional[str]
     iter: typing.Optional[int]

mlrun/common/schemas/runtime_resource.py

@@ -14,7 +14,7 @@
 #
 import typing
 
-import pydantic
+import pydantic.v1
 
 import mlrun.common.types
 
@@ -24,23 +24,23 @@ class ListRuntimeResourcesGroupByField(mlrun.common.types.StrEnum):
     project = "project"
 
 
-class RuntimeResource(pydantic.BaseModel):
+class RuntimeResource(pydantic.v1.BaseModel):
     name: str
     labels: dict[str, str] = {}
     status: typing.Optional[dict]
 
 
-class RuntimeResources(pydantic.BaseModel):
+class RuntimeResources(pydantic.v1.BaseModel):
     crd_resources: list[RuntimeResource] = []
     pod_resources: list[RuntimeResource] = []
     # only for dask runtime
     service_resources: typing.Optional[list[RuntimeResource]] = None
 
     class Config:
-        extra = pydantic.Extra.allow
+        extra = pydantic.v1.Extra.allow
 
 
-class KindRuntimeResources(pydantic.BaseModel):
+class KindRuntimeResources(pydantic.v1.BaseModel):
     kind: str
     resources: RuntimeResources
 

mlrun/common/schemas/schedule.py

@@ -15,7 +15,7 @@
 from datetime import datetime
 from typing import Any, Literal, Optional, Union
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 import mlrun.common.types
 from mlrun.common.schemas.auth import Credentials

mlrun/common/schemas/secret.py

@@ -14,7 +14,7 @@
 #
 from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic.v1 import BaseModel, Field
 
 import mlrun.common.types
 

mlrun/common/schemas/tag.py

@@ -13,17 +13,17 @@
 # limitations under the License.
 #
 
-import pydantic
+import pydantic.v1
 
 from .artifact import ArtifactIdentifier
 
 
-class Tag(pydantic.BaseModel):
+class Tag(pydantic.v1.BaseModel):
     name: str
     project: str
 
 
-class TagObjects(pydantic.BaseModel):
+class TagObjects(pydantic.v1.BaseModel):
     """Tag object"""
 
     kind: str

mlrun/common/schemas/workflow.py

@@ -14,14 +14,14 @@
 #
 import typing
 
-import pydantic
+import pydantic.v1
 
 from mlrun.common.schemas.notification import Notification
 from mlrun.common.schemas.schedule import ScheduleCronTrigger
 from mlrun.common.types import StrEnum
 
 
-class WorkflowSpec(pydantic.BaseModel):
+class WorkflowSpec(pydantic.v1.BaseModel):
     name: str
     engine: typing.Optional[str] = None
     code: typing.Optional[str] = None
@@ -36,7 +36,7 @@ class WorkflowSpec(pydantic.BaseModel):
     workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
 
 
-class WorkflowRequest(pydantic.BaseModel):
+class WorkflowRequest(pydantic.v1.BaseModel):
     spec: typing.Optional[WorkflowSpec] = None
     arguments: typing.Optional[dict] = None
     artifact_path: typing.Optional[str] = None
@@ -46,7 +46,7 @@ class WorkflowRequest(pydantic.BaseModel):
     notifications: typing.Optional[list[Notification]] = None
 
 
-class WorkflowResponse(pydantic.BaseModel):
+class WorkflowResponse(pydantic.v1.BaseModel):
     project: str = None
     name: str = None
     status: str = None
@@ -54,7 +54,7 @@ class WorkflowResponse(pydantic.BaseModel):
     schedule: typing.Union[str, ScheduleCronTrigger] = None
 
 
-class GetWorkflowResponse(pydantic.BaseModel):
+class GetWorkflowResponse(pydantic.v1.BaseModel):
     workflow_id: str = None
 
 

mlrun/config.py

@@ -102,6 +102,9 @@ default_config = {
     "log_level": "INFO",
     # log formatter (options: human | human_extended | json)
     "log_formatter": "human",
+    # custom logger format, workes only with log_formatter: custom
+    # Note that your custom format must include those 4 fields - timestamp, level, message and more
+    "log_format_override": None,
     "submit_timeout": "180",  # timeout when submitting a new k8s resource
     # runtimes cleanup interval in seconds
     "runtimes_cleanup_interval": "300",
@@ -120,14 +123,6 @@ default_config = {
         "projects": {
             "summaries": {
                 "cache_interval": "30",
-                "feature_gates": {
-                    "artifacts": "enabled",
-                    "schedules": "enabled",
-                    "feature_sets": "enabled",
-                    "models": "enabled",
-                    "runs": "enabled",
-                    "pipelines": "enabled",
-                },
             },
         },
     },
@@ -140,6 +135,12 @@ default_config = {
             "delete_crd_resources_timeout": "5 minutes",
         },
     },
+    "object_retentions": {
+        "alert_activation": 14 * 7,  # days
+    },
+    # A safety margin to account for delays
+    # This ensures that extra partitions are available beyond the specified retention period
+    "partitions_buffer_multiplier": 3,
     # the grace period (in seconds) that will be given to runtime resources (after they're in terminal state)
     # before deleting them (4 hours)
     "runtime_resources_deletion_grace_period": "14400",
@@ -314,7 +315,7 @@ default_config = {
                 },
                 "request_timeout": 45,  # seconds
             },
-            # see server.api.utils.helpers.ensure_running_on_chief
+            # see server.py.services.api.utils.helpers.ensure_running_on_chief
             "ensure_function_running_on_chief_mode": "enabled",
         },
         "port": 8080,
@@ -794,17 +795,36 @@ default_config = {
     "grafana_url": "",
     "alerts": {
         # supported modes: "enabled", "disabled".
-        "mode": "disabled",
+        "mode": "enabled",
         # maximum number of alerts we allow to be configured.
         # user will get an error when exceeding this
         "max_allowed": 10000,
         # maximum allowed value for count in criteria field inside AlertConfig
         "max_criteria_count": 100,
+        # interval for periodic events generation job
+        "events_generation_interval": "30",
     },
     "auth_with_client_id": {
         "enabled": False,
         "request_timeout": 5,
     },
+    "services": {
+        # The running service name. One of: "api", "alerts"
+        "service_name": "api",
+        "hydra": {
+            # Comma separated list of services to run on the instance.
+            # Currently, this is only considered when the service_name is "api".
+            # "*" starts all services on the same instance,
+            # other options are considered as running only the api service.
+            "services": "*",
+        },
+    },
+    "notifications": {
+        "smtp": {
+            "config_secret_name": "mlrun-smtp-config",
+            "refresh_interval": "30",
+        }
+    },
 }
 _is_running_as_api = None
 
@@ -851,6 +871,22 @@ class Config:
         name = self.__class__.__name__
         return f"{name}({self._cfg!r})"
 
+    def __iter__(self):
+        if isinstance(self._cfg, Mapping):
+            return self._cfg.__iter__()
+
+    def items(self):
+        if isinstance(self._cfg, Mapping):
+            return iter(self._cfg.items())
+
+    def keys(self):
+        if isinstance(self._cfg, Mapping):
+            return iter(self.data.keys())
+
+    def values(self):
+        if isinstance(self._cfg, Mapping):
+            return iter(self.data.values())
+
     def update(self, cfg, skip_errors=False):
         for key, value in cfg.items():
             if hasattr(self, key):
@@ -1043,6 +1079,17 @@ class Config:
                 f"is not allowed for iguazio version: {igz_version} < 3.5.1"
             )
 
+    def validate_object_retentions(self):
+        for table_name, retention_days in self.object_retentions.items():
+            if retention_days < 7 and not os.getenv("PARTITION_INTERVAL"):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"{table_name} partition interval must be greater than a week"
+                )
+            elif retention_days > 53 * 7:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"{table_name} partition interval must be less than a year"
+                )
+
     def resolve_chief_api_url(self) -> str:
         if self.httpdb.clusterization.chief.url:
             return self.httpdb.clusterization.chief.url
@@ -1201,9 +1248,9 @@ class Config:
 
     def get_model_monitoring_file_target_path(
         self,
-        project: str = "",
-        kind: str = "",
-        target: str = "online",
+        project: str,
+        kind: str,
+        target: typing.Literal["online", "offline"] = "online",
         artifact_path: typing.Optional[str] = None,
         function_name: typing.Optional[str] = None,
         **kwargs,
@@ -1381,9 +1428,12 @@ def _validate_config(config):
         pass
 
     config.verify_security_context_enrichment_mode_is_allowed()
+    config.validate_object_retentions()
 
 
-def _verify_gpu_requests_and_limits(requests_gpu: str = None, limits_gpu: str = None):
+def _verify_gpu_requests_and_limits(
+    requests_gpu: typing.Optional[str] = None, limits_gpu: typing.Optional[str] = None
+):
     # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/
     if requests_gpu and not limits_gpu:
         raise mlrun.errors.MLRunConflictError(
@@ -1396,7 +1446,7 @@ def _verify_gpu_requests_and_limits(requests_gpu: str = None, limits_gpu: str =
         )
 
 
-def _convert_resources_to_str(config: dict = None):
+def _convert_resources_to_str(config: typing.Optional[dict] = None):
     resources_types = ["cpu", "memory", "gpu"]
     resource_requirements = ["requests", "limits"]
     if not config.get("default_function_pod_resources"):

mlrun/data_types/__init__.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from .data_types import (
     InferOptions,

mlrun/data_types/data_types.py

@@ -124,7 +124,6 @@ def spark_to_value_type(data_type):
         "double": ValueType.DOUBLE,
         "boolean": ValueType.BOOL,
         "timestamp": ValueType.DATETIME,
-        "timestamp_ntz": ValueType.DATETIME,
         "string": ValueType.STRING,
         "array": "list",
         "map": "dict",

mlrun/data_types/infer.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from typing import Optional
+
 import numpy as np
 import packaging.version
 import pandas as pd
@@ -29,7 +31,7 @@ def infer_schema_from_df(
     df: pd.DataFrame,
     features,
     entities,
-    timestamp_key: str = None,
+    timestamp_key: Optional[str] = None,
     entity_columns=None,
     options: InferOptions = InferOptions.Null,
 ):

mlrun/data_types/spark.py

@@ -14,11 +14,12 @@
 #
 from datetime import datetime
 from os import environ
+from typing import Optional
 
 import numpy as np
 import pytz
 from pyspark.sql.functions import to_utc_timestamp
-from pyspark.sql.types import BooleanType, DoubleType
+from pyspark.sql.types import BooleanType, DoubleType, TimestampType
 
 from mlrun.feature_store.retrieval.spark_merger import spark_df_to_pandas
 from mlrun.utils import logger
@@ -35,7 +36,7 @@ def infer_schema_from_df_spark(
     df,
     features,
     entities,
-    timestamp_key: str = None,
+    timestamp_key: Optional[str] = None,
     entity_columns=None,
     options: InferOptions = InferOptions.Null,
 ):
@@ -143,8 +144,7 @@ def get_df_stats_spark(df, options, num_bins=20, sample_size=None):
     timestamp_columns = set()
     boolean_columns = set()
     for field in df_after_type_casts.schema.fields:
-        # covers TimestampType and TimestampNTZType, which was added in PySpark 3.4.0
-        is_timestamp = field.dataType.typeName().startswith("timestamp")
+        is_timestamp = isinstance(field.dataType, TimestampType)
         is_boolean = isinstance(field.dataType, BooleanType)
         if is_timestamp:
             df_after_type_casts = df_after_type_casts.withColumn(

mlrun/data_types/to_pandas.py

@@ -244,15 +244,6 @@ def _to_corrected_pandas_type(dt):
 
 
 def spark_df_to_pandas(spark_df):
-    import pyspark
-
-    if semver.parse(pyspark.__version__) >= semver.Version(3, 5, 0):
-
-        def to_pandas(spark_df_inner):
-            return spark_df_inner.toPandas()
-    else:
-        to_pandas = _to_pandas
-
     # as of pyspark 3.2.3, toPandas fails to convert timestamps unless we work around the issue
     # when we upgrade pyspark, we should check whether this workaround is still necessary
     # see https://stackoverflow.com/questions/76389694/transforming-pyspark-to-pandas-dataframe
@@ -271,9 +262,9 @@ def spark_df_to_pandas(spark_df):
                 )
                 type_conversion_dict[field.name] = "datetime64[ns]"
 
-        df = to_pandas(spark_df)
+        df = _to_pandas(spark_df)
         if type_conversion_dict:
             df = df.astype(type_conversion_dict)
         return df
     else:
-        return to_pandas(spark_df)
+        return _to_pandas(spark_df)

mlrun/datastore/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 __all__ = [
     "DataItem",
     "get_store_resource",