mlrun 1.7.2__py3-none-any.whl → 1.8.0rc1__py3-none-any.whl

mlrun/common/formatters/pipeline.py

@@ -15,11 +15,10 @@
 
 import typing
 
+import mlrun.common.types
 import mlrun_pipelines.common.ops
 import mlrun_pipelines.models
 
-import mlrun.common.types
-
 from .base import ObjectFormat
 
 

mlrun/common/model_monitoring/__init__.py

@@ -11,8 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from .helpers import create_model_endpoint_uid

mlrun/common/model_monitoring/helpers.py

@@ -65,7 +65,7 @@ def parse_model_endpoint_store_prefix(store_prefix: str):
 
 
 def parse_monitoring_stream_path(
-    stream_uri: str, project: str, function_name: str = None
+    stream_uri: str, project: str, function_name: typing.Optional[str] = None
 ):
     if stream_uri.startswith("kafka://"):
         if "?topic" in stream_uri:

mlrun/common/runtimes/constants.py

@@ -15,9 +15,8 @@
 import enum
 import typing
 
-import mlrun_pipelines.common.models
-
 import mlrun.common.constants as mlrun_constants
+import mlrun_pipelines.common.models
 
 
 class PodPhases:

mlrun/common/schemas/__init__.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from .alert import (
     AlertActiveState,
@@ -149,6 +147,10 @@ from .model_monitoring import (
     ModelEndpointMetadata,
     ModelEndpointSpec,
     ModelEndpointStatus,
+    ModelEndpointV2,
+    ModelEndpointV2Metadata,
+    ModelEndpointV2Spec,
+    ModelEndpointV2Status,
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,

mlrun/common/schemas/artifact.py

@@ -47,12 +47,6 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
                 True,
             )
 
-    @classmethod
-    def from_kind(cls, kind: str) -> "ArtifactCategories":
-        if kind in [cls.model.value, cls.dataset.value]:
-            return cls(kind)
-        return cls.other
-
 
 class ArtifactIdentifier(pydantic.BaseModel):
     # artifact kind

mlrun/common/schemas/common.py

@@ -16,6 +16,8 @@ import typing
 
 import pydantic
 
+import mlrun.errors
+
 
 class ImageBuilder(pydantic.BaseModel):
     functionSourceCode: typing.Optional[str] = None  # noqa: N815
@@ -41,3 +43,51 @@ class ImageBuilder(pydantic.BaseModel):
 
     class Config:
         extra = pydantic.Extra.allow
+
+
+class LabelsModel(pydantic.BaseModel):
+    """
+    This class accepts either a dictionary, a list, or a string for filtering by labels.
+
+    :param labels:
+        - If a dictionary is provided, it should be in the format {'label_name': 'value'}.
+          The values can also be `None`, which will result in the format 'label_name' (without a value).
+          This will be converted to a list of strings in the format 'label_name=value'.
+        - If a list is provided, all items must be strings. Each string can either
+          be a simple label name (e.g., 'label1') or a key-value pair in the format
+          'label=value'.
+        - If a string is provided, it should be a comma-separated list of labels
+          (e.g., 'label1,label2').
+        - If no labels are specified, the default is an empty list.
+    """
+
+    labels: typing.Optional[
+        typing.Union[str, dict[str, typing.Optional[str]], list[str]]
+    ]
+
+    @pydantic.validator("labels")
+    @classmethod
+    def validate(cls, labels) -> list[str]:
+        if labels is None:
+            return []
+
+        # If labels is a string, split it by commas
+        if isinstance(labels, str):
+            return [label.strip() for label in labels.split(",") if label.strip()]
+
+        if isinstance(labels, list):
+            if not all(isinstance(item, str) for item in labels):
+                raise mlrun.errors.MLRunValueError(
+                    "All items in the list must be strings."
+                )
+            return labels
+
+        if isinstance(labels, dict):
+            return [
+                f"{key}={value}" if value is not None else key
+                for key, value in labels.items()
+            ]
+
+        raise mlrun.errors.MLRunValueError(
+            "Invalid labels format. Must be a string, dictionary of strings, or a list of strings."
+        )

mlrun/common/schemas/model_monitoring/__init__.py

@@ -14,6 +14,7 @@
 
 from .constants import (
     V3IO_MODEL_MONITORING_DB,
+    ApplicationEvent,
     ControllerPolicy,
     DriftStatus,
     EndpointType,
@@ -26,6 +27,7 @@ from .constants import (
     FunctionURI,
     MetricData,
     ModelEndpointMonitoringMetricType,
+    ModelEndpointSchema,
     ModelEndpointTarget,
     ModelEndpointTargetSchemas,
     ModelMonitoringMode,
@@ -36,7 +38,6 @@ from .constants import (
     ResultData,
     ResultKindApp,
     ResultStatusApp,
-    SchedulingKeys,
     SpecialApps,
     TDEngineSuperTables,
     TSDBTarget,
@@ -54,6 +55,12 @@ from .grafana import (
     GrafanaTable,
     GrafanaTimeSeriesTarget,
 )
+from .model_endpoint_v2 import (
+    ModelEndpointV2,
+    ModelEndpointV2Metadata,
+    ModelEndpointV2Spec,
+    ModelEndpointV2Status,
+)
 from .model_endpoints import (
     Features,
     FeatureValues,

mlrun/common/schemas/model_monitoring/constants.py

@@ -29,6 +29,46 @@ class MonitoringStrEnum(StrEnum):
         return list(map(lambda c: c.value, cls))
 
 
+class ModelEndpointSchema(MonitoringStrEnum):
+    # metadata
+    UID = "uid"
+    PROJECT = "project"
+    ENDPOINT_TYPE = "endpoint_type"
+    NAME = "name"
+    CREATED = "created"
+    UPDATED = "updated"
+    LABELS = "labels"
+
+    # spec
+    FUNCTION_NAME = "function_name"
+    FUNCTION_UID = "function_uid"
+    MODEL_NAME = "model_name"
+    MODEL_TAG = "model_tag"
+    MODEL_CLASS = "model_class"
+    MODEL_UID = "model_uid"
+    FEATURE_NAMES = "feature_names"
+    LABEL_NAMES = "label_names"
+
+    # status
+    STATE = "state"
+    MONITORING_MODE = "monitoring_mode"
+    MONITORING_FEATURE_SET_URI = "monitoring_feature_set_uri"
+    CHILDREN = "children"
+    CHILDREN_UIDS = "children_uids"
+    FIRST_REQUEST = "first_request"
+    FUNCTION_URI = "function_uri"
+    MODEL_URI = "model_uri"
+
+    # status - operative
+    LAST_REQUEST = "last_request"
+    DRIFT_STATUS = "drift_status"
+    AVG_LATENCY = "avg_latency"
+    ERROR_COUNT = "error_count"
+    FEATURE_STATS = "feature_stats"
+    CURRENT_STATS = "current_stats"
+    DRIFT_MEASURES = "drift_measures"
+
+
 class EventFieldType:
     FUNCTION_URI = "function_uri"
     FUNCTION = "function"
@@ -55,6 +95,8 @@ class EventFieldType:
     NAMED_PREDICTIONS = "named_predictions"
     ERROR_COUNT = "error_count"
     MODEL_ERROR = "model_error"
+    ERROR_TYPE = "error_type"
+    INFER_ERROR = "infer_error"
     ENTITIES = "entities"
     FIRST_REQUEST = "first_request"
     LAST_REQUEST = "last_request"
@@ -114,13 +156,14 @@ class WriterEvent(MonitoringStrEnum):
     ENDPOINT_ID = "endpoint_id"
     START_INFER_TIME = "start_infer_time"
     END_INFER_TIME = "end_infer_time"
-    EVENT_KIND = "event_kind"  # metric or result
+    EVENT_KIND = "event_kind"  # metric or result or stats
     DATA = "data"
 
 
 class WriterEventKind(MonitoringStrEnum):
     METRIC = "metric"
     RESULT = "result"
+    STATS = "stats"
 
 
 class MetricData(MonitoringStrEnum):
@@ -134,7 +177,17 @@ class ResultData(MonitoringStrEnum):
     RESULT_KIND = "result_kind"
     RESULT_STATUS = "result_status"
     RESULT_EXTRA_DATA = "result_extra_data"
+
+
+class StatsData(MonitoringStrEnum):
+    STATS_NAME = "stats_name"
+    STATS = "stats"
+    TIMESTAMP = "timestamp"
+
+
+class StatsKind(MonitoringStrEnum):
     CURRENT_STATS = "current_stats"
+    DRIFT_MEASURES = "drift_measures"
 
 
 class EventLiveStats:
@@ -194,13 +247,6 @@ class ModelMonitoringStoreKinds:
     EVENTS = "events"
 
 
-class SchedulingKeys:
-    LAST_ANALYZED = "last_analyzed"
-    ENDPOINT_ID = "endpoint_id"
-    APPLICATION_NAME = "application_name"
-    UID = "uid"
-
-
 class FileTargetKind:
     ENDPOINTS = "endpoints"
     EVENTS = "events"
@@ -209,14 +255,13 @@ class FileTargetKind:
     PARQUET = "parquet"
     APPS_PARQUET = "apps_parquet"
     LOG_STREAM = "log_stream"
-    APP_RESULTS = "app_results"
-    APP_METRICS = "app_metrics"
     MONITORING_SCHEDULES = "monitoring_schedules"
     MONITORING_APPLICATION = "monitoring_application"
     ERRORS = "errors"
+    STATS = "stats"
 
 
-class ModelMonitoringMode(str, Enum):
+class ModelMonitoringMode(StrEnum):
     enabled = "enabled"
     disabled = "disabled"
 
@@ -225,6 +270,11 @@ class EndpointType(IntEnum):
     NODE_EP = 1  # end point that is not a child of a router
     ROUTER = 2  # endpoint that is router
     LEAF_EP = 3  # end point that is a child of a router
+    BATCH_EP = 4  # endpoint that is representing an offline batch endpoint
+
+    @classmethod
+    def top_level_list(cls):
+        return [cls.NODE_EP, cls.ROUTER, cls.BATCH_EP]
 
 
 class MonitoringFunctionNames(MonitoringStrEnum):
@@ -244,6 +294,7 @@ class TDEngineSuperTables(MonitoringStrEnum):
     APP_RESULTS = "app_results"
     METRICS = "metrics"
     PREDICTIONS = "predictions"
+    ERRORS = "errors"
 
 
 @dataclass
@@ -364,7 +415,6 @@ class SpecialApps:
 
 _RESERVED_FUNCTION_NAMES = MonitoringFunctionNames.list() + [SpecialApps.MLRUN_INFRA]
 
-
 V3IO_MODEL_MONITORING_DB = "v3io"
 
 

mlrun/common/schemas/model_monitoring/model_endpoint_v2.py

@@ -0,0 +1,149 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import abc
+from typing import Optional
+
+from pydantic import BaseModel, Field, constr
+
+from ..object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
+from .constants import (
+    PROJECT_PATTERN,
+    EndpointType,
+    ModelMonitoringMode,
+)
+from .model_endpoints import _mapping_attributes
+
+# TODO : replace ModelEndpoint
+
+
+class ModelEndpointParser(abc.ABC, BaseModel):
+    @classmethod
+    def json_parse_values(cls) -> list[str]:
+        return []
+
+    @classmethod
+    def from_flat_dict(
+        cls, endpoint_dict: dict, json_parse_values: Optional[list] = None
+    ):
+        """Create a `ModelEndpointMetadata` object from an endpoint dictionary
+
+        :param endpoint_dict:     Model endpoint dictionary.
+        :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
+                                  dictionary using json.loads().
+        """
+        if json_parse_values is None:
+            json_parse_values = cls.json_parse_values()
+
+        return _mapping_attributes(
+            model_class=cls,
+            flattened_dictionary=endpoint_dict,
+            json_parse_values=json_parse_values,
+        )
+
+
+class ModelEndpointV2Metadata(ObjectMetadata, ModelEndpointParser):
+    project: constr(regex=PROJECT_PATTERN)
+    endpoint_type: Optional[EndpointType] = EndpointType.NODE_EP.value
+
+
+class ModelEndpointV2Spec(ObjectSpec, ModelEndpointParser):
+    model_uid: Optional[str] = ""
+    model_name: Optional[str] = ""
+    model_tag: Optional[str] = ""
+    model_class: Optional[str] = ""
+    function_name: Optional[str] = ""
+    function_uid: Optional[str] = ""
+    feature_names: Optional[list[str]] = []
+    label_names: Optional[list[str]] = []
+
+
+class ModelEndpointV2Status(ObjectStatus, ModelEndpointParser):
+    state: Optional[str] = "unknown"  # will be updated according to the function state
+    first_request: Optional[str] = ""
+    children: Optional[list[str]] = []
+    children_uids: Optional[list[str]] = []
+    monitoring_feature_set_uri: Optional[str] = ""
+    monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled.value
+    function_uri: Optional[str] = ""  # <project_name>/<function_name>:<tag>
+    model_uri: Optional[str] = ""
+
+    # operative
+    last_request: Optional[str] = ""
+    drift_status: Optional[str] = ""
+    avg_latency: Optional[float] = None
+    error_count: Optional[int] = 0
+    feature_stats: Optional[dict] = {}
+    current_stats: Optional[dict] = {}
+    drift_measures: Optional[dict] = {}
+
+
+class ModelEndpointV2(BaseModel):
+    kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
+    metadata: ModelEndpointV2Metadata
+    spec: ModelEndpointV2Spec
+    status: ModelEndpointV2Status
+
+    def flat_dict(self, exclude: Optional[set] = None):
+        """Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
+        the model endpoint object in the database.
+
+        :return: Flattened `ModelEndpoint` dictionary.
+        """
+        # Convert the ModelEndpoint object into a dictionary using BaseModel dict() function
+        # In addition, remove the BaseModel kind as it is not required by the DB schema
+        if exclude:
+            exclude = exclude | {"kind", "tag"}
+        else:
+            exclude = {"kind", "tag"}
+        model_endpoint_dictionary = self.dict(exclude=exclude)
+
+        # Initialize a flattened dictionary that will be filled with the model endpoint dictionary attributes
+        flatten_dict = {}
+        for k_object in model_endpoint_dictionary:
+            for key in model_endpoint_dictionary[k_object]:
+                # Extract the value of the current field
+                flatten_dict[key] = model_endpoint_dictionary[k_object][key]
+
+        return flatten_dict
+
+    @classmethod
+    def from_flat_dict(cls, endpoint_dict: dict) -> "ModelEndpointV2":
+        """Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
+        is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
+
+        :param endpoint_dict:     Model endpoint dictionary.
+        """
+
+        return cls(
+            metadata=ModelEndpointV2Metadata.from_flat_dict(
+                endpoint_dict=endpoint_dict
+            ),
+            spec=ModelEndpointV2Spec.from_flat_dict(endpoint_dict=endpoint_dict),
+            status=ModelEndpointV2Status.from_flat_dict(endpoint_dict=endpoint_dict),
+        )
+
+    @classmethod
+    def _operative_data(cls) -> set:
+        return {
+            "last_request",
+            "drift_status",
+            "avg_latency",
+            "error_count",
+            "feature_stats",
+            "current_stats",
+            "drift_measures",
+            "function_uri",
+            "model_uri",
+        }

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -56,7 +56,9 @@ class ModelEndpointMetadata(BaseModel):
         extra = Extra.allow
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
+    def from_flat_dict(
+        cls, endpoint_dict: dict, json_parse_values: Optional[list] = None
+    ):
         """Create a `ModelEndpointMetadata` object from an endpoint dictionary
 
         :param endpoint_dict:     Model endpoint dictionary.
@@ -87,7 +89,9 @@ class ModelEndpointSpec(ObjectSpec):
     monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled.value
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
+    def from_flat_dict(
+        cls, endpoint_dict: dict, json_parse_values: Optional[list] = None
+    ):
         """Create a `ModelEndpointSpec` object from an endpoint dictionary
 
         :param endpoint_dict:     Model endpoint dictionary.
@@ -188,7 +192,9 @@ class ModelEndpointStatus(ObjectStatus):
         extra = Extra.allow
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
+    def from_flat_dict(
+        cls, endpoint_dict: dict, json_parse_values: Optional[list] = None
+    ):
         """Create a `ModelEndpointStatus` object from an endpoint dictionary
 
         :param endpoint_dict:     Model endpoint dictionary.
@@ -284,7 +290,14 @@ class ModelEndpointMonitoringMetric(BaseModel):
     app: str
     type: ModelEndpointMonitoringMetricType
     name: str
-    full_name: str
+    full_name: Optional[str] = None
+    kind: Optional[ResultKindApp] = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.full_name = _compose_full_name(
+            project=self.project, app=self.app, name=self.name, type=self.type
+        )
 
 
 def _compose_full_name(
@@ -315,6 +328,7 @@ class _ResultPoint(NamedTuple):
     timestamp: datetime
     value: float
     status: ResultStatusApp
+    extra_data: Optional[str] = ""
 
 
 class _ModelEndpointMonitoringMetricValuesBase(BaseModel):
@@ -365,8 +379,10 @@ def _mapping_attributes(
                 dict_to_parse[field_key] = _json_loads_if_not_none(
                     flattened_dictionary[field_key]
                 )
-            else:
+            elif flattened_dictionary[field_key] != "null":
                 dict_to_parse[field_key] = flattened_dictionary[field_key]
+            else:
+                dict_to_parse[field_key] = None
 
     return model_class.parse_obj(dict_to_parse)
 

mlrun/common/schemas/partition.py

@@ -0,0 +1,122 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import datetime, timedelta
+
+from mlrun.common.types import StrEnum
+
+
+class PartitionInterval(StrEnum):
+    DAY = "DAY"
+    MONTH = "MONTH"
+    YEARWEEK = "YEARWEEK"
+
+    @classmethod
+    def is_valid(cls, value: str) -> bool:
+        return value in cls._value2member_map_
+
+    @classmethod
+    def valid_intervals(cls) -> list:
+        return list(cls._value2member_map_.keys())
+
+    @classmethod
+    def from_function(cls, partition_function: str):
+        """
+        Returns the corresponding PartitionInterval for a given partition function,
+        or None if the function is not mapped.
+
+        :param partition_function: The partition function to map to an interval.
+        :return: PartitionInterval corresponding to the function, or None if no match is found.
+        """
+        partition_function_to_partitions_interval = {
+            "DAY": "DAY",
+            "DAYOFMONTH": "DAY",
+            "MONTH": "MONTH",
+            "YEARWEEK": "YEARWEEK",
+        }
+        interval = partition_function_to_partitions_interval.get(partition_function)
+        if interval and cls.is_valid(interval):
+            return cls[interval]
+        return None
+
+    def get_partition_info(
+        self,
+        start_datetime: datetime,
+        partition_number: int = 1,
+    ) -> list[tuple[str, str]]:
+        """
+        Generates partition details for a specified number of partitions starting from a given datetime.
+
+        :param start_datetime: The starting datetime used for generating partition details.
+        :param partition_number: The number of partitions to generate details for.
+
+        :return: A list of tuples:
+            - partition_name: The name for the partition.
+            - partition_value: The "LESS THAN" value for the next partition boundary.
+        """
+        partitioning_information_list = []
+        current_datetime = start_datetime
+
+        for _ in range(partition_number):
+            partition_name = self.get_partition_name(current_datetime)
+            partition_boundary_date = self.get_next_partition_time(current_datetime)
+            partition_value = self.get_partition_name(partition_boundary_date)
+            partitioning_information_list.append((partition_name, partition_value))
+
+            # Move to the next interval
+            current_datetime = partition_boundary_date
+
+        return partitioning_information_list
+
+    def get_next_partition_time(self, current_datetime: datetime) -> datetime:
+        """
+        Calculates the next partition boundary time based on the specified partition interval.
+        :param current_datetime: The current datetime from which the next interval is calculated.
+
+        :return: A datetime object representing the start of the next partition interval.
+            - If the interval is DAY, it advances by one day.
+            - If the interval is MONTH, it advances to the first day of the next month.
+            - If the interval is YEARWEEK, it advances by one week.
+        """
+        if self == PartitionInterval.DAY:
+            return current_datetime + timedelta(days=1)
+        elif self == PartitionInterval.MONTH:
+            return (current_datetime.replace(day=1) + timedelta(days=32)).replace(day=1)
+        elif self == PartitionInterval.YEARWEEK:
+            return current_datetime + timedelta(weeks=1)
+
+    def get_partition_name(self, current_datetime: datetime) -> str:
+        if self == PartitionInterval.DAY:
+            return current_datetime.strftime("%Y%m%d")
+        elif self == PartitionInterval.MONTH:
+            return current_datetime.strftime("%Y%m")
+        elif self == PartitionInterval.YEARWEEK:
+            year, week, _ = current_datetime.isocalendar()
+            return f"{year}{week:02d}"
+
+    def get_partition_expression(self):
+        if self == PartitionInterval.YEARWEEK:
+            return "YEARWEEK(activation_time, 1)"
+        else:
+            return f"{self}(activation_time)"
+
+    def get_number_of_partitions(self, days: int) -> int:
+        # Calculate the number partitions based on given number of days
+        if self == PartitionInterval.DAY:
+            return days
+        elif self == PartitionInterval.MONTH:
+            # Average number days in a month is 30.44
+            return int(days / 30.44)
+        elif self == PartitionInterval.YEARWEEK:
+            return int(days / 7)