mlrun 1.7.0rc22__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl

mlrun/__main__.py

@@ -50,12 +50,12 @@ from .run import (
 from .runtimes import RemoteRuntime, RunError, RuntimeKinds, ServingRuntime
 from .secrets import SecretsStore
 from .utils import (
+    RunKeys,
     dict_to_yaml,
     get_in,
     is_relative_path,
     list2dict,
     logger,
-    run_keys,
     update_in,
 )
 from .utils.version import Version
@@ -102,7 +102,9 @@ def main():
 )
 @click.option("--uid", help="unique run ID")
 @click.option("--name", help="run name")
-@click.option("--workflow", help="workflow name/id")
+@click.option(
+    "--workflow", help="sets the run labels to match the given workflow name/id"
+)
 @click.option("--project", help="project name/id")
 @click.option("--db", default="", help="save run results to path or DB url")
 @click.option(
@@ -378,15 +380,15 @@ def run(
     set_item(runobj.spec.hyper_param_options, hyper_param_strategy, "strategy")
     set_item(runobj.spec.hyper_param_options, selector, "selector")
 
-    set_item(runobj.spec, inputs, run_keys.inputs, list2dict(inputs))
+    set_item(runobj.spec, inputs, RunKeys.inputs, list2dict(inputs))
     set_item(
-        runobj.spec, returns, run_keys.returns, [py_eval(value) for value in returns]
+        runobj.spec, returns, RunKeys.returns, [py_eval(value) for value in returns]
     )
-    set_item(runobj.spec, in_path, run_keys.input_path)
-    set_item(runobj.spec, out_path, run_keys.output_path)
-    set_item(runobj.spec, outputs, run_keys.outputs, list(outputs))
+    set_item(runobj.spec, in_path, RunKeys.input_path)
+    set_item(runobj.spec, out_path, RunKeys.output_path)
+    set_item(runobj.spec, outputs, RunKeys.outputs, list(outputs))
     set_item(
-        runobj.spec, secrets, run_keys.secrets, line2keylist(secrets, "kind", "source")
+        runobj.spec, secrets, RunKeys.secrets, line2keylist(secrets, "kind", "source")
     )
     set_item(runobj.spec, verbose, "verbose")
     set_item(runobj.spec, scrape_metrics, "scrape_metrics")

mlrun/alerts/alert.py

@@ -26,7 +26,6 @@ class AlertConfig(ModelObj):
         "description",
         "summary",
         "severity",
-        "criteria",
         "reset_policy",
         "state",
     ]
@@ -34,6 +33,7 @@ class AlertConfig(ModelObj):
         "entities",
         "notifications",
         "trigger",
+        "criteria",
     ]
 
     def __init__(
@@ -104,6 +104,14 @@ class AlertConfig(ModelObj):
                     else self.trigger
                 )
             return None
+        if field_name == "criteria":
+            if self.criteria:
+                return (
+                    self.criteria.dict()
+                    if not isinstance(self.criteria, dict)
+                    else self.criteria
+                )
+            return None
         return super()._serialize_field(struct, field_name, strip)
 
     def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
@@ -137,6 +145,10 @@ class AlertConfig(ModelObj):
             trigger_obj = alert_objects.AlertTrigger.parse_obj(trigger_data)
             new_obj.trigger = trigger_obj
 
+        criteria_data = struct.get("criteria")
+        if criteria_data:
+            criteria_obj = alert_objects.AlertCriteria.parse_obj(criteria_data)
+            new_obj.criteria = criteria_obj
         return new_obj
 
     def with_notifications(self, notifications: list[alert_objects.AlertNotification]):

mlrun/artifacts/manager.py

@@ -100,6 +100,11 @@ class ArtifactProducer:
 
 def dict_to_artifact(struct: dict) -> Artifact:
     kind = struct.get("kind", "")
+
+    # TODO: remove this in 1.8.0
+    if mlrun.utils.is_legacy_artifact(struct):
+        return mlrun.artifacts.base.convert_legacy_artifact_to_new_format(struct)
+
     artifact_class = artifact_types[kind]
     return artifact_class.from_dict(struct)
 

mlrun/common/constants.py

@@ -64,12 +64,12 @@ class MLRunInternalLabels:
     username = f"{MLRUN_LABEL_PREFIX}username"
     username_domain = f"{MLRUN_LABEL_PREFIX}username_domain"
     task_name = f"{MLRUN_LABEL_PREFIX}task-name"
+    resource_name = f"{MLRUN_LABEL_PREFIX}resource_name"
+    created = f"{MLRUN_LABEL_PREFIX}created"
     host = "host"
     job_type = "job-type"
     kind = "kind"
     component = "component"
-    resource_name = "resource_name"
-    created = "mlrun-created"
 
     owner = "owner"
     v3io_user = "v3io_user"

mlrun/common/formatters/__init__.py

@@ -17,3 +17,4 @@ from .artifact import ArtifactFormat  # noqa
 from .function import FunctionFormat  # noqa
 from .pipeline import PipelineFormat  # noqa
 from .project import ProjectFormat  # noqa
+from .run import RunFormat  # noqa

mlrun/common/formatters/artifact.py

@@ -13,9 +13,32 @@
 # limitations under the License.
 #
 
+import typing
+
 import mlrun.common.types
 
+from .base import ObjectFormat
+
+
+class ArtifactFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    minimal = "minimal"
 
-# TODO: add a format that returns a minimal response with ObjectFormat
-class ArtifactFormat(mlrun.common.types.StrEnum):
-    full = "full"
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            ArtifactFormat.full: None,
+            ArtifactFormat.minimal: ArtifactFormat.filter_obj_method(
+                [
+                    "kind",
+                    "metadata",
+                    "status",
+                    "project",
+                    "spec.producer",
+                    "spec.db_key",
+                    "spec.size",
+                    "spec.framework",
+                    "spec.metrics",
+                    "spec.target_path",
+                ]
+            ),
+        }[_format]

mlrun/common/formatters/base.py

@@ -28,42 +28,42 @@ class ObjectFormat:
     full = "full"
 
     @staticmethod
-    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+    def format_method(format_: str) -> typing.Optional[typing.Callable]:
         """
         Get the formatting method for the provided format.
         A `None` value signifies a pass-through formatting method (no formatting).
-        :param _format: The format as a string representation.
+        :param format_: The format as a string representation.
         :return: The formatting method.
         """
         return {
             ObjectFormat.full: None,
-        }[_format]
+        }[format_]
 
     @classmethod
     def format_obj(
         cls,
         obj: typing.Any,
-        _format: str,
+        format_: str,
         exclude_formats: typing.Optional[list[str]] = None,
     ) -> typing.Any:
         """
         Format the provided object based on the provided format.
         :param obj: The object to format.
-        :param _format: The format as a string representation.
+        :param format_: The format as a string representation.
         :param exclude_formats: A list of formats to exclude from the formatting process. If the provided format is in
                                 this list, an invalid format exception will be raised.
         """
         exclude_formats = exclude_formats or []
-        _format = _format or cls.full
+        format_ = format_ or cls.full
         invalid_format_exc = mlrun.errors.MLRunBadRequestError(
-            f"Provided format is not supported. format={_format}"
+            f"Provided format is not supported. format={format_}"
         )
 
-        if _format in exclude_formats:
+        if format_ in exclude_formats:
             raise invalid_format_exc
 
         try:
-            format_method = cls.format_method(_format)
+            format_method = cls.format_method(format_)
         except KeyError:
             raise invalid_format_exc
 

mlrun/common/formatters/run.py

@@ -0,0 +1,26 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import mlrun.common.types
+from mlrun.common.formatters.base import ObjectFormat
+
+
+class RunFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    # No enrichment, data is pulled as-is from the database.
+    standard = "standard"
+
+    # Performs run enrichment, including the run's artifacts. Only available for the `get` run API.
+    full = "full"

mlrun/common/helpers.py

@@ -34,3 +34,14 @@ def parse_versioned_object_uri(
         uri = uri[:loc]
 
     return project, uri, tag, hash_key
+
+
+def generate_api_gateway_name(project: str, name: str) -> str:
+    """
+    Generate a unique (within project) api gateway name
+    :param project: project name
+    :param name: api gateway name
+
+    :return: the resolved api gateway name
+    """
+    return f"{project}-{name}" if project else name

mlrun/common/schemas/__init__.py

@@ -82,6 +82,7 @@ from .events import (
 )
 from .feature_store import (
     EntitiesOutput,
+    EntitiesOutputV2,
     Entity,
     EntityListOutput,
     EntityRecord,
@@ -90,7 +91,9 @@ from .feature_store import (
     FeatureRecord,
     FeatureSet,
     FeatureSetDigestOutput,
+    FeatureSetDigestOutputV2,
     FeatureSetDigestSpec,
+    FeatureSetDigestSpecV2,
     FeatureSetIngestInput,
     FeatureSetIngestOutput,
     FeatureSetRecord,
@@ -98,6 +101,7 @@ from .feature_store import (
     FeatureSetSpec,
     FeatureSetsTagsOutput,
     FeaturesOutput,
+    FeaturesOutputV2,
     FeatureVector,
     FeatureVectorRecord,
     FeatureVectorsOutput,

mlrun/common/schemas/alert.py

@@ -39,8 +39,8 @@ class EventKind(StrEnum):
     CONCEPT_DRIFT_SUSPECTED = "concept_drift_suspected"
     MODEL_PERFORMANCE_DETECTED = "model_performance_detected"
     MODEL_PERFORMANCE_SUSPECTED = "model_performance_suspected"
-    MODEL_SERVING_PERFORMANCE_DETECTED = "model_serving_performance_detected"
-    MODEL_SERVING_PERFORMANCE_SUSPECTED = "model_serving_performance_suspected"
+    SYSTEM_PERFORMANCE_DETECTED = "system_performance_detected"
+    SYSTEM_PERFORMANCE_SUSPECTED = "system_performance_suspected"
     MM_APP_ANOMALY_DETECTED = "mm_app_anomaly_detected"
     MM_APP_ANOMALY_SUSPECTED = "mm_app_anomaly_suspected"
     FAILED = "failed"
@@ -53,12 +53,8 @@ _event_kind_entity_map = {
     EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
     EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
     EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
-    EventKind.MODEL_SERVING_PERFORMANCE_DETECTED: [
-        EventEntityKind.MODEL_ENDPOINT_RESULT
-    ],
-    EventKind.MODEL_SERVING_PERFORMANCE_SUSPECTED: [
-        EventEntityKind.MODEL_ENDPOINT_RESULT
-    ],
+    EventKind.SYSTEM_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.SYSTEM_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
     EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
     EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
     EventKind.FAILED: [EventEntityKind.JOB],
@@ -104,7 +100,7 @@ class AlertCriteria(pydantic.BaseModel):
         pydantic.Field(
             description="Number of events to wait until notification is sent"
         ),
-    ] = 0
+    ] = 1
     period: Annotated[
         str,
         pydantic.Field(

mlrun/common/schemas/api_gateway.py

@@ -17,8 +17,10 @@ from typing import Optional
 
 import pydantic
 
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.types
 from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
+from mlrun.common.helpers import generate_api_gateway_name
 
 
 class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
@@ -100,7 +102,58 @@ class APIGateway(_APIGatewayBaseModel):
             if upstream.nucliofunction.get("name")
         ]
 
-    def enrich_mlrun_function_names(self):
+    def get_invoke_url(self):
+        return (
+            self.spec.host + self.spec.path
+            if self.spec.path and self.spec.host
+            else self.spec.host
+        )
+
+    def enrich_mlrun_names(self):
+        self._enrich_api_gateway_mlrun_name()
+        self._enrich_mlrun_function_names()
+        return self
+
+    def replace_nuclio_names_with_mlrun_names(self):
+        self._replace_nuclio_api_gateway_name_with_mlrun_name()
+        self._replace_nuclio_function_names_with_mlrun_names()
+        return self
+
+    def _replace_nuclio_function_names_with_mlrun_names(self):
+        # replace function names from nuclio names to mlrun names
+        # and adds mlrun function URI's to an api gateway annotations
+        # so when we then get api gateway entity from nuclio, we are able to get mlrun function names
+        mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
+        if mlrun_functions:
+            mlrun_function_uris = (
+                mlrun_functions.split("&")
+                if "&" in mlrun_functions
+                else [mlrun_functions]
+            )
+            if len(mlrun_function_uris) != len(self.spec.upstreams):
+                raise mlrun.errors.MLRunValueError(
+                    "Error when translating nuclio names to mlrun names in api gateway:"
+                    "  number of functions doesn't match the mlrun functions in annotation"
+                )
+            for i in range(len(mlrun_function_uris)):
+                self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
+        return self
+
+    def _replace_nuclio_api_gateway_name_with_mlrun_name(self):
+        # replace api gateway name
+        # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
+        # remove the project prefix from the name if it exists
+        project_name = self.metadata.labels.get(
+            mlrun_constants.MLRunInternalLabels.nuclio_project_name
+        )
+        if project_name and self.spec.name.startswith(f"{project_name}-"):
+            self.spec.name = self.spec.name[len(project_name) + 1 :]
+            self.metadata.name = self.spec.name
+        return self
+
+    def _enrich_mlrun_function_names(self):
+        # enrich mlrun names with nuclio prefixes
+        # and add mlrun function's URIs to Nuclio function annotations
         upstream_with_nuclio_names = []
         mlrun_function_uris = []
         for upstream in self.spec.upstreams:
@@ -126,21 +179,16 @@ class APIGateway(_APIGatewayBaseModel):
             )
         return self
 
-    def replace_nuclio_names_with_mlrun_uri(self):
-        mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
-        if mlrun_functions:
-            mlrun_function_uris = (
-                mlrun_functions.split("&")
-                if "&" in mlrun_functions
-                else [mlrun_functions]
-            )
-            if len(mlrun_function_uris) != len(self.spec.upstreams):
-                raise mlrun.errors.MLRunValueError(
-                    "Error when translating nuclio names to mlrun names in api gateway:"
-                    "  number of functions doesn't match the mlrun functions in annotation"
-                )
-            for i in range(len(mlrun_function_uris)):
-                self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
+    def _enrich_api_gateway_mlrun_name(self):
+        # replace api gateway name
+        # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
+        # add the project prefix to the name
+        project_name = self.metadata.labels.get(
+            mlrun_constants.MLRunInternalLabels.nuclio_project_name
+        )
+        if project_name:
+            self.spec.name = generate_api_gateway_name(project_name, self.spec.name)
+            self.metadata.name = self.spec.name
         return self
 
 

mlrun/common/schemas/artifact.py

@@ -15,6 +15,7 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
@@ -58,6 +59,16 @@ class ArtifactIdentifier(pydantic.BaseModel):
     # hash: typing.Optional[str]
 
 
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.ArtifactFormat instead.",
+    category=FutureWarning,
+)
+class ArtifactsFormat(mlrun.common.types.StrEnum):
+    full = "full"
+
+
 class ArtifactMetadata(pydantic.BaseModel):
     key: str
     project: str

mlrun/common/schemas/constants.py

@@ -120,10 +120,13 @@ class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
 
 class RunPartitionByField(mlrun.common.types.StrEnum):
     name = "name"  # Supported for runs objects
+    project_and_name = "project_and_name"  # Supported for runs objects
 
     def to_partition_by_db_field(self, db_cls):
         if self.value == RunPartitionByField.name:
             return db_cls.name
+        elif self.value == RunPartitionByField.project_and_name:
+            return db_cls.project, db_cls.name
         else:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Unknown group by field: {self.value}"

mlrun/common/schemas/feature_store.py

@@ -14,7 +14,7 @@
 #
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+import pydantic
 
 from .auth import AuthorizationResourceTypes, Credentials
 from .object import (
@@ -27,32 +27,42 @@ from .object import (
 )
 
 
-class Feature(BaseModel):
+class FeatureStoreBaseModel(pydantic.BaseModel):
+    """
+    Intermediate base class, in order to override pydantic's configuration, as per
+    https://docs.pydantic.dev/1.10/usage/model_config/#change-behaviour-globally
+    """
+
+    class Config:
+        copy_on_model_validation = "none"
+
+
+class Feature(FeatureStoreBaseModel):
     name: str
     value_type: str
     labels: Optional[dict] = {}
 
     class Config:
-        extra = Extra.allow
+        extra = pydantic.Extra.allow
 
 
-class Entity(BaseModel):
+class Entity(FeatureStoreBaseModel):
     name: str
     value_type: str
     labels: Optional[dict] = {}
 
     class Config:
-        extra = Extra.allow
+        extra = pydantic.Extra.allow
 
 
 class FeatureSetSpec(ObjectSpec):
     entities: list[Entity] = []
     features: list[Feature] = []
-    engine: Optional[str] = Field(default="storey")
+    engine: Optional[str] = pydantic.Field(default="storey")
 
 
-class FeatureSet(BaseModel):
-    kind: ObjectKind = Field(ObjectKind.feature_set, const=True)
+class FeatureSet(FeatureStoreBaseModel):
+    kind: ObjectKind = pydantic.Field(ObjectKind.feature_set, const=True)
     metadata: ObjectMetadata
     spec: FeatureSetSpec
     status: ObjectStatus
@@ -62,7 +72,7 @@ class FeatureSet(BaseModel):
         return AuthorizationResourceTypes.feature_set
 
 
-class EntityRecord(BaseModel):
+class EntityRecord(FeatureStoreBaseModel):
     name: str
     value_type: str
     labels: list[LabelRecord]
@@ -71,7 +81,7 @@ class EntityRecord(BaseModel):
         orm_mode = True
 
 
-class FeatureRecord(BaseModel):
+class FeatureRecord(FeatureStoreBaseModel):
     name: str
     value_type: str
     labels: list[LabelRecord]
@@ -88,44 +98,64 @@ class FeatureSetRecord(ObjectRecord):
         orm_mode = True
 
 
-class FeatureSetsOutput(BaseModel):
+class FeatureSetsOutput(FeatureStoreBaseModel):
     feature_sets: list[FeatureSet]
 
 
-class FeatureSetsTagsOutput(BaseModel):
+class FeatureSetsTagsOutput(FeatureStoreBaseModel):
     tags: list[str] = []
 
 
-class FeatureSetDigestSpec(BaseModel):
+class FeatureSetDigestSpec(FeatureStoreBaseModel):
     entities: list[Entity]
     features: list[Feature]
 
 
-class FeatureSetDigestOutput(BaseModel):
+class FeatureSetDigestOutput(FeatureStoreBaseModel):
     metadata: ObjectMetadata
     spec: FeatureSetDigestSpec
 
 
-class FeatureListOutput(BaseModel):
+class FeatureSetDigestSpecV2(FeatureStoreBaseModel):
+    entities: list[Entity]
+
+
+class FeatureSetDigestOutputV2(FeatureStoreBaseModel):
+    feature_set_index: int
+    metadata: ObjectMetadata
+    spec: FeatureSetDigestSpecV2
+
+
+class FeatureListOutput(FeatureStoreBaseModel):
     feature: Feature
     feature_set_digest: FeatureSetDigestOutput
 
 
-class FeaturesOutput(BaseModel):
+class FeaturesOutput(FeatureStoreBaseModel):
     features: list[FeatureListOutput]
 
 
-class EntityListOutput(BaseModel):
+class FeaturesOutputV2(FeatureStoreBaseModel):
+    features: list[Feature]
+    feature_set_digests: list[FeatureSetDigestOutputV2]
+
+
+class EntityListOutput(FeatureStoreBaseModel):
     entity: Entity
     feature_set_digest: FeatureSetDigestOutput
 
 
-class EntitiesOutput(BaseModel):
+class EntitiesOutputV2(FeatureStoreBaseModel):
+    entities: list[Entity]
+    feature_set_digests: list[FeatureSetDigestOutputV2]
+
+
+class EntitiesOutput(FeatureStoreBaseModel):
     entities: list[EntityListOutput]
 
 
-class FeatureVector(BaseModel):
-    kind: ObjectKind = Field(ObjectKind.feature_vector, const=True)
+class FeatureVector(FeatureStoreBaseModel):
+    kind: ObjectKind = pydantic.Field(ObjectKind.feature_vector, const=True)
     metadata: ObjectMetadata
     spec: ObjectSpec
     status: ObjectStatus
@@ -139,39 +169,39 @@ class FeatureVectorRecord(ObjectRecord):
     pass
 
 
-class FeatureVectorsOutput(BaseModel):
+class FeatureVectorsOutput(FeatureStoreBaseModel):
     feature_vectors: list[FeatureVector]
 
 
-class FeatureVectorsTagsOutput(BaseModel):
+class FeatureVectorsTagsOutput(FeatureStoreBaseModel):
     tags: list[str] = []
 
 
-class DataSource(BaseModel):
+class DataSource(FeatureStoreBaseModel):
     kind: str
     name: str
     path: str
 
     class Config:
-        extra = Extra.allow
+        extra = pydantic.Extra.allow
 
 
-class DataTarget(BaseModel):
+class DataTarget(FeatureStoreBaseModel):
     kind: str
     name: str
     path: Optional[str]
 
     class Config:
-        extra = Extra.allow
+        extra = pydantic.Extra.allow
 
 
-class FeatureSetIngestInput(BaseModel):
+class FeatureSetIngestInput(FeatureStoreBaseModel):
     source: Optional[DataSource]
     targets: Optional[list[DataTarget]]
     infer_options: Optional[int]
     credentials: Credentials = Credentials()
 
 
-class FeatureSetIngestOutput(BaseModel):
+class FeatureSetIngestOutput(FeatureStoreBaseModel):
     feature_set: FeatureSet
     run_object: dict