mlrun 1.7.0rc23__py3-none-any.whl → 1.7.0rc25__py3-none-any.whl

mlrun/__main__.py

@@ -102,7 +102,9 @@ def main():
 )
 @click.option("--uid", help="unique run ID")
 @click.option("--name", help="run name")
-@click.option("--workflow", help="workflow name/id")
+@click.option(
+    "--workflow", help="sets the run labels to match the given workflow name/id"
+)
 @click.option("--project", help="project name/id")
 @click.option("--db", default="", help="save run results to path or DB url")
 @click.option(

mlrun/common/formatters/__init__.py

@@ -17,3 +17,4 @@ from .artifact import ArtifactFormat  # noqa
 from .function import FunctionFormat  # noqa
 from .pipeline import PipelineFormat  # noqa
 from .project import ProjectFormat  # noqa
+from .run import RunFormat  # noqa

mlrun/common/formatters/artifact.py

@@ -13,9 +13,32 @@
 # limitations under the License.
 #
 
+import typing
+
 import mlrun.common.types
 
+from .base import ObjectFormat
+
+
+class ArtifactFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    minimal = "minimal"
 
-# TODO: add a format that returns a minimal response with ObjectFormat
-class ArtifactFormat(mlrun.common.types.StrEnum):
-    full = "full"
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            ArtifactFormat.full: None,
+            ArtifactFormat.minimal: ArtifactFormat.filter_obj_method(
+                [
+                    "kind",
+                    "metadata",
+                    "status",
+                    "project",
+                    "spec.producer",
+                    "spec.db_key",
+                    "spec.size",
+                    "spec.framework",
+                    "spec.metrics",
+                    "spec.target_path",
+                ]
+            ),
+        }[_format]

mlrun/common/formatters/run.py

@@ -0,0 +1,26 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import mlrun.common.types
+from mlrun.common.formatters.base import ObjectFormat
+
+
+class RunFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    # No enrichment, data is pulled as-is from the database.
+    standard = "standard"
+
+    # Performs run enrichment, including the run's artifacts. Only available for the `get` run API.
+    full = "full"

mlrun/common/helpers.py

@@ -34,3 +34,14 @@ def parse_versioned_object_uri(
         uri = uri[:loc]
 
     return project, uri, tag, hash_key
+
+
+def generate_api_gateway_name(project: str, name: str) -> str:
+    """
+    Generate a unique (within project) api gateway name
+    :param project: project name
+    :param name: api gateway name
+
+    :return: the resolved api gateway name
+    """
+    return f"{project}-{name}" if project else name

mlrun/common/schemas/__init__.py

@@ -91,7 +91,9 @@ from .feature_store import (
     FeatureRecord,
     FeatureSet,
     FeatureSetDigestOutput,
+    FeatureSetDigestOutputV2,
     FeatureSetDigestSpec,
+    FeatureSetDigestSpecV2,
     FeatureSetIngestInput,
     FeatureSetIngestOutput,
     FeatureSetRecord,

mlrun/common/schemas/alert.py

@@ -104,7 +104,7 @@ class AlertCriteria(pydantic.BaseModel):
         pydantic.Field(
             description="Number of events to wait until notification is sent"
         ),
-    ] = 0
+    ] = 1
     period: Annotated[
         str,
         pydantic.Field(

mlrun/common/schemas/api_gateway.py

@@ -17,8 +17,10 @@ from typing import Optional
 
 import pydantic
 
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.types
 from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
+from mlrun.common.helpers import generate_api_gateway_name
 
 
 class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
@@ -100,7 +102,51 @@ class APIGateway(_APIGatewayBaseModel):
             if upstream.nucliofunction.get("name")
         ]
 
-    def enrich_mlrun_function_names(self):
+    def enrich_mlrun_names(self):
+        self._enrich_api_gateway_mlrun_name()
+        self._enrich_mlrun_function_names()
+        return self
+
+    def replace_nuclio_names_with_mlrun_names(self):
+        self._replace_nuclio_api_gateway_name_with_mlrun_name()
+        self._replace_nuclio_function_names_with_mlrun_names()
+        return self
+
+    def _replace_nuclio_function_names_with_mlrun_names(self):
+        # replace function names from nuclio names to mlrun names
+        # and adds mlrun function URI's to an api gateway annotations
+        # so when we then get api gateway entity from nuclio, we are able to get mlrun function names
+        mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
+        if mlrun_functions:
+            mlrun_function_uris = (
+                mlrun_functions.split("&")
+                if "&" in mlrun_functions
+                else [mlrun_functions]
+            )
+            if len(mlrun_function_uris) != len(self.spec.upstreams):
+                raise mlrun.errors.MLRunValueError(
+                    "Error when translating nuclio names to mlrun names in api gateway:"
+                    "  number of functions doesn't match the mlrun functions in annotation"
+                )
+            for i in range(len(mlrun_function_uris)):
+                self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
+        return self
+
+    def _replace_nuclio_api_gateway_name_with_mlrun_name(self):
+        # replace api gateway name
+        # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
+        # remove the project prefix from the name if it exists
+        project_name = self.metadata.labels.get(
+            mlrun_constants.MLRunInternalLabels.nuclio_project_name
+        )
+        if project_name and self.spec.name.startswith(f"{project_name}-"):
+            self.spec.name = self.spec.name[len(project_name) + 1 :]
+            self.metadata.name = self.spec.name
+        return self
+
+    def _enrich_mlrun_function_names(self):
+        # enrich mlrun names with nuclio prefixes
+        # and add mlrun function's URIs to Nuclio function annotations
         upstream_with_nuclio_names = []
         mlrun_function_uris = []
         for upstream in self.spec.upstreams:
@@ -126,21 +172,16 @@ class APIGateway(_APIGatewayBaseModel):
             )
         return self
 
-    def replace_nuclio_names_with_mlrun_uri(self):
-        mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
-        if mlrun_functions:
-            mlrun_function_uris = (
-                mlrun_functions.split("&")
-                if "&" in mlrun_functions
-                else [mlrun_functions]
-            )
-            if len(mlrun_function_uris) != len(self.spec.upstreams):
-                raise mlrun.errors.MLRunValueError(
-                    "Error when translating nuclio names to mlrun names in api gateway:"
-                    "  number of functions doesn't match the mlrun functions in annotation"
-                )
-            for i in range(len(mlrun_function_uris)):
-                self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
+    def _enrich_api_gateway_mlrun_name(self):
+        # replace api gateway name
+        # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
+        # add the project prefix to the name
+        project_name = self.metadata.labels.get(
+            mlrun_constants.MLRunInternalLabels.nuclio_project_name
+        )
+        if project_name:
+            self.spec.name = generate_api_gateway_name(project_name, self.spec.name)
+            self.metadata.name = self.spec.name
         return self
 
 

mlrun/common/schemas/artifact.py

@@ -15,6 +15,7 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
@@ -58,6 +59,16 @@ class ArtifactIdentifier(pydantic.BaseModel):
     # hash: typing.Optional[str]
 
 
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.ArtifactFormat instead.",
+    category=FutureWarning,
+)
+class ArtifactsFormat(mlrun.common.types.StrEnum):
+    full = "full"
+
+
 class ArtifactMetadata(pydantic.BaseModel):
     key: str
     project: str

mlrun/common/schemas/feature_store.py

@@ -46,16 +46,6 @@ class Feature(FeatureStoreBaseModel):
         extra = pydantic.Extra.allow
 
 
-class QualifiedFeature(FeatureStoreBaseModel):
-    name: str
-    value_type: str
-    feature_set_index: int
-    labels: Optional[dict] = {}
-
-    class Config:
-        extra = pydantic.Extra.allow
-
-
 class Entity(FeatureStoreBaseModel):
     name: str
     value_type: str
@@ -65,16 +55,6 @@ class Entity(FeatureStoreBaseModel):
         extra = pydantic.Extra.allow
 
 
-class QualifiedEntity(FeatureStoreBaseModel):
-    name: str
-    value_type: str
-    feature_set_index: int
-    labels: Optional[dict] = {}
-
-    class Config:
-        extra = pydantic.Extra.allow
-
-
 class FeatureSetSpec(ObjectSpec):
     entities: list[Entity] = []
     features: list[Feature] = []
@@ -156,7 +136,7 @@ class FeaturesOutput(FeatureStoreBaseModel):
 
 
 class FeaturesOutputV2(FeatureStoreBaseModel):
-    features: list[QualifiedFeature]
+    features: list[Feature]
     feature_set_digests: list[FeatureSetDigestOutputV2]
 
 
@@ -166,7 +146,7 @@ class EntityListOutput(FeatureStoreBaseModel):
 
 
 class EntitiesOutputV2(FeatureStoreBaseModel):
-    entities: list[QualifiedEntity]
+    entities: list[Entity]
     feature_set_digests: list[FeatureSetDigestOutputV2]
 
 

mlrun/common/schemas/pipeline.py

@@ -15,6 +15,22 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
+
+import mlrun.common.types
+
+
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.PipelineFormat instead.",
+    category=FutureWarning,
+)
+class PipelinesFormat(mlrun.common.types.StrEnum):
+    full = "full"
+    metadata_only = "metadata_only"
+    summary = "summary"
+    name_only = "name_only"
 
 
 class PipelinesPagination(str):

mlrun/common/schemas/project.py

@@ -16,6 +16,7 @@ import datetime
 import typing
 
 import pydantic
+from deprecated import deprecated
 
 import mlrun.common.types
 
@@ -23,6 +24,22 @@ from .common import ImageBuilder
 from .object import ObjectKind, ObjectStatus
 
 
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.ProjectsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.ProjectFormat instead.",
+    category=FutureWarning,
+)
+class ProjectsFormat(mlrun.common.types.StrEnum):
+    full = "full"
+    name_only = "name_only"
+    # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
+    # and is used for faster response times (in the UI)
+    minimal = "minimal"
+    # internal - allowed only in follower mode, only for the leader for upgrade purposes
+    leader = "leader"
+
+
 class ProjectMetadata(pydantic.BaseModel):
     name: str
     created: typing.Optional[datetime.datetime] = None

mlrun/common/schemas/runs.py

@@ -15,9 +15,26 @@
 import typing
 
 import pydantic
+from deprecated import deprecated
+
+import mlrun.common.types
 
 
 class RunIdentifier(pydantic.BaseModel):
     kind: typing.Literal["run"] = "run"
     uid: typing.Optional[str]
     iter: typing.Optional[int]
+
+
+@deprecated(
+    version="1.7.0",
+    reason="mlrun.common.schemas.RunsFormat is deprecated and will be removed in 1.9.0. "
+    "Use mlrun.common.formatters.RunFormat instead.",
+    category=FutureWarning,
+)
+class RunsFormat(mlrun.common.types.StrEnum):
+    # No enrichment, data is pulled as-is from the database.
+    standard = "standard"
+
+    # Performs run enrichment, including the run's artifacts. Only available for the `get` run API.
+    full = "full"

mlrun/common/types.py

@@ -30,3 +30,8 @@ class HTTPMethod(StrEnum):
     GET = "GET"
     POST = "POST"
     DELETE = "DELETE"
+
+
+class Operation(StrEnum):
+    ADD = "add"
+    REMOVE = "remove"

mlrun/config.py

@@ -707,7 +707,7 @@ default_config = {
         "mode": "enabled",
         # maximum number of alerts we allow to be configured.
         # user will get an error when exceeding this
-        "max_allowed": 1000,
+        "max_allowed": 10000,
     },
     "auth_with_client_id": {
         "enabled": False,
@@ -938,24 +938,6 @@ class Config:
                 f"is not allowed for iguazio version: {igz_version} < 3.5.1"
             )
 
-    def resolve_kfp_url(self, namespace=None):
-        if config.kfp_url:
-            return config.kfp_url
-        igz_version = self.get_parsed_igz_version()
-        # TODO: When Iguazio 3.4 will deprecate we can remove this line
-        if igz_version and igz_version <= semver.VersionInfo.parse("3.6.0-b1"):
-            if namespace is None:
-                if not config.namespace:
-                    raise mlrun.errors.MLRunNotFoundError(
-                        "For KubeFlow Pipelines to function, a namespace must be configured"
-                    )
-                namespace = config.namespace
-            # When instead of host we provided namespace we tackled this issue
-            # https://github.com/canonical/bundle-kubeflow/issues/412
-            # TODO: When we'll move to kfp 1.4.0 (server side) it should be resolved
-            return f"http://ml-pipeline.{namespace}.svc.cluster.local:8888"
-        return None
-
     def resolve_chief_api_url(self) -> str:
         if self.httpdb.clusterization.chief.url:
             return self.httpdb.clusterization.chief.url

mlrun/datastore/targets.py

@@ -696,6 +696,7 @@ class BaseStoreTarget(DataTargetBase):
             self.kind, self.name, self.get_target_templated_path()
         )
         target = self._target
+        target.attributes = self.attributes
         target.run_id = self.run_id
         target.status = status or target.status or "created"
         target.updated = now_date().isoformat()
@@ -727,8 +728,18 @@ class BaseStoreTarget(DataTargetBase):
         raise NotImplementedError()
 
     def purge(self):
+        """
+        Delete the files of the target.
+
+        Do not use this function directly from the sdk. Use FeatureSet.purge_targets.
+        """
         store, path_in_store, target_path = self._get_store_and_path()
-        store.rm(target_path, recursive=True)
+        if path_in_store not in ["", "/"]:
+            store.rm(path_in_store, recursive=True)
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Unable to delete target. Please Use purge_targets from FeatureSet object."
+            )
 
     def as_df(
         self,

mlrun/db/base.py

@@ -16,6 +16,8 @@ import datetime
 from abc import ABC, abstractmethod
 from typing import Optional, Union
 
+from deprecated import deprecated
+
 import mlrun.alerts
 import mlrun.common
 import mlrun.common.formatters
@@ -56,7 +58,13 @@ class RunDBInterface(ABC):
         pass
 
     @abstractmethod
-    def read_run(self, uid, project="", iter=0):
+    def read_run(
+        self,
+        uid: str,
+        project: str = "",
+        iter: int = 0,
+        format_: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
+    ):
         pass
 
     @abstractmethod
@@ -103,7 +111,16 @@ class RunDBInterface(ABC):
         pass
 
     @abstractmethod
-    def read_artifact(self, key, tag="", iter=None, project="", tree=None, uid=None):
+    def read_artifact(
+        self,
+        key,
+        tag="",
+        iter=None,
+        project="",
+        tree=None,
+        uid=None,
+        format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
+    ):
         pass
 
     @abstractmethod
@@ -120,6 +137,7 @@ class RunDBInterface(ABC):
         kind: str = None,
         category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
         tree: str = None,
+        format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
     ):
         pass
 
@@ -302,6 +320,12 @@ class RunDBInterface(ABC):
     ) -> dict:
         pass
 
+    # TODO: remove in 1.9.0
+    @deprecated(
+        version="1.9.0",
+        reason="'list_features' will be removed in 1.9.0, use 'list_features_v2' instead",
+        category=FutureWarning,
+    )
     @abstractmethod
     def list_features(
         self,
@@ -313,6 +337,23 @@ class RunDBInterface(ABC):
     ) -> mlrun.common.schemas.FeaturesOutput:
         pass
 
+    @abstractmethod
+    def list_features_v2(
+        self,
+        project: str,
+        name: str = None,
+        tag: str = None,
+        entities: list[str] = None,
+        labels: list[str] = None,
+    ) -> mlrun.common.schemas.FeaturesOutputV2:
+        pass
+
+    # TODO: remove in 1.9.0
+    @deprecated(
+        version="1.9.0",
+        reason="'list_entities' will be removed in 1.9.0, use 'list_entities_v2' instead",
+        category=FutureWarning,
+    )
     @abstractmethod
     def list_entities(
         self,
@@ -323,6 +364,16 @@ class RunDBInterface(ABC):
     ) -> mlrun.common.schemas.EntitiesOutput:
         pass
 
+    @abstractmethod
+    def list_entities_v2(
+        self,
+        project: str,
+        name: str = None,
+        tag: str = None,
+        labels: list[str] = None,
+    ) -> mlrun.common.schemas.EntitiesOutputV2:
+        pass
+
     @abstractmethod
     def list_feature_sets(
         self,