mlrun 1.7.0rc5__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/common/formatters/base.py

@@ -0,0 +1,113 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.errors
+
+
+class ObjectFormat:
+    """
+    MLRun object formatter. Any class that inherits from this class should implement the `format_method` method
+    to specify the formatting method for each format.
+    A `filter_obj_method` utility method is provided to filter the object based on a list of keys.
+    """
+
+    full = "full"
+
+    @staticmethod
+    def format_method(format_: str) -> typing.Optional[typing.Callable]:
+        """
+        Get the formatting method for the provided format.
+        A `None` value signifies a pass-through formatting method (no formatting).
+        :param format_: The format as a string representation.
+        :return: The formatting method.
+        """
+        return {
+            ObjectFormat.full: None,
+        }[format_]
+
+    @classmethod
+    def format_obj(
+        cls,
+        obj: typing.Any,
+        format_: str,
+        exclude_formats: typing.Optional[list[str]] = None,
+    ) -> typing.Any:
+        """
+        Format the provided object based on the provided format.
+        :param obj: The object to format.
+        :param format_: The format as a string representation.
+        :param exclude_formats: A list of formats to exclude from the formatting process. If the provided format is in
+                                this list, an invalid format exception will be raised.
+        """
+        exclude_formats = exclude_formats or []
+        format_ = format_ or cls.full
+        invalid_format_exc = mlrun.errors.MLRunBadRequestError(
+            f"Provided format is not supported. format={format_}"
+        )
+
+        if format_ in exclude_formats:
+            raise invalid_format_exc
+
+        try:
+            format_method = cls.format_method(format_)
+        except KeyError:
+            raise invalid_format_exc
+
+        if not format_method:
+            return obj
+
+        return format_method(obj)
+
+    @staticmethod
+    def filter_obj_method(_filter: list[str]) -> typing.Callable:
+        """
+        Returns a method that filters the object based on the provided list of keys.
+        The keys should be in a dot-separated format, denoting the path within the dictionary to the desired key.
+        The object maintains its structure, with the filtered keys and their values, while all other keys are removed.
+        :param _filter: The list of keys to filter by.
+                        Example:
+                        [
+                            "kind",
+                            "metadata.name",
+                            "spec.something.else",
+                        ]
+
+        :return: The filtering method.
+        """
+
+        def _filter_method(obj: dict) -> dict:
+            formatted_obj = {}
+            for key in _filter:
+                key_list = key.split(".")
+                obj_recursive_iterator = obj
+                formatted_obj_recursive_iterator = formatted_obj
+                for idx, key in enumerate(key_list):
+                    if key not in obj_recursive_iterator:
+                        break
+                    value = (
+                        {} if idx < len(key_list) - 1 else obj_recursive_iterator[key]
+                    )
+                    formatted_obj_recursive_iterator.setdefault(key, value)
+
+                    obj_recursive_iterator = obj_recursive_iterator[key]
+                    formatted_obj_recursive_iterator = formatted_obj_recursive_iterator[
+                        key
+                    ]
+
+            return formatted_obj
+
+        return _filter_method

mlrun/common/formatters/feature_set.py

@@ -0,0 +1,44 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class FeatureSetFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    minimal = "minimal"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            FeatureSetFormat.full: None,
+            FeatureSetFormat.minimal: FeatureSetFormat.filter_obj_method(
+                [
+                    "metadata.name",
+                    "metadata.project",
+                    "metadata.tag",
+                    "metadata.uid",
+                    "metadata.labels",
+                    "spec.entities",
+                    "spec.description",
+                    "spec.targets",
+                    "spec.engine",  # It's not needed by the UI, but we override it anyway to storey if empty
+                    "status.state",
+                ]
+            ),
+        }[_format]

mlrun/common/formatters/function.py

@@ -0,0 +1,46 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class FunctionFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    minimal = "minimal"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            FunctionFormat.full: None,
+            FunctionFormat.minimal: FunctionFormat.filter_obj_method(
+                [
+                    "kind",
+                    "metadata",
+                    "status",
+                    "spec.description",
+                    "spec.command",
+                    "spec.image",
+                    "spec.default_handler",
+                    "spec.default_class",
+                    "spec.graph",
+                    "spec.preemption_mode",
+                    "spec.node_selector",
+                    "spec.priority_class_name",
+                ]
+            ),
+        }[_format]

mlrun/common/formatters/pipeline.py

@@ -0,0 +1,53 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun_pipelines.common.ops
+import mlrun_pipelines.models
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class PipelineFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+    metadata_only = "metadata_only"
+    name_only = "name_only"
+    summary = "summary"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        def _full(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return run.to_dict()
+
+        def _metadata_only(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return mlrun.utils.helpers.format_run(run, with_project=True)
+
+        def _name_only(run: mlrun_pipelines.models.PipelineRun) -> str:
+            return run.get("name")
+
+        def _summary(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return mlrun_pipelines.common.ops.format_summary_from_kfp_run(
+                run, run["project"]
+            )
+
+        return {
+            PipelineFormat.full: _full,
+            PipelineFormat.metadata_only: _metadata_only,
+            PipelineFormat.name_only: _name_only,
+            PipelineFormat.summary: _summary,
+        }[_format]

mlrun/common/formatters/project.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.schemas
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class ProjectFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+    name_only = "name_only"
+    # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
+    # and is used for faster response times (in the UI)
+    minimal = "minimal"
+    # internal - allowed only in follower mode, only for the leader for upgrade purposes
+    leader = "leader"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        def _name_only(project: mlrun.common.schemas.Project) -> str:
+            return project.metadata.name
+
+        def _minimal(
+            project: mlrun.common.schemas.Project,
+        ) -> mlrun.common.schemas.Project:
+            project.spec.functions = None
+            project.spec.workflows = None
+            project.spec.artifacts = None
+            return project
+
+        return {
+            ProjectFormat.full: None,
+            ProjectFormat.name_only: _name_only,
+            ProjectFormat.minimal: _minimal,
+            ProjectFormat.leader: None,
+        }[_format]

mlrun/common/formatters/run.py

@@ -0,0 +1,29 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import mlrun.common.types
+from mlrun.common.formatters.base import ObjectFormat
+
+
+class RunFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    # No enrichment, data is pulled as-is from the database.
+    standard = "standard"
+
+    # Enrich run with full notifications since the notification params are subtracted from the run body.
+    notifications = "notifications"
+
+    # Performs run enrichment, including the run's artifacts. Only available for the `get` run API.
+    full = "full"

mlrun/common/helpers.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 
 def parse_versioned_object_uri(
@@ -34,3 +33,14 @@ def parse_versioned_object_uri(
         uri = uri[:loc]
 
     return project, uri, tag, hash_key
+
+
+def generate_api_gateway_name(project: str, name: str) -> str:
+    """
+    Generate a unique (within project) api gateway name
+    :param project: project name
+    :param name: api gateway name
+
+    :return: the resolved api gateway name
+    """
+    return f"{project}-{name}" if project else name

mlrun/{runtimes → common/runtimes}/constants.py

@@ -15,6 +15,10 @@
 import enum
 import typing
 
+import mlrun_pipelines.common.models
+
+import mlrun.common.constants as mlrun_constants
+
 
 class PodPhases:
     """
@@ -122,8 +126,8 @@ class MPIJobCRDVersions:
     @staticmethod
     def role_label_by_version(version):
         return {
-            MPIJobCRDVersions.v1alpha1: "mpi_role_type",
-            MPIJobCRDVersions.v1: "mpi-job-role",
+            MPIJobCRDVersions.v1alpha1: mlrun_constants.MLRunInternalLabels.mpi_role_type,
+            MPIJobCRDVersions.v1: mlrun_constants.MLRunInternalLabels.mpi_job_role,
         }[version]
 
 
@@ -136,6 +140,7 @@ class RunStates:
     unknown = "unknown"
     aborted = "aborted"
     aborting = "aborting"
+    skipped = "skipped"
 
     @staticmethod
     def all():
@@ -148,6 +153,7 @@ class RunStates:
             RunStates.unknown,
             RunStates.aborted,
             RunStates.aborting,
+            RunStates.skipped,
         ]
 
     @staticmethod
@@ -156,6 +162,7 @@ class RunStates:
             RunStates.completed,
             RunStates.error,
             RunStates.aborted,
+            RunStates.skipped,
         ]
 
     @staticmethod
@@ -188,10 +195,31 @@ class RunStates:
             # TODO: add aborting state once we have it
         ]
 
+    @staticmethod
+    def run_state_to_pipeline_run_status(run_state: str):
+        if not run_state:
+            return mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified
 
+        if run_state not in RunStates.all():
+            raise ValueError(f"Invalid run state: {run_state}")
+
+        return {
+            RunStates.completed: mlrun_pipelines.common.models.RunStatuses.succeeded,
+            RunStates.error: mlrun_pipelines.common.models.RunStatuses.failed,
+            RunStates.running: mlrun_pipelines.common.models.RunStatuses.running,
+            RunStates.created: mlrun_pipelines.common.models.RunStatuses.pending,
+            RunStates.pending: mlrun_pipelines.common.models.RunStatuses.pending,
+            RunStates.unknown: mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified,
+            RunStates.aborted: mlrun_pipelines.common.models.RunStatuses.canceled,
+            RunStates.aborting: mlrun_pipelines.common.models.RunStatuses.canceling,
+            RunStates.skipped: mlrun_pipelines.common.models.RunStatuses.skipped,
+        }[run_state]
+
+
+# TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
 class RunLabels(enum.Enum):
-    owner = "owner"
-    v3io_user = "v3io_user"
+    owner = mlrun_constants.MLRunInternalLabels.owner
+    v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user
 
     @staticmethod
     def all():

mlrun/common/schemas/__init__.py

@@ -14,6 +14,13 @@
 #
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
+from .alert import (
+    AlertActiveState,
+    AlertConfig,
+    AlertNotification,
+    AlertTemplate,
+    Event,
+)
 from .api_gateway import (
     APIGateway,
     APIGatewayAuthenticationMode,
@@ -21,6 +28,7 @@ from .api_gateway import (
     APIGatewayMetadata,
     APIGatewaysOutput,
     APIGatewaySpec,
+    APIGatewayState,
     APIGatewayStatus,
     APIGatewayUpstream,
 )
@@ -29,7 +37,6 @@ from .artifact import (
     ArtifactCategories,
     ArtifactIdentifier,
     ArtifactMetadata,
-    ArtifactsFormat,
     ArtifactSpec,
 )
 from .auth import (
@@ -75,6 +82,7 @@ from .events import (
 )
 from .feature_store import (
     EntitiesOutput,
+    EntitiesOutputV2,
     Entity,
     EntityListOutput,
     EntityRecord,
@@ -83,7 +91,9 @@ from .feature_store import (
     FeatureRecord,
     FeatureSet,
     FeatureSetDigestOutput,
+    FeatureSetDigestOutputV2,
     FeatureSetDigestSpec,
+    FeatureSetDigestSpecV2,
     FeatureSetIngestInput,
     FeatureSetIngestOutput,
     FeatureSetRecord,
@@ -91,12 +101,14 @@ from .feature_store import (
     FeatureSetSpec,
     FeatureSetsTagsOutput,
     FeaturesOutput,
+    FeaturesOutputV2,
     FeatureVector,
     FeatureVectorRecord,
     FeatureVectorsOutput,
     FeatureVectorsTagsOutput,
 )
 from .frontend_spec import (
+    ArtifactLimits,
     AuthenticationFeatureFlag,
     FeatureFlags,
     FrontendSpec,
@@ -124,6 +136,7 @@ from .model_monitoring import (
     EventFieldType,
     EventKeyMetrics,
     Features,
+    FeatureSetFeatures,
     FeatureValues,
     GrafanaColumn,
     GrafanaDataPoint,
@@ -139,7 +152,8 @@ from .model_monitoring import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
-    TimeSeriesTarget,
+    TSDBTarget,
+    V3IOTSDBTables,
 )
 from .notification import (
     Notification,
@@ -149,16 +163,19 @@ from .notification import (
     SetNotificationRequest,
 )
 from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
-from .pipeline import PipelinesFormat, PipelinesOutput, PipelinesPagination
+from .pagination import PaginationInfo
+from .pipeline import PipelinesOutput, PipelinesPagination
 from .project import (
     IguazioProject,
     Project,
     ProjectDesiredState,
     ProjectMetadata,
+    ProjectOut,
+    ProjectOutput,
     ProjectOwner,
-    ProjectsFormat,
     ProjectsOutput,
     ProjectSpec,
+    ProjectSpecOut,
     ProjectState,
     ProjectStatus,
     ProjectSummariesOutput,