mlrun 1.6.4rc7__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/common/formatters/base.py

@@ -0,0 +1,113 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.errors
+
+
+class ObjectFormat:
+    """
+    MLRun object formatter. Any class that inherits from this class should implement the `format_method` method
+    to specify the formatting method for each format.
+    A `filter_obj_method` utility method is provided to filter the object based on a list of keys.
+    """
+
+    full = "full"
+
+    @staticmethod
+    def format_method(format_: str) -> typing.Optional[typing.Callable]:
+        """
+        Get the formatting method for the provided format.
+        A `None` value signifies a pass-through formatting method (no formatting).
+        :param format_: The format as a string representation.
+        :return: The formatting method.
+        """
+        return {
+            ObjectFormat.full: None,
+        }[format_]
+
+    @classmethod
+    def format_obj(
+        cls,
+        obj: typing.Any,
+        format_: str,
+        exclude_formats: typing.Optional[list[str]] = None,
+    ) -> typing.Any:
+        """
+        Format the provided object based on the provided format.
+        :param obj: The object to format.
+        :param format_: The format as a string representation.
+        :param exclude_formats: A list of formats to exclude from the formatting process. If the provided format is in
+                                this list, an invalid format exception will be raised.
+        """
+        exclude_formats = exclude_formats or []
+        format_ = format_ or cls.full
+        invalid_format_exc = mlrun.errors.MLRunBadRequestError(
+            f"Provided format is not supported. format={format_}"
+        )
+
+        if format_ in exclude_formats:
+            raise invalid_format_exc
+
+        try:
+            format_method = cls.format_method(format_)
+        except KeyError:
+            raise invalid_format_exc
+
+        if not format_method:
+            return obj
+
+        return format_method(obj)
+
+    @staticmethod
+    def filter_obj_method(_filter: list[str]) -> typing.Callable:
+        """
+        Returns a method that filters the object based on the provided list of keys.
+        The keys should be in a dot-separated format, denoting the path within the dictionary to the desired key.
+        The object maintains its structure, with the filtered keys and their values, while all other keys are removed.
+        :param _filter: The list of keys to filter by.
+                        Example:
+                        [
+                            "kind",
+                            "metadata.name",
+                            "spec.something.else",
+                        ]
+
+        :return: The filtering method.
+        """
+
+        def _filter_method(obj: dict) -> dict:
+            formatted_obj = {}
+            for key in _filter:
+                key_list = key.split(".")
+                obj_recursive_iterator = obj
+                formatted_obj_recursive_iterator = formatted_obj
+                for idx, key in enumerate(key_list):
+                    if key not in obj_recursive_iterator:
+                        break
+                    value = (
+                        {} if idx < len(key_list) - 1 else obj_recursive_iterator[key]
+                    )
+                    formatted_obj_recursive_iterator.setdefault(key, value)
+
+                    obj_recursive_iterator = obj_recursive_iterator[key]
+                    formatted_obj_recursive_iterator = formatted_obj_recursive_iterator[
+                        key
+                    ]
+
+            return formatted_obj
+
+        return _filter_method

mlrun/common/formatters/feature_set.py

@@ -0,0 +1,33 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class FeatureSetFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    minimal = "minimal"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            FeatureSetFormat.full: None,
+            FeatureSetFormat.minimal: FeatureSetFormat.filter_obj_method(
+                ["kind", "metadata", "spec", "status.state"]
+            ),
+        }[_format]

mlrun/common/formatters/function.py

@@ -0,0 +1,46 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class FunctionFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    minimal = "minimal"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            FunctionFormat.full: None,
+            FunctionFormat.minimal: FunctionFormat.filter_obj_method(
+                [
+                    "kind",
+                    "metadata",
+                    "status",
+                    "spec.description",
+                    "spec.command",
+                    "spec.image",
+                    "spec.default_handler",
+                    "spec.default_class",
+                    "spec.graph",
+                    "spec.preemption_mode",
+                    "spec.node_selector",
+                    "spec.priority_class_name",
+                ]
+            ),
+        }[_format]

mlrun/common/formatters/pipeline.py

@@ -0,0 +1,53 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun_pipelines.common.ops
+import mlrun_pipelines.models
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class PipelineFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+    metadata_only = "metadata_only"
+    name_only = "name_only"
+    summary = "summary"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        def _full(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return run.to_dict()
+
+        def _metadata_only(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return mlrun.utils.helpers.format_run(run, with_project=True)
+
+        def _name_only(run: mlrun_pipelines.models.PipelineRun) -> str:
+            return run.get("name")
+
+        def _summary(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return mlrun_pipelines.common.ops.format_summary_from_kfp_run(
+                run, run["project"]
+            )
+
+        return {
+            PipelineFormat.full: _full,
+            PipelineFormat.metadata_only: _metadata_only,
+            PipelineFormat.name_only: _name_only,
+            PipelineFormat.summary: _summary,
+        }[_format]

mlrun/common/formatters/project.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.schemas
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class ProjectFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+    name_only = "name_only"
+    # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
+    # and is used for faster response times (in the UI)
+    minimal = "minimal"
+    # internal - allowed only in follower mode, only for the leader for upgrade purposes
+    leader = "leader"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        def _name_only(project: mlrun.common.schemas.Project) -> str:
+            return project.metadata.name
+
+        def _minimal(
+            project: mlrun.common.schemas.Project,
+        ) -> mlrun.common.schemas.Project:
+            project.spec.functions = None
+            project.spec.workflows = None
+            project.spec.artifacts = None
+            return project
+
+        return {
+            ProjectFormat.full: None,
+            ProjectFormat.name_only: _name_only,
+            ProjectFormat.minimal: _minimal,
+            ProjectFormat.leader: None,
+        }[_format]

mlrun/common/formatters/run.py

@@ -0,0 +1,29 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import mlrun.common.types
+from mlrun.common.formatters.base import ObjectFormat
+
+
+class RunFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    # No enrichment, data is pulled as-is from the database.
+    standard = "standard"
+
+    # Enrich run with full notifications since the notification params are subtracted from the run body.
+    notifications = "notifications"
+
+    # Performs run enrichment, including the run's artifacts. Only available for the `get` run API.
+    full = "full"

mlrun/common/helpers.py

@@ -11,13 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-import typing
 
 
 def parse_versioned_object_uri(
     uri: str, default_project: str = ""
-) -> typing.Tuple[str, str, str, str]:
+) -> tuple[str, str, str, str]:
     project = default_project
     tag = ""
     hash_key = ""
@@ -35,3 +33,14 @@ def parse_versioned_object_uri(
         uri = uri[:loc]
 
     return project, uri, tag, hash_key
+
+
+def generate_api_gateway_name(project: str, name: str) -> str:
+    """
+    Generate a unique (within project) api gateway name
+    :param project: project name
+    :param name: api gateway name
+
+    :return: the resolved api gateway name
+    """
+    return f"{project}-{name}" if project else name

mlrun/common/model_monitoring/helpers.py

@@ -16,6 +16,7 @@ import sys
 import typing
 
 import mlrun.common
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
 from mlrun.common.schemas.model_monitoring import (
     EndpointUID,
     FunctionURI,
@@ -64,7 +65,7 @@ def parse_model_endpoint_store_prefix(store_prefix: str):
 
 
 def parse_monitoring_stream_path(
-    stream_uri: str, project: str, application_name: str = None
+    stream_uri: str, project: str, function_name: str = None
 ):
     if stream_uri.startswith("kafka://"):
         if "?topic" in stream_uri:
@@ -72,14 +73,17 @@ def parse_monitoring_stream_path(
                 "Custom kafka topic is not allowed"
             )
         # Add topic to stream kafka uri
-        if application_name is None:
+        if (
+            function_name is None
+            or function_name == mm_constants.MonitoringFunctionNames.STREAM
+        ):
             stream_uri += f"?topic=monitoring_stream_{project}"
         else:
-            stream_uri += f"?topic=monitoring_stream_{project}_{application_name}"
+            stream_uri += f"?topic=monitoring_stream_{project}_{function_name}"
 
     elif stream_uri.startswith("v3io://") and mlrun.mlconf.is_ce_mode():
         # V3IO is not supported in CE mode, generating a default http stream path
-        if application_name is None:
+        if function_name is None:
             stream_uri = (
                 mlrun.mlconf.model_endpoint_monitoring.default_http_sink.format(
                     project=project, namespace=mlrun.mlconf.namespace
@@ -89,7 +93,7 @@ def parse_monitoring_stream_path(
             stream_uri = (
                 mlrun.mlconf.model_endpoint_monitoring.default_http_sink_app.format(
                     project=project,
-                    application_name=application_name,
+                    application_name=function_name,
                     namespace=mlrun.mlconf.namespace,
                 )
             )

mlrun/{runtimes → common/runtimes}/constants.py

@@ -15,6 +15,10 @@
 import enum
 import typing
 
+import mlrun_pipelines.common.models
+
+import mlrun.common.constants as mlrun_constants
+
 
 class PodPhases:
     """
@@ -75,7 +79,7 @@ class ThresholdStates:
         ]
 
     @staticmethod
-    def from_pod_phase(pod_phase: str, pod: typing.Dict) -> typing.Optional[str]:
+    def from_pod_phase(pod_phase: str, pod: dict) -> typing.Optional[str]:
         if pod_phase == PodPhases.pending:
             if ThresholdStates.is_pod_in_image_pull_backoff(pod):
                 return ThresholdStates.image_pull_backoff
@@ -90,7 +94,7 @@ class ThresholdStates:
         return None
 
     @staticmethod
-    def is_pod_scheduled(pod: typing.Dict):
+    def is_pod_scheduled(pod: dict):
         conditions = pod["status"].get("conditions", []) or []
         for condition in conditions:
             if condition["type"] == "PodScheduled" and condition["status"] == "True":
@@ -98,7 +102,7 @@ class ThresholdStates:
         return False
 
     @staticmethod
-    def is_pod_in_image_pull_backoff(pod: typing.Dict):
+    def is_pod_in_image_pull_backoff(pod: dict):
         container_statuses = pod.get("status").get("container_statuses", []) or []
         for container_status in container_statuses:
             state_waiting = container_status.get("state", {}).get("waiting", {}) or {}
@@ -107,7 +111,7 @@ class ThresholdStates:
         return False
 
 
-class MPIJobCRDVersions(object):
+class MPIJobCRDVersions:
     v1 = "v1"
     v1alpha1 = "v1alpha1"
 
@@ -122,12 +126,12 @@ class MPIJobCRDVersions(object):
     @staticmethod
     def role_label_by_version(version):
         return {
-            MPIJobCRDVersions.v1alpha1: "mpi_role_type",
-            MPIJobCRDVersions.v1: "mpi-job-role",
+            MPIJobCRDVersions.v1alpha1: mlrun_constants.MLRunInternalLabels.mpi_role_type,
+            MPIJobCRDVersions.v1: mlrun_constants.MLRunInternalLabels.mpi_job_role,
         }[version]
 
 
-class RunStates(object):
+class RunStates:
     completed = "completed"
     error = "error"
     running = "running"
@@ -136,6 +140,7 @@ class RunStates(object):
     unknown = "unknown"
     aborted = "aborted"
     aborting = "aborting"
+    skipped = "skipped"
 
     @staticmethod
     def all():
@@ -148,6 +153,7 @@ class RunStates(object):
             RunStates.unknown,
             RunStates.aborted,
             RunStates.aborting,
+            RunStates.skipped,
         ]
 
     @staticmethod
@@ -156,6 +162,7 @@ class RunStates(object):
             RunStates.completed,
             RunStates.error,
             RunStates.aborted,
+            RunStates.skipped,
         ]
 
     @staticmethod
@@ -188,10 +195,31 @@ class RunStates(object):
             # TODO: add aborting state once we have it
         ]
 
+    @staticmethod
+    def run_state_to_pipeline_run_status(run_state: str):
+        if not run_state:
+            return mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified
 
+        if run_state not in RunStates.all():
+            raise ValueError(f"Invalid run state: {run_state}")
+
+        return {
+            RunStates.completed: mlrun_pipelines.common.models.RunStatuses.succeeded,
+            RunStates.error: mlrun_pipelines.common.models.RunStatuses.failed,
+            RunStates.running: mlrun_pipelines.common.models.RunStatuses.running,
+            RunStates.created: mlrun_pipelines.common.models.RunStatuses.pending,
+            RunStates.pending: mlrun_pipelines.common.models.RunStatuses.pending,
+            RunStates.unknown: mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified,
+            RunStates.aborted: mlrun_pipelines.common.models.RunStatuses.canceled,
+            RunStates.aborting: mlrun_pipelines.common.models.RunStatuses.canceling,
+            RunStates.skipped: mlrun_pipelines.common.models.RunStatuses.skipped,
+        }[run_state]
+
+
+# TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
 class RunLabels(enum.Enum):
-    owner = "owner"
-    v3io_user = "v3io_user"
+    owner = mlrun_constants.MLRunInternalLabels.owner
+    v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user
 
     @staticmethod
     def all():

mlrun/common/schemas/__init__.py

@@ -14,12 +14,29 @@
 #
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
+from .alert import (
+    AlertActiveState,
+    AlertConfig,
+    AlertNotification,
+    AlertTemplate,
+    Event,
+)
+from .api_gateway import (
+    APIGateway,
+    APIGatewayAuthenticationMode,
+    APIGatewayBasicAuth,
+    APIGatewayMetadata,
+    APIGatewaysOutput,
+    APIGatewaySpec,
+    APIGatewayState,
+    APIGatewayStatus,
+    APIGatewayUpstream,
+)
 from .artifact import (
     Artifact,
     ArtifactCategories,
     ArtifactIdentifier,
     ArtifactMetadata,
-    ArtifactsFormat,
     ArtifactSpec,
 )
 from .auth import (
@@ -65,6 +82,7 @@ from .events import (
 )
 from .feature_store import (
     EntitiesOutput,
+    EntitiesOutputV2,
     Entity,
     EntityListOutput,
     EntityRecord,
@@ -73,7 +91,9 @@ from .feature_store import (
     FeatureRecord,
     FeatureSet,
     FeatureSetDigestOutput,
+    FeatureSetDigestOutputV2,
     FeatureSetDigestSpec,
+    FeatureSetDigestSpecV2,
     FeatureSetIngestInput,
     FeatureSetIngestOutput,
     FeatureSetRecord,
@@ -81,12 +101,14 @@ from .feature_store import (
     FeatureSetSpec,
     FeatureSetsTagsOutput,
     FeaturesOutput,
+    FeaturesOutputV2,
     FeatureVector,
     FeatureVectorRecord,
     FeatureVectorsOutput,
     FeatureVectorsTagsOutput,
 )
 from .frontend_spec import (
+    ArtifactLimits,
     AuthenticationFeatureFlag,
     FeatureFlags,
     FrontendSpec,
@@ -130,7 +152,8 @@ from .model_monitoring import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
-    TimeSeriesTarget,
+    TSDBTarget,
+    V3IOTSDBTables,
 )
 from .notification import (
     Notification,
@@ -140,23 +163,26 @@ from .notification import (
     SetNotificationRequest,
 )
 from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
-from .pipeline import PipelinesFormat, PipelinesOutput, PipelinesPagination
+from .pagination import PaginationInfo
+from .pipeline import PipelinesOutput, PipelinesPagination
 from .project import (
     IguazioProject,
     Project,
     ProjectDesiredState,
     ProjectMetadata,
+    ProjectOut,
+    ProjectOutput,
     ProjectOwner,
-    ProjectsFormat,
     ProjectsOutput,
     ProjectSpec,
+    ProjectSpecOut,
     ProjectState,
     ProjectStatus,
     ProjectSummariesOutput,
     ProjectSummary,
 )
 from .regex import RegexMatchModes
-from .runs import RunIdentifier, RunsFormat
+from .runs import RunIdentifier
 from .runtime_resource import (
     GroupedByJobRuntimeResourcesOutput,
     GroupedByProjectRuntimeResourcesOutput,