mlrun 1.10.0rc8__py3-none-any.whl → 1.10.0rc10__py3-none-any.whl

mlrun/common/constants.py

@@ -76,6 +76,7 @@ class MLRunInternalLabels:
     kind = "kind"
     component = "component"
     mlrun_type = "mlrun__type"
+    original_workflow_id = "original-workflow-id"
 
     owner = "owner"
     v3io_user = "v3io_user"

mlrun/common/db/dialects.py

@@ -0,0 +1,25 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import mlrun.common.types
+
+
+class Dialects(mlrun.common.types.StrEnum):
+    MYSQL = "mysql"
+    POSTGRESQL = "postgresql"
+    SQLITE = "sqlite"
+
+    @classmethod
+    def all(cls) -> list[str]:
+        """Return all dialects as a list of strings."""
+        return [dialect.value for dialect in cls]

mlrun/common/schemas/__init__.py

@@ -218,6 +218,7 @@ from .serving import ModelRunnerStepData, MonitoringData
 from .tag import Tag, TagObjects
 from .workflow import (
     GetWorkflowResponse,
+    RerunWorkflowRequest,
     WorkflowRequest,
     WorkflowResponse,
     WorkflowSpec,

mlrun/common/schemas/function.py

@@ -47,6 +47,7 @@ class FunctionState:
 
     # for pipeline steps
     skipped = "skipped"
+    initialized = "initialized"
 
     @classmethod
     def get_function_state_from_pod_state(cls, pod_state: str):

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -15,7 +15,9 @@ import abc
 import json
 from datetime import datetime
 from typing import Any, NamedTuple, Optional, TypeVar
+from uuid import UUID
 
+from pydantic import validator  # use `validator` if you’re still on Pydantic v1
 from pydantic.v1 import BaseModel, Field, constr
 
 # TODO: remove the unused import below after `mlrun.datastore` and `mlrun.utils` usage is removed.
@@ -121,6 +123,12 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
     def mutable_fields(cls):
         return ["labels"]
 
+    @validator("uid", pre=True)
+    def _uid_to_str(cls, v):  # noqa: N805
+        if isinstance(v, UUID):
+            return str(v)
+        return v
+
 
 class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
     model_class: Optional[str] = ""

mlrun/common/schemas/partition.py

@@ -14,10 +14,10 @@
 
 from datetime import datetime, timedelta
 
-from mlrun.common.types import StrEnum
+import mlrun.common.types
 
 
-class PartitionInterval(StrEnum):
+class PartitionInterval(mlrun.common.types.StrEnum):
     DAY = "DAY"
     MONTH = "MONTH"
     YEARWEEK = "YEARWEEK"
@@ -44,6 +44,8 @@ class PartitionInterval(StrEnum):
             return timedelta(days=30)
         elif self == PartitionInterval.YEARWEEK:
             return timedelta(weeks=1)
+        else:
+            raise ValueError(f"Unsupported PartitionInterval: {self}")
 
     @classmethod
     def from_expression(cls, partition_expression: str):
@@ -83,7 +85,7 @@ class PartitionInterval(StrEnum):
         current_datetime = start_datetime
 
         for _ in range(partition_number):
-            partition_name = self.get_partition_name(current_datetime)
+            partition_name = f"p{self.get_partition_name(current_datetime)}"
             partition_boundary_date = self.get_next_partition_time(current_datetime)
             partition_value = self.get_partition_name(partition_boundary_date)
             partitioning_information_list.append((partition_name, partition_value))
@@ -109,6 +111,8 @@ class PartitionInterval(StrEnum):
             return (current_datetime.replace(day=1) + timedelta(days=32)).replace(day=1)
         elif self == PartitionInterval.YEARWEEK:
             return current_datetime + timedelta(weeks=1)
+        else:
+            raise ValueError(f"Unsupported PartitionInterval: {self}")
 
     def get_partition_name(self, current_datetime: datetime) -> str:
         if self == PartitionInterval.DAY:
@@ -118,6 +122,8 @@ class PartitionInterval(StrEnum):
         elif self == PartitionInterval.YEARWEEK:
             year, week, _ = current_datetime.isocalendar()
             return f"{year}{week:02d}"
+        else:
+            raise ValueError(f"Unsupported PartitionInterval: {self}")
 
     def get_partition_expression(self, column_name: str):
         if self == PartitionInterval.YEARWEEK:
@@ -130,6 +136,8 @@ class PartitionInterval(StrEnum):
             # generates value in format %Y%m in mysql
             # mysql query example: `select YEAR(NOW())*100 + MONTH(NOW());`
             return f"YEAR({column_name}) * 100 + MONTH({column_name})"
+        else:
+            raise ValueError(f"Unsupported PartitionInterval: {self}")
 
     def get_number_of_partitions(self, days: int) -> int:
         # Calculate the number partitions based on given number of days
@@ -140,3 +148,5 @@ class PartitionInterval(StrEnum):
             return int(days / 30.44)
         elif self == PartitionInterval.YEARWEEK:
             return int(days / 7)
+        else:
+            raise ValueError(f"Unsupported PartitionInterval: {self}")

mlrun/common/schemas/workflow.py

@@ -46,6 +46,13 @@ class WorkflowRequest(pydantic.v1.BaseModel):
     notifications: typing.Optional[list[Notification]] = None
 
 
+class RerunWorkflowRequest(pydantic.v1.BaseModel):
+    run_name: typing.Optional[str] = (None,)
+    run_id: typing.Optional[str] = (None,)
+    notifications: typing.Optional[list[Notification]] = None
+    workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
+
+
 class WorkflowResponse(pydantic.v1.BaseModel):
     project: str = None
     name: str = None

mlrun/datastore/utils.py

@@ -150,7 +150,6 @@ def _generate_sql_query_with_time_filter(
     table = sqlalchemy.Table(
         table_name,
         sqlalchemy.MetaData(),
-        autoload=True,
         autoload_with=engine,
     )
     query = sqlalchemy.select(table)

mlrun/db/__init__.py

@@ -14,6 +14,7 @@
 from os import environ
 
 from ..config import config
+from . import sql_types
 from .base import RunDBError, RunDBInterface  # noqa
 
 

mlrun/db/base.py

@@ -638,6 +638,16 @@ class RunDBInterface(ABC):
     ):
         pass
 
+    @abstractmethod
+    def retry_pipeline(
+        self,
+        run_id: str,
+        project: str,
+        namespace: Optional[str] = None,
+        timeout: int = 30,
+    ):
+        pass
+
     @abstractmethod
     def list_project_secrets(
         self,
@@ -1034,6 +1044,13 @@ class RunDBInterface(ABC):
     ):
         pass
 
+    def get_project_background_task(
+        self,
+        project: str,
+        name: str,
+    ) -> mlrun.common.schemas.BackgroundTask:
+        pass
+
     @abstractmethod
     def submit_workflow(
         self,

mlrun/db/nopdb.py

@@ -524,6 +524,15 @@ class NopDB(RunDBInterface):
     ):
         pass
 
+    def retry_pipeline(
+        self,
+        run_id: str,
+        project: str,
+        namespace: Optional[str] = None,
+        timeout: int = 30,
+    ):
+        pass
+
     def list_pipelines(
         self,
         project: str,

mlrun/db/sql_types.py

@@ -0,0 +1,160 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This module provides SQLAlchemy TypeDecorator subclasses that are aware of
+database dialects (MySQL, PostgreSQL, SQLite) and automatically select
+appropriate native types (e.g., UUID, BLOB, TIMESTAMP with precision) or
+fallbacks (e.g., hex-string storage) to ensure consistent behavior across
+different database backends.
+"""
+
+import uuid
+from typing import Any, Optional, Union
+
+import sqlalchemy.types
+from sqlalchemy import CHAR, Text
+from sqlalchemy.dialects.mysql import DATETIME as MYSQL_DATETIME
+from sqlalchemy.dialects.mysql import MEDIUMBLOB
+from sqlalchemy.dialects.postgresql import BYTEA
+from sqlalchemy.dialects.postgresql import TIMESTAMP as PG_TIMESTAMP
+from sqlalchemy.dialects.postgresql import UUID as PG_UUID
+from sqlalchemy.engine.interfaces import Dialect
+from sqlalchemy.types import TypeDecorator
+
+import mlrun.common.db.dialects
+
+
+class DateTime(TypeDecorator):
+    impl = sqlalchemy.types.DateTime
+    cache_ok = True
+    precision: int = 3
+
+    def load_dialect_impl(
+        self,
+        dialect: Dialect,
+    ) -> sqlalchemy.types.TypeEngine:
+        if dialect.name == mlrun.common.db.dialects.Dialects.MYSQL:
+            return dialect.type_descriptor(
+                MYSQL_DATETIME(
+                    fsp=self.precision,
+                    timezone=True,
+                )
+            )
+        if dialect.name == mlrun.common.db.dialects.Dialects.POSTGRESQL:
+            return dialect.type_descriptor(
+                PG_TIMESTAMP(
+                    precision=self.precision,
+                    timezone=True,
+                )
+            )
+        return dialect.type_descriptor(sqlalchemy.types.DateTime)
+
+
+class MicroSecondDateTime(DateTime):
+    cache_ok = True
+    precision: int = 6
+
+
+class Blob(TypeDecorator):
+    impl = sqlalchemy.types.LargeBinary
+    cache_ok = True
+
+    def load_dialect_impl(
+        self,
+        dialect: Dialect,
+    ) -> sqlalchemy.types.TypeEngine:
+        if dialect.name == mlrun.common.db.dialects.Dialects.MYSQL:
+            return dialect.type_descriptor(MEDIUMBLOB)
+        if dialect.name == mlrun.common.db.dialects.Dialects.POSTGRESQL:
+            return dialect.type_descriptor(BYTEA)
+        return dialect.type_descriptor(self.impl)
+
+
+class Utf8BinText(TypeDecorator):
+    impl = Text
+    cache_ok = True
+
+    def load_dialect_impl(
+        self,
+        dialect: Dialect,
+    ) -> sqlalchemy.types.TypeEngine:
+        if dialect.name == mlrun.common.db.dialects.Dialects.MYSQL:
+            return dialect.type_descriptor(
+                sqlalchemy.dialects.mysql.VARCHAR(
+                    collation="utf8_bin",
+                    length=255,
+                )
+            )
+        if dialect.name == mlrun.common.db.dialects.Dialects.POSTGRESQL:
+            # This collation is created as part of the database creation
+            return dialect.type_descriptor(
+                Text(
+                    collation="utf8_bin",
+                )
+            )
+        if dialect.name == mlrun.common.db.dialects.Dialects.SQLITE:
+            return dialect.type_descriptor(
+                Text(
+                    collation="BINARY",
+                )
+            )
+        return dialect.type_descriptor(self.impl)
+
+
+class UuidType(TypeDecorator):
+    """
+    A UUID type which stores as native UUID on Postgres (as_uuid=True)
+    and as 32-char hex strings on other dialects.
+    """
+
+    impl = CHAR(32)
+    cache_ok = True
+
+    def load_dialect_impl(self, dialect: Dialect) -> sqlalchemy.types.TypeEngine:
+        if dialect.name == mlrun.common.db.dialects.Dialects.POSTGRESQL:
+            return dialect.type_descriptor(PG_UUID(as_uuid=True))
+        return dialect.type_descriptor(CHAR(32))
+
+    def process_bind_param(
+        self,
+        value: Optional[Union[uuid.UUID, str]],
+        dialect: Dialect,
+    ) -> Optional[Union[uuid.UUID, str]]:
+        if value is None:
+            return None
+        if isinstance(value, uuid.UUID):
+            return (
+                value
+                if dialect.name == mlrun.common.db.dialects.Dialects.POSTGRESQL
+                else value.hex
+            )
+        if isinstance(value, str):
+            u = uuid.UUID(value)
+            return (
+                u
+                if dialect.name == mlrun.common.db.dialects.Dialects.POSTGRESQL
+                else u.hex
+            )
+        raise ValueError(f"Cannot bind UUID value {value!r}")
+
+    def process_result_value(
+        self, value: Optional[Union[uuid.UUID, bytes, str]], dialect: Dialect
+    ) -> Optional[uuid.UUID]:
+        if value is None:
+            return None
+        return value if isinstance(value, uuid.UUID) else uuid.UUID(value)
+
+    def coerce_compared_value(self, op: Any, value: Any) -> TypeDecorator:
+        # ensure STR comparisons are coerced through this type
+        return self

mlrun/frameworks/tf_keras/mlrun_interface.py

@@ -280,7 +280,10 @@ class TFKerasMLRunInterface(MLRunInterface, ABC):
             print(f"Horovod worker #{self._hvd.rank()} is using CPU")
 
         # Adjust learning rate based on the number of GPUs:
-        optimizer.lr = optimizer.lr * self._hvd.size()
+        if hasattr(optimizer, "lr"):
+            optimizer.lr *= self._hvd.size()
+        else:
+            optimizer.learning_rate *= self._hvd.size()
 
         # Wrap the optimizer in horovod's distributed optimizer: 'hvd.DistributedOptimizer'.
         optimizer = self._hvd.DistributedOptimizer(optimizer)

mlrun/frameworks/tf_keras/model_handler.py

@@ -518,7 +518,6 @@ class TFKerasModelHandler(DLModelHandler):
         )
 
         # Read additional files according to the model format used:
-        # # ModelFormats.SAVED_MODEL - Unzip the SavedModel archive:
         if self._model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL:
             # Unzip the SavedModel directory:
             with zipfile.ZipFile(self._model_file, "r") as zip_file:
@@ -527,11 +526,18 @@ class TFKerasModelHandler(DLModelHandler):
             self._model_file = os.path.join(
                 os.path.dirname(self._model_file), self._model_name
             )
-        # # ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS - Get the weights file:
-        elif (
+        elif self._model_format == TFKerasModelHandler.ModelFormats.KERAS:
+            # Rename the model file suffix:
+            self._rename_model_file_suffix(suffix="keras")
+        elif self._model_format == TFKerasModelHandler.ModelFormats.H5:
+            # Rename the model file suffix:
+            self._rename_model_file_suffix(suffix="h5")
+        elif (  # ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS
             self._model_format
             == TFKerasModelHandler.ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS
         ):
+            # Rename the model file suffix:
+            self._rename_model_file_suffix(suffix="json")
             # Get the weights file:
             self._weights_file = self._extra_data[
                 self._get_weights_file_artifact_name()
@@ -540,6 +546,20 @@ class TFKerasModelHandler(DLModelHandler):
         # Continue collecting from abstract class:
         super()._collect_files_from_store_object()
 
+    def _rename_model_file_suffix(self, suffix: str):
+        """
+        Rename the model file suffix to the given one.
+
+        This is used for the case of loading a model from a store object that was saved with a different suffix as when
+        keras tries to load it, it validates the suffix. The `artifacts.model.get_model` function is downloading the
+        file to a temp file with a `pkl` suffix, so it needs to be replaced:than the one keras expects.
+
+        :param suffix: The suffix to rename the model file to (without the trailing dot).
+        """
+        new_name = self._model_file.rsplit(".", 1)[0] + f".{suffix}"
+        os.rename(self._model_file, new_name)
+        self._model_file = new_name
+
     def _collect_files_from_local_path(self):
         """
         If the model path given is of a local path, search for the needed model files and collect them into this handler

mlrun/launcher/base.py

@@ -82,7 +82,6 @@ class BaseLauncher(abc.ABC):
         runtime: "mlrun.runtimes.base.BaseRuntime",
         project_name: Optional[str] = "",
         full: bool = True,
-        client_version: str = "",
     ):
         pass
 

mlrun/launcher/client.py

@@ -36,7 +36,6 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
         runtime: "mlrun.runtimes.base.BaseRuntime",
         project_name: Optional[str] = "",
         full: bool = True,
-        client_version: str = "",
     ):
         runtime.try_auto_mount_based_on_config()
         runtime._fill_credentials()

mlrun/launcher/local.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 import os
 import pathlib
-from os import environ
 from typing import Callable, Optional, Union
 
 import mlrun.common.constants as mlrun_constants
@@ -252,9 +251,6 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
             # copy the code/base-spec to the local function (for the UI and code logging)
             fn.spec.description = runtime.spec.description
             fn.spec.build = runtime.spec.build
-            serving_spec = getattr(runtime.spec, "serving_spec", None)
-            if serving_spec:
-                environ["SERVING_SPEC_ENV"] = serving_spec
 
         run.spec.handler = handler
         run.spec.reset_on_run = reset_on_run

mlrun/model_monitoring/applications/base.py

@@ -166,13 +166,29 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         return result
 
     @staticmethod
+    def _check_writer_is_up(project: "mlrun.MlrunProject") -> None:
+        try:
+            project.get_function(
+                mm_constants.MonitoringFunctionNames.WRITER, ignore_cache=True
+            )
+        except mlrun.errors.MLRunNotFoundError:
+            raise mlrun.errors.MLRunValueError(
+                "Writing outputs to the databases is blocked as the model monitoring infrastructure is disabled.\n"
+                "To unblock, enable model monitoring with `project.enable_model_monitoring()`."
+            )
+
+    @classmethod
     @contextmanager
     def _push_to_writer(
+        cls,
         *,
         write_output: bool,
         stream_profile: Optional[ds_profile.DatastoreProfile],
+        project: "mlrun.MlrunProject",
     ) -> Iterator[dict[str, list[tuple]]]:
         endpoints_output: dict[str, list[tuple]] = defaultdict(list)
+        if write_output:
+            cls._check_writer_is_up(project)
         try:
             yield endpoints_output
         finally:
@@ -220,6 +236,9 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         for an MLRun job.
         This method should not be called directly.
         """
+        project = context.get_project_object()
+        if not project:
+            raise mlrun.errors.MLRunValueError("Could not load project from context")
 
         if write_output and (
             not endpoints or sample_data is not None or reference_data is not None
@@ -236,7 +255,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         )
 
         with self._push_to_writer(
-            write_output=write_output, stream_profile=stream_profile
+            write_output=write_output, stream_profile=stream_profile, project=project
         ) as endpoints_output:
 
             def call_do_tracking(event: Optional[dict] = None):
@@ -249,6 +268,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                         event=event,
                         application_name=self.__class__.__name__,
                         context=context,
+                        project=project,
                         sample_df=sample_data,
                         feature_stats=feature_stats,
                     )

mlrun/model_monitoring/applications/context.py

@@ -137,13 +137,14 @@ class MonitoringApplicationContext:
         cls,
         context: "mlrun.MLClientCtx",
         *,
+        project: Optional["mlrun.MlrunProject"] = None,
         application_name: str,
         event: dict[str, Any],
         model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
         sample_df: Optional[pd.DataFrame] = None,
         feature_stats: Optional[FeatureStats] = None,
     ) -> "MonitoringApplicationContext":
-        project = context.get_project_object()
+        project = project or context.get_project_object()
         if not project:
             raise mlrun.errors.MLRunValueError("Could not load project from context")
         logger = context.logger

mlrun/projects/__init__.py

@@ -32,6 +32,7 @@ from .pipelines import (
     load_and_run_workflow,
     load_and_run,
     pipeline_context,
+    rerun_workflow,
 )  # noqa
 from .project import (
     MlrunProject,

mlrun/projects/pipelines.py

@@ -1070,6 +1070,40 @@ def github_webhook(request):
     return {"msg": "pushed"}
 
 
+def rerun_workflow(
+    context: mlrun.execution.MLClientCtx, run_uid: str, project_name: str
+):
+    """
+    Re-run a workflow by retrying a previously failed KFP pipeline.
+
+    :param context:      MLRun context.
+    :param run_uid:      The run UID of the original workflow to retry.
+    :param project_name: The project name.
+    """
+
+    try:
+        # TODO in followups: handle start and running notifications
+
+        # Retry the pipeline  - TODO: add submit-direct flag when created
+        db = mlrun.get_run_db()
+        new_pipeline_id = db.retry_pipeline(run_uid, project_name)
+
+        # Store result for observability
+        context.set_label("workflow-id", new_pipeline_id)
+        context.log_result("workflow_id", new_pipeline_id)
+
+        # wait for pipeline completion so monitor will push terminal notifications
+        wait_for_pipeline_completion(
+            new_pipeline_id,
+            project=project_name,
+        )
+
+    # Temporary exception
+    except Exception as exc:
+        context.logger.error("Failed to rerun workflow", exc=err_to_str(exc))
+        raise
+
+
 def load_and_run(context, *args, **kwargs):
     """
     This function serves as an alias to `load_and_run_workflow`,
@@ -1153,6 +1187,7 @@ def load_and_run_workflow(
     project = mlrun.get_or_create_project(
         context=project_context or f"./{project_name}",
         name=project_name,
+        allow_cross_project=True,
     )
 
     # extract "start" notification if exists
@@ -1245,6 +1280,7 @@ def pull_remote_project_files(
             subpath=subpath,
             clone=clone,
             save=False,
+            allow_cross_project=True,
         )
     except Exception as error:
         notify_scheduled_workflow_failure(

mlrun/projects/project.py

@@ -2961,19 +2961,6 @@ class MlrunProject(ModelObj):
             mlrun.db.get_run_db().delete_function(name=name, project=self.metadata.name)
         self.spec.remove_function(name)
 
-    def remove_model_monitoring_function(self, name: Union[str, list[str]]):
-        """delete the specified model-monitoring-app function/s
-
-        :param name: name of the model-monitoring-function/s (under the project)
-        """
-        # TODO: Remove this in 1.10.0
-        warnings.warn(
-            "'remove_model_monitoring_function' is deprecated in 1.7.0 and will be removed in 1.10.0. "
-            "Please use `delete_model_monitoring_function` instead.",
-            FutureWarning,
-        )
-        self.delete_model_monitoring_function(name)
-
     def delete_model_monitoring_function(self, name: Union[str, list[str]]):
         """delete the specified model-monitoring-app function/s
 

mlrun/runtimes/daskjob.py

@@ -92,7 +92,6 @@ class DaskSpec(KubeResourceSpec):
         preemption_mode=None,
         security_context=None,
         state_thresholds=None,
-        serving_spec=None,
     ):
         super().__init__(
             command=command,
@@ -122,7 +121,6 @@ class DaskSpec(KubeResourceSpec):
             preemption_mode=preemption_mode,
             security_context=security_context,
             state_thresholds=state_thresholds,
-            serving_spec=serving_spec,
         )
         self.args = args
 

mlrun/runtimes/kubejob.py

@@ -207,7 +207,3 @@ class KubejobRuntime(KubeResource):
         raise NotImplementedError(
             f"Running a {self.kind} function from the client is not supported. Use .run() to submit the job to the API."
         )
-
-    @property
-    def serving_spec(self):
-        return self.spec.serving_spec