mlrun 1.7.2rc4__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/artifacts/model.py

@@ -30,6 +30,7 @@ from ..utils import StorePrefix, is_relative_path
 from .base import Artifact, ArtifactSpec, upload_extra_data
 
 model_spec_filename = "model_spec.yaml"
+MODEL_OPTIONAL_SUFFIXES = [".tar.gz", ".pkl", ".bin", ".pickle"]
 
 
 class ModelArtifactSpec(ArtifactSpec):
@@ -151,7 +152,7 @@ class ModelArtifact(Artifact):
     ):
         if key or body or format or target_path:
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )
@@ -278,7 +279,11 @@ class ModelArtifact(Artifact):
         )
         if label_columns:
             inferer.infer_schema(
-                df[label_columns], self.spec.outputs, {}, options=InferOptions.Features
+                df[label_columns],
+                self.spec.outputs,
+                {},
+                options=InferOptions.Features,
+                push_at_start=True,
             )
         if with_stats:
             self.spec.feature_stats = inferer.get_stats(
@@ -303,7 +308,7 @@ class ModelArtifact(Artifact):
             self.metadata.labels = self.metadata.labels or {}
             self.metadata.labels["framework"] = self.spec.framework
 
-    def upload(self, artifact_path: str = None):
+    def upload(self, artifact_path: Optional[str] = None):
         """
         internal, upload to target store
         :param artifact_path: required only for when generating target_path from artifact hash
@@ -324,9 +329,7 @@ class ModelArtifact(Artifact):
             artifact=self, extra_data=self.spec.extra_data, artifact_path=artifact_path
         )
 
-        # the model spec yaml should not include the tag, as the same model can be used with different tags,
-        # and the tag is not part of the model spec but the metadata of the model artifact
-        spec_body = _remove_tag_from_spec_yaml(self)
+        spec_body = _sanitize_and_serialize_model_spec_yaml(self)
         spec_target_path = None
 
         if mlrun.mlconf.artifacts.generate_target_path_from_artifact_hash:
@@ -355,7 +358,7 @@ class ModelArtifact(Artifact):
     def _upload_body_or_file(
         self,
         artifact_path: str,
-        target_model_path: str = None,
+        target_model_path: Optional[str] = None,
     ):
         body = self.spec.get_body()
         if body:
@@ -403,12 +406,6 @@ class ModelArtifact(Artifact):
         return mlrun.get_dataitem(target_model_path).get()
 
 
-def _get_src_path(model_spec: ModelArtifact, filename):
-    if model_spec.src_path:
-        return path.join(model_spec.src_path, filename)
-    return filename
-
-
 def get_model(model_dir, suffix=""):
     """return model file, model spec object, and list of extra data items
 
@@ -434,10 +431,20 @@ def get_model(model_dir, suffix=""):
     model_file = ""
     model_spec = None
     extra_dataitems = {}
-    suffix = suffix or ".pkl"
+    default_suffix = ".pkl"
+
     if hasattr(model_dir, "artifact_url"):
         model_dir = model_dir.artifact_url
 
+    alternative_suffix = next(
+        (
+            optional_suffix
+            for optional_suffix in MODEL_OPTIONAL_SUFFIXES
+            if model_dir.lower().endswith(optional_suffix)
+        ),
+        None,
+    )
+
     if mlrun.datastore.is_store_uri(model_dir):
         model_spec, target = mlrun.datastore.store_manager.get_store_artifact(model_dir)
         if not model_spec or model_spec.kind != "model":
@@ -448,15 +455,19 @@ def get_model(model_dir, suffix=""):
             target, model_spec.model_target_file or model_spec.model_file
         )
         extra_dataitems = _get_extra(target, model_spec.extra_data)
-
+        suffix = suffix or default_suffix
     elif model_dir.lower().endswith(".yaml"):
         model_spec = _load_model_spec(model_dir)
         model_file = _get_file_path(model_dir, model_spec.model_file)
         extra_dataitems = _get_extra(model_dir, model_spec.extra_data)
-
-    elif model_dir.endswith(suffix):
+        suffix = suffix or default_suffix
+    elif suffix and model_dir.endswith(suffix):
+        model_file = model_dir
+    elif not suffix and alternative_suffix:
+        suffix = alternative_suffix
         model_file = model_dir
     else:
+        suffix = suffix or default_suffix
         dirobj = mlrun.datastore.store_manager.object(url=model_dir)
         model_dir_list = dirobj.listdir()
         if model_spec_filename in model_dir_list:
@@ -483,49 +494,20 @@ def get_model(model_dir, suffix=""):
     return temp_path, model_spec, extra_dataitems
 
 
-def _load_model_spec(spec_path):
-    data = mlrun.datastore.store_manager.object(url=spec_path).get()
-    spec = yaml.load(data, Loader=yaml.FullLoader)
-    return ModelArtifact.from_dict(spec)
-
-
-def _get_file_path(base_path: str, name: str, isdir=False):
-    if not is_relative_path(name):
-        return name
-    if not isdir:
-        base_path = path.dirname(base_path)
-    return path.join(base_path, name).replace("\\", "/")
-
-
-def _get_extra(target, extra_data, is_dir=False):
-    extra_dataitems = {}
-    for k, v in extra_data.items():
-        extra_dataitems[k] = mlrun.datastore.store_manager.object(
-            url=_get_file_path(target, v, isdir=is_dir), key=k
-        )
-    return extra_dataitems
-
-
-def _remove_tag_from_spec_yaml(model_spec):
-    spec_dict = model_spec.to_dict()
-    spec_dict["metadata"].pop("tag", None)
-    return yaml.safe_dump(spec_dict)
-
-
 def update_model(
     model_artifact,
-    parameters: dict = None,
-    metrics: dict = None,
-    extra_data: dict = None,
-    inputs: list[Feature] = None,
-    outputs: list[Feature] = None,
-    feature_vector: str = None,
-    feature_weights: list = None,
+    parameters: Optional[dict] = None,
+    metrics: Optional[dict] = None,
+    extra_data: Optional[dict] = None,
+    inputs: Optional[list[Feature]] = None,
+    outputs: Optional[list[Feature]] = None,
+    feature_vector: Optional[str] = None,
+    feature_weights: Optional[list] = None,
     key_prefix: str = "",
-    labels: dict = None,
+    labels: Optional[dict] = None,
     write_spec_copy=True,
     store_object: bool = True,
-):
+) -> ModelArtifact:
     """Update model object attributes
 
     this method will edit or add attributes to a model object
@@ -593,10 +575,7 @@ def update_model(
 
     if write_spec_copy:
         spec_path = path.join(model_spec.target_path, model_spec_filename)
-
-        # the model spec yaml should not include the tag, as the same model can be used with different tags,
-        # and the tag is not part of the model spec but the metadata of the model artifact
-        model_spec_yaml = _remove_tag_from_spec_yaml(model_spec)
+        model_spec_yaml = _sanitize_and_serialize_model_spec_yaml(model_spec)
         mlrun.datastore.store_manager.object(url=spec_path).put(model_spec_yaml)
 
     model_spec.db_key = model_spec.db_key or model_spec.key
@@ -609,3 +588,56 @@ def update_model(
             project=model_spec.project,
         )
     return model_spec
+
+
+def _get_src_path(model_spec: ModelArtifact, filename: str) -> str:
+    return path.join(model_spec.src_path, filename) if model_spec.src_path else filename
+
+
+def _load_model_spec(spec_path) -> ModelArtifact:
+    data = mlrun.datastore.store_manager.object(url=spec_path).get()
+    spec = yaml.load(data, Loader=yaml.FullLoader)
+    return ModelArtifact.from_dict(spec)
+
+
+def _get_file_path(base_path: str, name: str, isdir: bool = False) -> str:
+    if not is_relative_path(name):
+        return name
+    if not isdir:
+        base_path = path.dirname(base_path)
+    return path.join(base_path, name).replace("\\", "/")
+
+
+def _get_extra(target: str, extra_data: dict, is_dir: bool = False) -> dict:
+    extra_dataitems = {}
+    for k, v in extra_data.items():
+        extra_dataitems[k] = mlrun.datastore.store_manager.object(
+            url=_get_file_path(target, v, isdir=is_dir), key=k
+        )
+    return extra_dataitems
+
+
+def _sanitize_and_serialize_model_spec_yaml(model: ModelArtifact) -> str:
+    model_dict = _sanitize_model_spec(model)
+    return _serialize_model_spec_yaml(model_dict)
+
+
+def _sanitize_model_spec(model: ModelArtifact) -> dict:
+    model_dict = model.to_dict()
+
+    # The model spec yaml should not include the tag, as the same model can be used with different tags,
+    # and the tag is not part of the model spec but the metadata of the model artifact
+    model_dict["metadata"].pop("tag", None)
+
+    # Remove future packaging links
+    if model_dict["spec"].get("extra_data"):
+        model_dict["spec"]["extra_data"] = {
+            key: item
+            for key, item in model_dict["spec"]["extra_data"].items()
+            if item is not ...
+        }
+    return model_dict
+
+
+def _serialize_model_spec_yaml(model_dict: dict) -> str:
+    return yaml.safe_dump(model_dict)

mlrun/artifacts/plots.py

@@ -37,7 +37,7 @@ class PlotArtifact(Artifact):
     ):
         if key or body or is_inline or target_path:
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )
@@ -96,7 +96,7 @@ class PlotlyArtifact(Artifact):
         """
         if key or target_path:
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )

mlrun/common/constants.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import mlrun.common.types
 
 IMAGE_NAME_ENRICH_REGISTRY_PREFIX = "."  # prefix for image name to enrich with registry
 MLRUN_SERVING_CONF = "serving-conf"
@@ -24,6 +25,7 @@ MYSQL_MEDIUMBLOB_SIZE_BYTES = 16 * 1024 * 1024
 MLRUN_LABEL_PREFIX = "mlrun/"
 DASK_LABEL_PREFIX = "dask.org/"
 NUCLIO_LABEL_PREFIX = "nuclio.io/"
+RESERVED_TAG_NAME_LATEST = "latest"
 
 
 class MLRunInternalLabels:
@@ -68,6 +70,7 @@ class MLRunInternalLabels:
     producer_type = f"{MLRUN_LABEL_PREFIX}producer-type"
     app_name = f"{MLRUN_LABEL_PREFIX}app-name"
     endpoint_id = f"{MLRUN_LABEL_PREFIX}endpoint-id"
+    endpoint_name = f"{MLRUN_LABEL_PREFIX}endpoint-name"
     host = "host"
     job_type = "job-type"
     kind = "kind"
@@ -86,3 +89,8 @@ class MLRunInternalLabels:
             for key, value in cls.__dict__.items()
             if not key.startswith("__") and isinstance(value, str)
         ]
+
+
+class DeployStatusTextKind(mlrun.common.types.StrEnum):
+    logs = "logs"
+    events = "events"

mlrun/common/formatters/__init__.py

@@ -19,3 +19,4 @@ from .pipeline import PipelineFormat  # noqa
 from .project import ProjectFormat  # noqa
 from .run import RunFormat  # noqa
 from .feature_set import FeatureSetFormat  # noqa
+from .model_endpoint import ModelEndpointFormat  # noqa

mlrun/common/formatters/artifact.py

@@ -32,7 +32,7 @@ class ArtifactFormat(ObjectFormat, mlrun.common.types.StrEnum):
                 [
                     "kind",
                     "metadata",
-                    "status",
+                    "status.state",
                     "project",
                     "spec.producer",
                     "spec.db_key",

mlrun/common/formatters/feature_set.py

@@ -21,6 +21,7 @@ from .base import ObjectFormat
 
 
 class FeatureSetFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
     minimal = "minimal"
 
     @staticmethod
@@ -32,6 +33,7 @@ class FeatureSetFormat(ObjectFormat, mlrun.common.types.StrEnum):
                     "metadata.name",
                     "metadata.project",
                     "metadata.tag",
+                    "metadata.updated",
                     "metadata.uid",
                     "metadata.labels",
                     "spec.entities",

mlrun/common/formatters/function.py

@@ -21,6 +21,7 @@ from .base import ObjectFormat
 
 
 class FunctionFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
     minimal = "minimal"
 
     @staticmethod

mlrun/{model_monitoring/db/stores/v3io_kv/__init__.py → common/formatters/model_endpoint.py}

@@ -11,3 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+
+import typing
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class ModelEndpointFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            ModelEndpointFormat.full: None,
+        }[_format]

mlrun/common/formatters/pipeline.py

@@ -15,11 +15,10 @@
 
 import typing
 
+import mlrun.common.types
 import mlrun_pipelines.common.ops
 import mlrun_pipelines.models
 
-import mlrun.common.types
-
 from .base import ObjectFormat
 
 

mlrun/common/formatters/project.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 #
 
+import datetime
 import typing
 
 import mlrun.common.schemas
@@ -30,11 +31,18 @@ class ProjectFormat(ObjectFormat, mlrun.common.types.StrEnum):
     # internal - allowed only in follower mode, only for the leader for upgrade purposes
     leader = "leader"
 
+    name_and_creation_time = "name_and_creation_time"
+
     @staticmethod
     def format_method(_format: str) -> typing.Optional[typing.Callable]:
         def _name_only(project: mlrun.common.schemas.Project) -> str:
             return project.metadata.name
 
+        def _name_and_creation_time(
+            project: mlrun.common.schemas.Project,
+        ) -> tuple[str, datetime.datetime]:
+            return project.metadata.name, project.metadata.created
+
         def _minimal(
             project: mlrun.common.schemas.Project,
         ) -> mlrun.common.schemas.Project:
@@ -48,4 +56,5 @@ class ProjectFormat(ObjectFormat, mlrun.common.types.StrEnum):
             ProjectFormat.name_only: _name_only,
             ProjectFormat.minimal: _minimal,
             ProjectFormat.leader: None,
+            ProjectFormat.name_and_creation_time: _name_and_creation_time,
         }[_format]

mlrun/common/model_monitoring/__init__.py

@@ -11,8 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
-from .helpers import create_model_endpoint_uid

mlrun/common/model_monitoring/helpers.py

@@ -17,11 +17,6 @@ import typing
 
 import mlrun.common
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
-from mlrun.common.schemas.model_monitoring import (
-    EndpointUID,
-    FunctionURI,
-    VersionedModel,
-)
 
 FeatureStats = typing.NewType("FeatureStats", dict[str, dict[str, typing.Any]])
 Histogram = typing.NewType("Histogram", list[list])
@@ -31,29 +26,6 @@ BinEdges = typing.NewType("BinEdges", list[float])
 _MAX_FLOAT = sys.float_info.max
 
 
-def create_model_endpoint_uid(function_uri: str, versioned_model: str):
-    function_uri = FunctionURI.from_string(function_uri)
-    versioned_model = VersionedModel.from_string(versioned_model)
-
-    if (
-        not function_uri.project
-        or not function_uri.function
-        or not versioned_model.model
-    ):
-        raise ValueError("Both function_uri and versioned_model have to be initialized")
-
-    uid = EndpointUID(
-        function_uri.project,
-        function_uri.function,
-        function_uri.tag,
-        function_uri.hash_key,
-        versioned_model.model,
-        versioned_model.version,
-    )
-
-    return uid
-
-
 def parse_model_endpoint_project_prefix(path: str, project_name: str):
     return path.split(project_name, 1)[0] + project_name
 
@@ -64,40 +36,18 @@ def parse_model_endpoint_store_prefix(store_prefix: str):
     return endpoint, container, path
 
 
-def parse_monitoring_stream_path(
-    stream_uri: str, project: str, function_name: str = None
-):
-    if stream_uri.startswith("kafka://"):
-        if "?topic" in stream_uri:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Custom kafka topic is not allowed"
-            )
-        # Add topic to stream kafka uri
-        if (
-            function_name is None
-            or function_name == mm_constants.MonitoringFunctionNames.STREAM
-        ):
-            stream_uri += f"?topic=monitoring_stream_{project}"
-        else:
-            stream_uri += f"?topic=monitoring_stream_{project}_{function_name}"
-
-    elif stream_uri.startswith("v3io://") and mlrun.mlconf.is_ce_mode():
-        # V3IO is not supported in CE mode, generating a default http stream path
-        if function_name is None:
-            stream_uri = (
-                mlrun.mlconf.model_endpoint_monitoring.default_http_sink.format(
-                    project=project, namespace=mlrun.mlconf.namespace
-                )
-            )
-        else:
-            stream_uri = (
-                mlrun.mlconf.model_endpoint_monitoring.default_http_sink_app.format(
-                    project=project,
-                    application_name=function_name,
-                    namespace=mlrun.mlconf.namespace,
-                )
-            )
-    return stream_uri
+def get_kafka_topic(project: str, function_name: typing.Optional[str] = None) -> str:
+    if (
+        function_name is None
+        or function_name == mm_constants.MonitoringFunctionNames.STREAM
+    ):
+        function_specifier = ""
+    else:
+        function_specifier = f"_{function_name}"
+
+    return (
+        f"monitoring_stream_{mlrun.mlconf.system_id}_{project}{function_specifier}_v1"
+    )
 
 
 def _get_counts(hist: Histogram) -> BinCounts:

mlrun/common/runtimes/constants.py

@@ -15,9 +15,8 @@
 import enum
 import typing
 
-import mlrun_pipelines.common.models
-
 import mlrun.common.constants as mlrun_constants
+import mlrun_pipelines.common.models
 
 
 class PodPhases:
@@ -195,6 +194,10 @@ class RunStates:
             # TODO: add aborting state once we have it
         ]
 
+    @staticmethod
+    def notification_states():
+        return RunStates.terminal_states() + [RunStates.running]
+
     @staticmethod
     def run_state_to_pipeline_run_status(run_state: str):
         if not run_state:
@@ -215,8 +218,26 @@ class RunStates:
             RunStates.skipped: mlrun_pipelines.common.models.RunStatuses.skipped,
         }[run_state]
 
-
-# TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
+    @staticmethod
+    def pipeline_run_status_to_run_state(pipeline_run_status):
+        if pipeline_run_status not in mlrun_pipelines.common.models.RunStatuses.all():
+            raise ValueError(f"Invalid pipeline run status: {pipeline_run_status}")
+        return {
+            mlrun_pipelines.common.models.RunStatuses.succeeded: RunStates.completed,
+            mlrun_pipelines.common.models.RunStatuses.failed: RunStates.error,
+            mlrun_pipelines.common.models.RunStatuses.running: RunStates.running,
+            mlrun_pipelines.common.models.RunStatuses.pending: RunStates.pending,
+            mlrun_pipelines.common.models.RunStatuses.canceled: RunStates.aborted,
+            mlrun_pipelines.common.models.RunStatuses.canceling: RunStates.aborting,
+            mlrun_pipelines.common.models.RunStatuses.skipped: RunStates.skipped,
+            mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified: RunStates.unknown,
+            mlrun_pipelines.common.models.RunStatuses.error: RunStates.error,
+            mlrun_pipelines.common.models.RunStatuses.paused: RunStates.unknown,
+            mlrun_pipelines.common.models.RunStatuses.unknown: RunStates.unknown,
+        }[pipeline_run_status]
+
+
+# TODO: remove this class in 1.10.0 - use only MlrunInternalLabels
 class RunLabels(enum.Enum):
     owner = mlrun_constants.MLRunInternalLabels.owner
     v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user

mlrun/common/schemas/__init__.py

@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from .alert import (
+    AlertActivation,
+    AlertActivations,
     AlertActiveState,
     AlertConfig,
     AlertNotification,
@@ -63,6 +63,7 @@ from .clusterization_spec import (
 from .common import ImageBuilder
 from .constants import (
     APIStates,
+    ArtifactPartitionByField,
     ClusterizationRole,
     DeletionStrategy,
     FeatureStorePartitionByField,
@@ -138,19 +139,19 @@ from .model_monitoring import (
     Features,
     FeatureSetFeatures,
     FeatureValues,
+    FileTargetKind,
     GrafanaColumn,
-    GrafanaDataPoint,
     GrafanaNumberColumn,
     GrafanaStringColumn,
     GrafanaTable,
-    GrafanaTimeSeriesTarget,
     ModelEndpoint,
+    ModelEndpointCreationStrategy,
     ModelEndpointList,
     ModelEndpointMetadata,
+    ModelEndpointSchema,
     ModelEndpointSpec,
     ModelEndpointStatus,
     ModelMonitoringMode,
-    ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
     TSDBTarget,
     V3IOTSDBTables,
@@ -159,11 +160,14 @@ from .notification import (
     Notification,
     NotificationKind,
     NotificationSeverity,
+    NotificationState,
     NotificationStatus,
+    NotificationSummary,
     SetNotificationRequest,
 )
 from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
 from .pagination import PaginationInfo
+from .partition import PartitionInterval
 from .pipeline import PipelinesOutput, PipelinesPagination
 from .project import (
     IguazioProject,