mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc21__py3-none-any.whl

mlrun/artifacts/manager.py

@@ -12,11 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import pathlib
+import re
 import typing
 from os.path import exists, isdir
 from urllib.parse import urlparse
 
+import mlrun.common.schemas.artifact
 import mlrun.config
+import mlrun.utils.regex
 from mlrun.utils.helpers import (
     get_local_file_schema,
     template_artifact_path,
@@ -24,7 +27,6 @@ from mlrun.utils.helpers import (
 )
 
 from ..utils import (
-    is_legacy_artifact,
     is_relative_path,
     logger,
     validate_artifact_key_name,
@@ -33,56 +35,28 @@ from ..utils import (
 from .base import (
     Artifact,
     DirArtifact,
-    LegacyArtifact,
-    LegacyDirArtifact,
-    LegacyLinkArtifact,
     LinkArtifact,
 )
 from .dataset import (
     DatasetArtifact,
-    LegacyDatasetArtifact,
-    LegacyTableArtifact,
     TableArtifact,
 )
-from .model import LegacyModelArtifact, ModelArtifact
+from .model import ModelArtifact
 from .plots import (
-    BokehArtifact,
-    ChartArtifact,
-    LegacyBokehArtifact,
-    LegacyChartArtifact,
-    LegacyPlotArtifact,
-    LegacyPlotlyArtifact,
     PlotArtifact,
     PlotlyArtifact,
 )
 
-# TODO - Remove deprecated types when deleted in 1.7.0
 artifact_types = {
     "": Artifact,
     "artifact": Artifact,
     "dir": DirArtifact,
     "link": LinkArtifact,
     "plot": PlotArtifact,
-    "chart": ChartArtifact,
     "table": TableArtifact,
     "model": ModelArtifact,
     "dataset": DatasetArtifact,
     "plotly": PlotlyArtifact,
-    "bokeh": BokehArtifact,
-}
-
-# TODO - Remove this when legacy types are deleted in 1.7.0
-legacy_artifact_types = {
-    "": LegacyArtifact,
-    "dir": LegacyDirArtifact,
-    "link": LegacyLinkArtifact,
-    "plot": LegacyPlotArtifact,
-    "chart": LegacyChartArtifact,
-    "table": LegacyTableArtifact,
-    "model": LegacyModelArtifact,
-    "dataset": LegacyDatasetArtifact,
-    "plotly": LegacyPlotlyArtifact,
-    "bokeh": LegacyBokehArtifact,
 }
 
 
@@ -100,17 +74,33 @@ class ArtifactProducer:
     def get_meta(self) -> dict:
         return {"kind": self.kind, "name": self.name, "tag": self.tag}
 
+    @property
+    def uid(self):
+        return None
 
-def dict_to_artifact(struct: dict) -> Artifact:
-    # Need to distinguish between LegacyArtifact classes and Artifact classes. Use existence of the "metadata"
-    # property to make this distinction
-    kind = struct.get("kind", "")
+    @staticmethod
+    def parse_uri(uri: str) -> tuple[str, str, str]:
+        """Parse artifact producer's uri
 
-    if is_legacy_artifact(struct):
-        return mlrun.artifacts.base.convert_legacy_artifact_to_new_format(struct)
+        :param uri: artifact producer's uri in the format <project>/<uid>[-<iteration>]
+        :returns: tuple of project, uid, iteration
+        """
+        uri_pattern = mlrun.utils.regex.artifact_producer_uri_pattern
+        match = re.match(uri_pattern, uri)
+        if not match:
+            return "", "", ""
+        group_dict = match.groupdict()
+
+        return (
+            group_dict["project"] or "",
+            group_dict["uid"] or "",
+            group_dict["iteration"] or "",
+        )
 
-    artifact_class = artifact_types[kind]
 
+def dict_to_artifact(struct: dict) -> Artifact:
+    kind = struct.get("kind", "")
+    artifact_class = artifact_types[kind]
     return artifact_class.from_dict(struct)
 
 
@@ -297,7 +287,7 @@ class ArtifactManager:
         if target_path and item.is_dir and not target_path.endswith("/"):
             target_path += "/"
         target_path = template_artifact_path(
-            artifact_path=target_path, project=producer.project
+            artifact_path=target_path, project=producer.project, run_uid=producer.uid
         )
         item.target_path = target_path
 
@@ -308,10 +298,7 @@ class ArtifactManager:
             # before uploading the item, we want to ensure that its tags are valid,
             # so that we don't upload something that won't be stored later
             validate_tag_name(item.metadata.tag, "artifact.metadata.tag")
-            if is_legacy_artifact(item):
-                item.upload()
-            else:
-                item.upload(artifact_path=artifact_path)
+            item.upload(artifact_path=artifact_path)
 
         if db_key:
             self._log_to_db(db_key, project, producer.inputs, item)
@@ -382,6 +369,23 @@ class ArtifactManager:
                 project=project,
             )
 
+    def delete_artifact(
+        self,
+        item: Artifact,
+        deletion_strategy: mlrun.common.schemas.artifact.ArtifactsDeletionStrategies = (
+            mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
+        ),
+        secrets: dict = None,
+    ):
+        self.artifact_db.del_artifact(
+            key=item.db_key,
+            project=item.project,
+            tag=item.tag,
+            tree=item.tree,
+            deletion_strategy=deletion_strategy,
+            secrets=secrets,
+        )
+
 
 def extend_artifact_path(artifact_path: str, default_artifact_path: str):
     artifact_path = str(artifact_path or "")

mlrun/artifacts/model.py

@@ -13,12 +13,12 @@
 # limitations under the License.
 
 import tempfile
+import warnings
 from os import path
 from typing import Any, Optional
 
 import pandas as pd
 import yaml
-from deprecated import deprecated
 
 import mlrun
 import mlrun.datastore
@@ -27,7 +27,7 @@ from ..data_types import InferOptions, get_infer_interface
 from ..features import Feature
 from ..model import ObjectList
 from ..utils import StorePrefix, is_relative_path
-from .base import Artifact, ArtifactSpec, LegacyArtifact, upload_extra_data
+from .base import Artifact, ArtifactSpec, upload_extra_data
 
 model_spec_filename = "model_spec.yaml"
 
@@ -149,6 +149,12 @@ class ModelArtifact(Artifact):
         model_dir=None,
         **kwargs,
     ):
+        if key or body or format or target_path:
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
         super().__init__(key, body, format=format, target_path=target_path, **kwargs)
         model_file = str(model_file or "")
         if model_file and "/" in model_file:
@@ -397,144 +403,6 @@ class ModelArtifact(Artifact):
         return mlrun.get_dataitem(target_model_path).get()
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyModelArtifact' will be removed in 1.7.0, use 'ModelArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyModelArtifact(LegacyArtifact):
-    """ML Model artifact
-
-    Store link to ML model file(s) along with the model metrics, parameters, schema, and stats
-    """
-
-    _dict_fields = LegacyArtifact._dict_fields + [
-        "model_file",
-        "metrics",
-        "parameters",
-        "inputs",
-        "outputs",
-        "framework",
-        "algorithm",
-        "extra_data",
-        "feature_vector",
-        "feature_weights",
-        "feature_stats",
-        "model_target_file",
-    ]
-    kind = "model"
-    _store_prefix = StorePrefix.Model
-
-    def __init__(
-        self,
-        key=None,
-        body=None,
-        format=None,
-        model_file=None,
-        metrics=None,
-        target_path=None,
-        parameters=None,
-        inputs=None,
-        outputs=None,
-        framework=None,
-        algorithm=None,
-        feature_vector=None,
-        feature_weights=None,
-        extra_data=None,
-        model_target_file=None,
-        **kwargs,
-    ):
-        super().__init__(key, body, format=format, target_path=target_path, **kwargs)
-        self._inputs: Optional[ObjectList] = None
-        self._outputs: Optional[ObjectList] = None
-
-        self.model_file = model_file
-        self.parameters = parameters or {}
-        self.metrics = metrics or {}
-        self.inputs = inputs or []
-        self.outputs = outputs or []
-        self.extra_data = extra_data or {}
-        self.framework = framework
-        self.algorithm = algorithm
-        self.feature_vector = feature_vector
-        self.feature_weights = feature_weights
-        self.feature_stats = None
-        self.model_target_file = model_target_file
-
-    @property
-    def inputs(self) -> Optional[ObjectList]:
-        """input feature list"""
-        return self._inputs
-
-    @inputs.setter
-    def inputs(self, inputs: list[Feature]) -> None:
-        self._inputs = ObjectList.from_list(Feature, inputs)
-
-    @property
-    def outputs(self) -> Optional[ObjectList]:
-        """output feature list"""
-        return self._outputs
-
-    @outputs.setter
-    def outputs(self, outputs: list[Feature]) -> None:
-        self._outputs = ObjectList.from_list(Feature, outputs)
-
-    def infer_from_df(self, df, label_columns=None, with_stats=True, num_bins=None):
-        """infer inputs, outputs, and stats from provided df (training set)
-
-        :param df:      dataframe to infer from
-        :param label_columns: name of the label (target) column
-        :param with_stats:    infer statistics (min, max, .. histogram)
-        :param num_bins:      number of bins for histogram
-        """
-        subset = df
-        inferer = get_infer_interface(subset)
-        if label_columns:
-            if not isinstance(label_columns, list):
-                label_columns = [label_columns]
-            subset = df.drop(columns=label_columns)
-        inferer.infer_schema(subset, self.inputs, {}, options=InferOptions.Features)
-        if label_columns:
-            inferer.infer_schema(
-                df[label_columns], self.outputs, {}, options=InferOptions.Features
-            )
-        if with_stats:
-            self.feature_stats = inferer.get_stats(
-                df, options=InferOptions.Histogram, num_bins=num_bins
-            )
-
-    @property
-    def is_dir(self):
-        return True
-
-    def before_log(self):
-        if not self.model_file:
-            raise ValueError("model_file attr must be specified")
-
-        super().before_log()
-
-        if self.framework:
-            self.labels = self.labels or {}
-            self.labels["framework"] = self.framework
-
-    def upload(self):
-        target_model_path = path.join(self.target_path, self.model_file)
-        body = self.get_body()
-        if body:
-            self._upload_body(body, target=target_model_path)
-        else:
-            src_model_path = _get_src_path(self, self.model_file)
-            if not path.isfile(src_model_path):
-                raise ValueError(f"model file {src_model_path} not found")
-            self._upload_file(src_model_path, target=target_model_path)
-
-        upload_extra_data(self, self.extra_data)
-
-        spec_path = path.join(self.target_path, model_spec_filename)
-        mlrun.datastore.store_manager.object(url=spec_path).put(self.to_yaml())
-
-
 def _get_src_path(model_spec: ModelArtifact, filename):
     if model_spec.src_path:
         return path.join(model_spec.src_path, filename)
@@ -641,7 +509,7 @@ def _get_extra(target, extra_data, is_dir=False):
 def _remove_tag_from_spec_yaml(model_spec):
     spec_dict = model_spec.to_dict()
     spec_dict["metadata"].pop("tag", None)
-    return yaml.dump(spec_dict)
+    return yaml.safe_dump(spec_dict)
 
 
 def update_model(