mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0rc2__py3-none-any.whl

mlrun/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 __all__ = [
     "get_version",
     "set_environment",
@@ -29,9 +27,9 @@ __all__ = [
 ]
 
 from os import environ, path
+from typing import Optional
 
 import dotenv
-import mlrun_pipelines
 
 from .config import config as mlconf
 from .datastore import DataItem, store_manager
@@ -63,16 +61,16 @@ from .run import (
     new_function,
     wait_for_pipeline_completion,
 )
-from .runtimes import new_model_server
+from .runtimes import mounts, new_model_server
 from .secrets import get_secret_or_env
 from .utils.version import Version
 
 __version__ = Version().get()["version"]
 
-VolumeMount = mlrun_pipelines.common.mounts.VolumeMount
-mount_v3io = mlrun_pipelines.mounts.mount_v3io
-v3io_cred = mlrun_pipelines.mounts.v3io_cred
-auto_mount = mlrun_pipelines.mounts.auto_mount
+VolumeMount = mounts.VolumeMount
+mount_v3io = mounts.mount_v3io
+v3io_cred = mounts.v3io_cred
+auto_mount = mounts.auto_mount
 
 
 def get_version():
@@ -88,12 +86,12 @@ if "IGZ_NAMESPACE_DOMAIN" in environ:
 
 
 def set_environment(
-    api_path: str = None,
+    api_path: Optional[str] = None,
     artifact_path: str = "",
-    access_key: str = None,
-    username: str = None,
-    env_file: str = None,
-    mock_functions: str = None,
+    access_key: Optional[str] = None,
+    username: Optional[str] = None,
+    env_file: Optional[str] = None,
+    mock_functions: Optional[str] = None,
 ):
     """set and test default config for: api path, artifact_path and project
 
@@ -136,15 +134,16 @@ def set_environment(
     if not mlconf.dbpath:
         raise ValueError("DB/API path was not detected, please specify its address")
 
-    if mock_functions is not None:
-        mock_functions = "1" if mock_functions is True else mock_functions
-        mlconf.force_run_local = mock_functions
-        mlconf.mock_nuclio_deployment = mock_functions
-
     # check connectivity and load remote defaults
     get_run_db()
     if api_path:
         environ["MLRUN_DBPATH"] = mlconf.dbpath
+    mlconf.reload()
+
+    if mock_functions is not None:
+        mock_functions = "1" if mock_functions is True else mock_functions
+        mlconf.force_run_local = mock_functions
+        mlconf.mock_nuclio_deployment = mock_functions
 
     if not mlconf.artifact_path and not artifact_path:
         raise ValueError(
@@ -159,6 +158,7 @@ def set_environment(
                 "artifact_path must refer to an absolute path" " or a valid url"
             )
         mlconf.artifact_path = artifact_path
+
     return mlconf.default_project, mlconf.artifact_path
 
 

mlrun/__main__.py

@@ -27,13 +27,13 @@ import click
 import dotenv
 import pandas as pd
 import yaml
-from mlrun_pipelines.mounts import auto_mount as auto_mount_modifier
 from tabulate import tabulate
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 from mlrun.common.helpers import parse_versioned_object_uri
+from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
 
 from .config import config as mlconf
 from .db import get_run_db
@@ -1248,10 +1248,10 @@ def show_or_set_config(
         }
         for key, value in env_dict.items():
             if value:
-                dotenv.set_key(filename, key, value, quote_mode="")
+                dotenv.set_key(filename, key, value, quote_mode="always")
         if env_vars:
             for key, value in list2dict(env_vars).items():
-                dotenv.set_key(filename, key, value, quote_mode="")
+                dotenv.set_key(filename, key, value, quote_mode="always")
         if env_file:
             # if its not the default file print the usage details
             print(

mlrun/alerts/alert.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Union
+from typing import Optional, Union
 
 import mlrun
 import mlrun.common.schemas.alert as alert_objects
@@ -40,21 +40,21 @@ class AlertConfig(ModelObj):
 
     def __init__(
         self,
-        project: str = None,
-        name: str = None,
+        project: Optional[str] = None,
+        name: Optional[str] = None,
         template: Union[alert_objects.AlertTemplate, str] = None,
-        description: str = None,
-        summary: str = None,
+        description: Optional[str] = None,
+        summary: Optional[str] = None,
         severity: alert_objects.AlertSeverity = None,
         trigger: alert_objects.AlertTrigger = None,
         criteria: alert_objects.AlertCriteria = None,
         reset_policy: alert_objects.ResetPolicy = None,
-        notifications: list[alert_objects.AlertNotification] = None,
+        notifications: Optional[list[alert_objects.AlertNotification]] = None,
         entities: alert_objects.EventEntities = None,
-        id: int = None,
+        id: Optional[int] = None,
         state: alert_objects.AlertActiveState = None,
-        created: str = None,
-        count: int = None,
+        created: Optional[str] = None,
+        count: Optional[int] = None,
     ):
         """Alert config object
 
@@ -142,7 +142,7 @@ class AlertConfig(ModelObj):
             raise mlrun.errors.MLRunInvalidArgumentError("Alert name must be provided")
 
     def _serialize_field(
-        self, struct: dict, field_name: str = None, strip: bool = False
+        self, struct: dict, field_name: Optional[str] = None, strip: bool = False
     ):
         if field_name == "entities":
             if self.entities:
@@ -179,7 +179,12 @@ class AlertConfig(ModelObj):
             return None
         return super()._serialize_field(struct, field_name, strip)
 
-    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+    def to_dict(
+        self,
+        fields: Optional[list] = None,
+        exclude: Optional[list] = None,
+        strip: bool = False,
+    ):
         if self.entities is None:
             raise mlrun.errors.MLRunBadRequestError("Alert entity field is missing")
         if not self.notifications:
@@ -189,7 +194,9 @@ class AlertConfig(ModelObj):
         return super().to_dict(self._dict_fields)
 
     @classmethod
-    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+    def from_dict(
+        cls, struct=None, fields=None, deprecated_fields: Optional[dict] = None
+    ):
         new_obj = super().from_dict(struct, fields=fields)
 
         entity_data = struct.get("entities")

mlrun/artifacts/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 # Don't remove this, used by sphinx documentation
 __all__ = ["get_model", "update_model"]
 

mlrun/artifacts/base.py

@@ -62,14 +62,21 @@ class ArtifactMetadata(ModelObj):
     def base_dict(self):
         return super().to_dict()
 
-    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+    def to_dict(
+        self,
+        fields: typing.Optional[list] = None,
+        exclude: typing.Optional[list] = None,
+        strip: bool = False,
+    ):
         """return long dict form of the artifact"""
         return super().to_dict(
             self._dict_fields + self._extra_fields, exclude=exclude, strip=strip
         )
 
     @classmethod
-    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+    def from_dict(
+        cls, struct=None, fields=None, deprecated_fields: typing.Optional[dict] = None
+    ):
         fields = fields or cls._dict_fields + cls._extra_fields
         return super().from_dict(
             struct, fields=fields, deprecated_fields=deprecated_fields
@@ -109,7 +116,7 @@ class ArtifactSpec(ModelObj):
         db_key=None,
         extra_data=None,
         body=None,
-        unpackaging_instructions: dict = None,
+        unpackaging_instructions: typing.Optional[dict] = None,
     ):
         self.src_path = src_path
         self.target_path = target_path
@@ -131,14 +138,21 @@ class ArtifactSpec(ModelObj):
     def base_dict(self):
         return super().to_dict()
 
-    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+    def to_dict(
+        self,
+        fields: typing.Optional[list] = None,
+        exclude: typing.Optional[list] = None,
+        strip: bool = False,
+    ):
         """return long dict form of the artifact"""
         return super().to_dict(
             self._dict_fields + self._extra_fields, exclude=exclude, strip=strip
         )
 
     @classmethod
-    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+    def from_dict(
+        cls, struct=None, fields=None, deprecated_fields: typing.Optional[dict] = None
+    ):
         fields = fields or cls._dict_fields + cls._extra_fields
         return super().from_dict(
             struct, fields=fields, deprecated_fields=deprecated_fields
@@ -192,7 +206,7 @@ class Artifact(ModelObj):
         size=None,
         target_path=None,
         project=None,
-        src_path: str = None,
+        src_path: typing.Optional[str] = None,
         # All params up until here are legacy params for compatibility with legacy artifacts.
         # TODO: remove them in 1.9.0.
         metadata: ArtifactMetadata = None,
@@ -366,7 +380,7 @@ class Artifact(ModelObj):
                 struct[field] = val.base_dict()
         return struct
 
-    def upload(self, artifact_path: str = None):
+    def upload(self, artifact_path: typing.Optional[str] = None):
         """
         internal, upload to target store
         :param artifact_path: required only for when generating target_path from artifact hash
@@ -379,7 +393,9 @@ class Artifact(ModelObj):
             if src_path and os.path.isfile(src_path):
                 self._upload_file(source_path=src_path, artifact_path=artifact_path)
 
-    def _upload_body(self, body, target=None, artifact_path: str = None):
+    def _upload_body(
+        self, body, target=None, artifact_path: typing.Optional[str] = None
+    ):
         body_hash = None
         if not target and not self.spec.target_path:
             if not mlrun.mlconf.artifacts.generate_target_path_from_artifact_hash:
@@ -400,7 +416,10 @@ class Artifact(ModelObj):
         )
 
     def _upload_file(
-        self, source_path: str, target_path: str = None, artifact_path: str = None
+        self,
+        source_path: str,
+        target_path: typing.Optional[str] = None,
+        artifact_path: typing.Optional[str] = None,
     ):
         file_hash = None
         if not target_path and not self.spec.target_path:
@@ -651,7 +670,7 @@ class DirArtifact(Artifact):
     def is_dir(self):
         return True
 
-    def upload(self, artifact_path: str = None):
+    def upload(self, artifact_path: typing.Optional[str] = None):
         """
         internal, upload to target store
         :param artifact_path: required only for when generating target_path from artifact hash
@@ -758,13 +777,17 @@ def upload_extra_data(
     extra_data: dict,
     prefix="",
     update_spec=False,
-    artifact_path: str = None,
+    artifact_path: typing.Optional[str] = None,
 ):
     """upload extra data to the artifact store"""
     if not extra_data:
         return
     target_path = artifact.target_path
     for key, item in extra_data.items():
+        if item is ...:
+            # Skip future links (packagers feature for linking artifacts before they are logged)
+            continue
+
         if isinstance(item, bytes):
             if target_path:
                 target = os.path.join(target_path, prefix + key)

mlrun/artifacts/dataset.py

@@ -149,16 +149,16 @@ class DatasetArtifact(Artifact):
 
     def __init__(
         self,
-        key: str = None,
+        key: Optional[str] = None,
         df=None,
-        preview: int = None,
+        preview: Optional[int] = None,
         format: str = "",  # TODO: should be changed to 'fmt'.
-        stats: bool = None,
-        target_path: str = None,
-        extra_data: dict = None,
-        column_metadata: dict = None,
+        stats: Optional[bool] = None,
+        target_path: Optional[str] = None,
+        extra_data: Optional[dict] = None,
+        column_metadata: Optional[dict] = None,
         ignore_preview_limits: bool = False,
-        label_column: str = None,
+        label_column: Optional[str] = None,
         **kwargs,
     ):
         if key or format or target_path:
@@ -207,7 +207,7 @@ class DatasetArtifact(Artifact):
     def spec(self, spec):
         self._spec = self._verify_dict(spec, "spec", DatasetArtifactSpec)
 
-    def upload(self, artifact_path: str = None):
+    def upload(self, artifact_path: Optional[str] = None):
         """
         internal, upload to target store
         :param artifact_path: required only for when generating target_path from artifact hash
@@ -396,13 +396,13 @@ def get_df_stats(df):
 def update_dataset_meta(
     artifact,
     from_df=None,
-    schema: dict = None,
-    header: list = None,
-    preview: list = None,
-    stats: dict = None,
-    extra_data: dict = None,
-    column_metadata: dict = None,
-    labels: dict = None,
+    schema: Optional[dict] = None,
+    header: Optional[list] = None,
+    preview: Optional[list] = None,
+    stats: Optional[dict] = None,
+    extra_data: Optional[dict] = None,
+    column_metadata: Optional[dict] = None,
+    labels: Optional[dict] = None,
     ignore_preview_limits: bool = False,
 ):
     """Update dataset object attributes/metadata
@@ -495,4 +495,4 @@ def upload_dataframe(
         size = target_class(path=target_path).write_dataframe(df, **kw)
         return size, None
 
-    raise mlrun.errors.MLRunInvalidArgumentError(f"format {format} not implemented yet")
+    raise mlrun.errors.MLRunInvalidArgumentError(f"Format {format} not implemented yet")

mlrun/artifacts/manager.py

@@ -124,7 +124,7 @@ class ArtifactManager:
 
         self.artifact_db = db
         self.input_artifacts = {}
-        self.artifacts = {}
+        self.artifact_uris = {}
 
     @staticmethod
     def ensure_artifact_source_file_exists(item, path, body):
@@ -156,14 +156,12 @@ class ArtifactManager:
 
     def artifact_list(self, full=False):
         artifacts = []
-        for artifact in self.artifacts.values():
-            if isinstance(artifact, dict):
-                artifacts.append(artifact)
+        for artifacts_uri in self.artifact_uris.values():
+            artifact: Artifact = mlrun.datastore.get_store_resource(artifacts_uri)
+            if full:
+                artifacts.append(artifact.to_dict())
             else:
-                if full:
-                    artifacts.append(artifact.to_dict())
-                else:
-                    artifacts.append(artifact.base_dict())
+                artifacts.append(artifact.base_dict())
         return artifacts
 
     def log_artifact(
@@ -246,6 +244,8 @@ class ArtifactManager:
                 # otherwise, we do not want to override it.
                 # this is mainly relevant for imported artifacts that have an explicit db_key value already set
                 db_key = item.db_key or key
+        if db_key != key:
+            validate_artifact_key_name(db_key, "artifact.db_key")
         item.db_key = db_key or ""
         item.viewer = viewer or item.viewer
         item.tree = producer.tag
@@ -304,7 +304,7 @@ class ArtifactManager:
         item.target_path = target_path
 
         item.before_log()
-        self.artifacts[key] = item
+        self.artifact_uris[key] = item.uri
 
         if ((upload is None and item.kind != "dir") or upload) and not item.is_inline():
             # before uploading the item, we want to ensure that its tags are valid,
@@ -317,12 +317,12 @@ class ArtifactManager:
         size = str(item.size) or "?"
         db_str = "Y" if (self.artifact_db and db_key) else "N"
         logger.debug(
-            f"log artifact {key} at {item.target_path}, size: {size}, db: {db_str}"
+            f"Log artifact {key} at {item.target_path}, size: {size}, db: {db_str}"
         )
         return item
 
-    def update_artifact(self, producer, item):
-        self.artifacts[item.key] = item
+    def update_artifact(self, producer, item: Artifact):
+        self.artifact_uris[item.key] = item.uri
         self._log_to_db(item.db_key, producer.project, producer.inputs, item)
 
     def _log_to_db(self, key, project, sources, item, tag=None):
@@ -387,7 +387,7 @@ class ArtifactManager:
         deletion_strategy: mlrun.common.schemas.artifact.ArtifactsDeletionStrategies = (
             mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
         ),
-        secrets: dict = None,
+        secrets: typing.Optional[dict] = None,
     ):
         self.artifact_db.del_artifact(
             key=item.db_key,

mlrun/artifacts/model.py

@@ -303,7 +303,7 @@ class ModelArtifact(Artifact):
             self.metadata.labels = self.metadata.labels or {}
             self.metadata.labels["framework"] = self.spec.framework
 
-    def upload(self, artifact_path: str = None):
+    def upload(self, artifact_path: Optional[str] = None):
         """
         internal, upload to target store
         :param artifact_path: required only for when generating target_path from artifact hash
@@ -324,9 +324,7 @@ class ModelArtifact(Artifact):
             artifact=self, extra_data=self.spec.extra_data, artifact_path=artifact_path
         )
 
-        # the model spec yaml should not include the tag, as the same model can be used with different tags,
-        # and the tag is not part of the model spec but the metadata of the model artifact
-        spec_body = _remove_tag_from_spec_yaml(self)
+        spec_body = _sanitize_and_serialize_model_spec_yaml(self)
         spec_target_path = None
 
         if mlrun.mlconf.artifacts.generate_target_path_from_artifact_hash:
@@ -355,7 +353,7 @@ class ModelArtifact(Artifact):
     def _upload_body_or_file(
         self,
         artifact_path: str,
-        target_model_path: str = None,
+        target_model_path: Optional[str] = None,
     ):
         body = self.spec.get_body()
         if body:
@@ -403,12 +401,6 @@ class ModelArtifact(Artifact):
         return mlrun.get_dataitem(target_model_path).get()
 
 
-def _get_src_path(model_spec: ModelArtifact, filename):
-    if model_spec.src_path:
-        return path.join(model_spec.src_path, filename)
-    return filename
-
-
 def get_model(model_dir, suffix=""):
     """return model file, model spec object, and list of extra data items
 
@@ -483,49 +475,20 @@ def get_model(model_dir, suffix=""):
     return temp_path, model_spec, extra_dataitems
 
 
-def _load_model_spec(spec_path):
-    data = mlrun.datastore.store_manager.object(url=spec_path).get()
-    spec = yaml.load(data, Loader=yaml.FullLoader)
-    return ModelArtifact.from_dict(spec)
-
-
-def _get_file_path(base_path: str, name: str, isdir=False):
-    if not is_relative_path(name):
-        return name
-    if not isdir:
-        base_path = path.dirname(base_path)
-    return path.join(base_path, name).replace("\\", "/")
-
-
-def _get_extra(target, extra_data, is_dir=False):
-    extra_dataitems = {}
-    for k, v in extra_data.items():
-        extra_dataitems[k] = mlrun.datastore.store_manager.object(
-            url=_get_file_path(target, v, isdir=is_dir), key=k
-        )
-    return extra_dataitems
-
-
-def _remove_tag_from_spec_yaml(model_spec):
-    spec_dict = model_spec.to_dict()
-    spec_dict["metadata"].pop("tag", None)
-    return yaml.safe_dump(spec_dict)
-
-
 def update_model(
     model_artifact,
-    parameters: dict = None,
-    metrics: dict = None,
-    extra_data: dict = None,
-    inputs: list[Feature] = None,
-    outputs: list[Feature] = None,
-    feature_vector: str = None,
-    feature_weights: list = None,
+    parameters: Optional[dict] = None,
+    metrics: Optional[dict] = None,
+    extra_data: Optional[dict] = None,
+    inputs: Optional[list[Feature]] = None,
+    outputs: Optional[list[Feature]] = None,
+    feature_vector: Optional[str] = None,
+    feature_weights: Optional[list] = None,
     key_prefix: str = "",
-    labels: dict = None,
+    labels: Optional[dict] = None,
     write_spec_copy=True,
     store_object: bool = True,
-):
+) -> ModelArtifact:
     """Update model object attributes
 
     this method will edit or add attributes to a model object
@@ -593,10 +556,7 @@ def update_model(
 
     if write_spec_copy:
         spec_path = path.join(model_spec.target_path, model_spec_filename)
-
-        # the model spec yaml should not include the tag, as the same model can be used with different tags,
-        # and the tag is not part of the model spec but the metadata of the model artifact
-        model_spec_yaml = _remove_tag_from_spec_yaml(model_spec)
+        model_spec_yaml = _sanitize_and_serialize_model_spec_yaml(model_spec)
         mlrun.datastore.store_manager.object(url=spec_path).put(model_spec_yaml)
 
     model_spec.db_key = model_spec.db_key or model_spec.key
@@ -609,3 +569,56 @@ def update_model(
             project=model_spec.project,
         )
     return model_spec
+
+
+def _get_src_path(model_spec: ModelArtifact, filename: str) -> str:
+    return path.join(model_spec.src_path, filename) if model_spec.src_path else filename
+
+
+def _load_model_spec(spec_path) -> ModelArtifact:
+    data = mlrun.datastore.store_manager.object(url=spec_path).get()
+    spec = yaml.load(data, Loader=yaml.FullLoader)
+    return ModelArtifact.from_dict(spec)
+
+
+def _get_file_path(base_path: str, name: str, isdir: bool = False) -> str:
+    if not is_relative_path(name):
+        return name
+    if not isdir:
+        base_path = path.dirname(base_path)
+    return path.join(base_path, name).replace("\\", "/")
+
+
+def _get_extra(target: str, extra_data: dict, is_dir: bool = False) -> dict:
+    extra_dataitems = {}
+    for k, v in extra_data.items():
+        extra_dataitems[k] = mlrun.datastore.store_manager.object(
+            url=_get_file_path(target, v, isdir=is_dir), key=k
+        )
+    return extra_dataitems
+
+
+def _sanitize_and_serialize_model_spec_yaml(model: ModelArtifact) -> str:
+    model_dict = _sanitize_model_spec(model)
+    return _serialize_model_spec_yaml(model_dict)
+
+
+def _sanitize_model_spec(model: ModelArtifact) -> dict:
+    model_dict = model.to_dict()
+
+    # The model spec yaml should not include the tag, as the same model can be used with different tags,
+    # and the tag is not part of the model spec but the metadata of the model artifact
+    model_dict["metadata"].pop("tag", None)
+
+    # Remove future packaging links
+    if model_dict["spec"].get("extra_data"):
+        model_dict["spec"]["extra_data"] = {
+            key: item
+            for key, item in model_dict["spec"]["extra_data"].items()
+            if item is not ...
+        }
+    return model_dict
+
+
+def _serialize_model_spec_yaml(model_dict: dict) -> str:
+    return yaml.safe_dump(model_dict)

mlrun/common/constants.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import mlrun.common.types
 
 IMAGE_NAME_ENRICH_REGISTRY_PREFIX = "."  # prefix for image name to enrich with registry
 MLRUN_SERVING_CONF = "serving-conf"
@@ -86,3 +87,8 @@ class MLRunInternalLabels:
             for key, value in cls.__dict__.items()
             if not key.startswith("__") and isinstance(value, str)
         ]
+
+
+class DeployStatusTextKind(mlrun.common.types.StrEnum):
+    logs = "logs"
+    events = "events"

mlrun/common/formatters/__init__.py

@@ -19,3 +19,4 @@ from .pipeline import PipelineFormat  # noqa
 from .project import ProjectFormat  # noqa
 from .run import RunFormat  # noqa
 from .feature_set import FeatureSetFormat  # noqa
+from .model_endpoint import ModelEndpointFormat  # noqa

mlrun/common/formatters/feature_set.py

@@ -21,6 +21,7 @@ from .base import ObjectFormat
 
 
 class FeatureSetFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
     minimal = "minimal"
 
     @staticmethod

mlrun/common/formatters/function.py

@@ -21,6 +21,7 @@ from .base import ObjectFormat
 
 
 class FunctionFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
     minimal = "minimal"
 
     @staticmethod