mlrun 1.10.0rc3__py3-none-any.whl → 1.10.0rc4__py3-none-any.whl

mlrun/artifacts/__init__.py

@@ -24,6 +24,7 @@ from .base import (
 )
 from .dataset import DatasetArtifact, TableArtifact, update_dataset_meta
 from .document import DocumentArtifact, DocumentLoaderSpec, MLRunLoader
+from .llm_prompt import LLMPromptArtifact, LLMPromptArtifactSpec
 from .manager import (
     ArtifactManager,
     ArtifactProducer,

mlrun/artifacts/base.py

@@ -106,6 +106,8 @@ class ArtifactSpec(ModelObj):
         "extra_data",
         "unpackaging_instructions",
         "producer",
+        "parent_uri",
+        "has_children",
     ]
 
     _extra_fields = ["annotations", "sources", "license", "encoding"]
@@ -128,6 +130,7 @@ class ArtifactSpec(ModelObj):
         extra_data=None,
         body=None,
         unpackaging_instructions: typing.Optional[dict] = None,
+        parent_uri: typing.Optional[str] = None,
     ):
         self.src_path = src_path
         self.target_path = target_path
@@ -138,6 +141,8 @@ class ArtifactSpec(ModelObj):
         self.db_key = db_key
         self.extra_data = extra_data or {}
         self.unpackaging_instructions = unpackaging_instructions
+        self.parent_uri = parent_uri
+        self.has_children = False
 
         self._body = body
         self.encoding = None
@@ -253,7 +258,10 @@ class Artifact(ModelObj):
         self.spec.target_path = target_path or self.spec.target_path
         self.spec.format = format or self.spec.format
         self.spec.viewer = viewer or self.spec.viewer
-        self.spec.src_path = src_path
+        self.spec.src_path = src_path or self.spec.src_path
+
+        # temp flag to indicate if the source path is a temporary file (if True it will be deleted after upload)
+        self._src_is_temp = False
 
         if body:
             self.spec._body = body
@@ -341,7 +349,7 @@ class Artifact(ModelObj):
 
     def before_log(self):
         for key, item in self.spec.extra_data.items():
-            if hasattr(item, "get_target_path"):
+            if hasattr(item, "get_target_path") and item.get_target_path():
                 self.spec.extra_data[key] = item.get_target_path()
 
     @property
@@ -451,6 +459,10 @@ class Artifact(ModelObj):
             url=target_path or self.spec.target_path
         ).upload(source_path)
 
+        if self._src_is_temp and os.path.exists(self.spec.src_path):
+            # delete the temporary file if it was created for the upload
+            os.remove(self.spec.src_path)
+
     def resolve_body_target_hash_path(
         self, body: typing.Union[bytes, str], artifact_path: str
     ) -> (str, str):

mlrun/artifacts/helpers.py

@@ -0,0 +1,40 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mlrun.datastore
+
+
+def check_artifact_parent(
+    artifact_project: str,
+    expected_parent_uri: str,
+) -> None:
+    """
+    Check if the artifact's parent URI is valid and under the same project.
+    :param artifact_project:     Artifact project name
+    :param expected_parent_uri:  Expected parent URI of the artifact
+    :raise: MLRunInvalidArgumentError if the parent URI is invalid or not under the same project
+    """
+    # check if the parent_uri is a valid artifact uri and it is under the same project
+    if mlrun.datastore.is_store_uri(expected_parent_uri):
+        project, _, _, _, _, _ = mlrun.utils.parse_artifact_uri(
+            mlrun.datastore.parse_store_uri(expected_parent_uri)[1]
+        )
+        if project != artifact_project:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"parent_uri ({expected_parent_uri}) must be under the same project ({artifact_project})"
+            )
+    else:
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            f"parent_uri ({expected_parent_uri}) must be a valid artifact URI"
+        )

mlrun/artifacts/llm_prompt.py

@@ -0,0 +1,165 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import tempfile
+from typing import Optional, Union
+
+import mlrun
+import mlrun.artifacts.model as model_art
+import mlrun.common
+from mlrun.artifacts import Artifact, ArtifactMetadata, ArtifactSpec
+from mlrun.utils import StorePrefix, logger
+
+MAX_PROMPT_LENGTH = 1024
+
+
+class LLMPromptArtifactSpec(ArtifactSpec):
+    _dict_fields = ArtifactSpec._dict_fields + [
+        "prompt_string",
+        "prompt_legend",
+        "model_configuration",
+        "description",
+    ]
+
+    def __init__(
+        self,
+        model_artifact: Union[model_art.ModelArtifact, str] = None,
+        prompt_string: Optional[str] = None,
+        prompt_path: Optional[str] = None,
+        prompt_legend: Optional[dict] = None,
+        model_configuration: Optional[dict] = None,
+        description: Optional[str] = None,
+        target_path: Optional[str] = None,
+        **kwargs,
+    ):
+        if prompt_string and prompt_path:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Cannot specify both 'prompt_string' and 'prompt_path'"
+            )
+
+        super().__init__(
+            src_path=prompt_path,
+            target_path=target_path,
+            parent_uri=model_artifact.uri
+            if isinstance(model_artifact, model_art.ModelArtifact)
+            else model_artifact,
+            body=prompt_string,
+            **kwargs,
+        )
+
+        self.prompt_string = prompt_string
+        self.prompt_legend = prompt_legend
+        self.model_configuration = model_configuration
+        self.description = description
+        self._model_artifact = None
+
+    @property
+    def model_uri(self):
+        return self.parent_uri
+
+
+class LLMPromptArtifact(Artifact):
+    """
+    LLM Prompt Artifact
+
+    This artifact is used to store and manage LLM prompts.
+    Stores the prompt string/path and a link to the related model artifact.
+    """
+
+    kind = mlrun.common.schemas.ArtifactCategories.llm_prompt
+    _store_prefix = StorePrefix.LLMPrompt
+
+    def __init__(
+        self,
+        key: Optional[str] = None,
+        project: Optional[str] = None,
+        model_artifact: Union[
+            model_art.ModelArtifact, str
+        ] = None,  # TODO support partial model uri
+        prompt_string: Optional[str] = None,
+        prompt_path: Optional[str] = None,
+        prompt_legend: Optional[dict] = None,
+        model_configuration: Optional[dict] = None,
+        description: Optional[str] = None,
+        target_path=None,
+        **kwargs,
+    ):
+        llm_prompt_spec = LLMPromptArtifactSpec(
+            prompt_string=prompt_string,
+            prompt_path=prompt_path,
+            prompt_legend=prompt_legend,
+            model_artifact=model_artifact,
+            model_configuration=model_configuration,
+            target_path=target_path,
+            description=description,
+        )
+
+        llm_metadata = ArtifactMetadata(
+            key=key,
+            project=project or "",
+        )
+
+        super().__init__(spec=llm_prompt_spec, metadata=llm_metadata, **kwargs)
+
+    @property
+    def spec(self) -> LLMPromptArtifactSpec:
+        return self._spec
+
+    @spec.setter
+    def spec(self, spec: LLMPromptArtifactSpec):
+        self._spec = self._verify_dict(spec, "spec", LLMPromptArtifactSpec)
+
+    @property
+    def model_artifact(self) -> Optional[model_art.ModelArtifact]:
+        """
+        Get the model artifact linked to this prompt artifact.
+        """
+        if self.spec._model_artifact:
+            return self.spec._model_artifact
+        if self.spec.model_uri:
+            self.spec._model_artifact, target = (
+                mlrun.datastore.store_manager.get_store_artifact(self.spec.model_uri)
+            )
+            return self.spec._model_artifact
+        return None
+
+    def read_prompt(self) -> Optional[str]:
+        """
+        Read the prompt string from the artifact.
+        """
+        if self.spec.prompt_string:
+            return self.spec.prompt_string
+        if self.spec.target_path:
+            with mlrun.datastore.store_manager.object(url=self.spec.target_path).open(
+                mode="r"
+            ) as p_file:
+                return p_file.read()
+
+    def before_log(self):
+        """
+        Prepare the artifact before logging.
+        This method is called before the artifact is logged.
+        """
+        if self.spec.prompt_string and len(self.spec.prompt_string) > MAX_PROMPT_LENGTH:
+            logger.debug(
+                "Prompt string exceeds maximum length, saving to a temporary file."
+            )
+            with tempfile.NamedTemporaryFile(
+                delete=False, mode="w", suffix=".txt"
+            ) as temp_file:
+                temp_file.write(self.spec.prompt_string)
+            self.spec.src_path = temp_file.name
+            self.spec.prompt_string = None
+            self._src_is_temp = True
+
+        super().before_log()

mlrun/artifacts/manager.py

@@ -17,6 +17,7 @@ import typing
 from os.path import exists, isdir
 from urllib.parse import urlparse
 
+import mlrun.artifacts.helpers
 import mlrun.common.schemas.artifact
 import mlrun.config
 import mlrun.utils.regex
@@ -42,6 +43,7 @@ from .dataset import (
     TableArtifact,
 )
 from .document import DocumentArtifact
+from .llm_prompt import LLMPromptArtifact
 from .model import ModelArtifact
 from .plots import (
     PlotArtifact,
@@ -59,6 +61,7 @@ artifact_types = {
     "dataset": DatasetArtifact,
     "plotly": PlotlyArtifact,
     "document": DocumentArtifact,
+    "llm-prompt": LLMPromptArtifact,
 }
 
 
@@ -221,7 +224,12 @@ class ArtifactManager:
         else:
             key = item.key
             target_path = target_path or item.target_path
-
+        if isinstance(item, ModelArtifact) and item.model_url:
+            if upload:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "log_artifact of ModelArtifact does not accept arguments for both upload and model_url parameters"
+                )
+            upload = False
         validate_artifact_key_name(key, "artifact.key")
         validate_inline_artifact_body_size(item.spec.inline)
         src_path = local_path or item.src_path  # TODO: remove src_path
@@ -262,6 +270,10 @@ class ArtifactManager:
         item.iter = producer.iteration
         project = project or producer.project
         item.project = project
+        if item.spec.parent_uri:
+            mlrun.artifacts.helpers.check_artifact_parent(
+                artifact_project=item.project, expected_parent_uri=item.spec.parent_uri
+            )
         if is_retained_producer:
             # if the producer is retained, we want to use the original target path
             target_path = target_path or item.target_path

mlrun/artifacts/model.py

@@ -15,7 +15,7 @@
 import tempfile
 import warnings
 from os import path
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 import pandas as pd
 import yaml
@@ -46,6 +46,7 @@ class ModelArtifactSpec(ArtifactSpec):
         "feature_weights",
         "feature_stats",
         "model_target_file",
+        "model_url",
     ]
     _exclude_fields_from_uid_hash = ArtifactSpec._exclude_fields_from_uid_hash + [
         "metrics",
@@ -79,6 +80,7 @@ class ModelArtifactSpec(ArtifactSpec):
         feature_weights=None,
         feature_stats=None,
         model_target_file=None,
+        model_url=None,
     ):
         super().__init__(
             src_path,
@@ -102,6 +104,7 @@ class ModelArtifactSpec(ArtifactSpec):
         self.feature_weights = feature_weights
         self.feature_stats = feature_stats
         self.model_target_file = model_target_file
+        self.model_url = model_url
 
     @property
     def inputs(self) -> ObjectList:
@@ -121,6 +124,18 @@ class ModelArtifactSpec(ArtifactSpec):
     def outputs(self, outputs: list[Feature]) -> None:
         self._outputs = ObjectList.from_list(Feature, outputs)
 
+    @property
+    def default_config(self):
+        return self.parameters.get("default_config", {})
+
+    @default_config.setter
+    def default_config(self, default_config):
+        # skip storing 'default_config' if value is empty or unset
+        if default_config:
+            self.parameters["default_config"] = default_config
+        else:
+            self.parameters.pop("default_config", None)
+
 
 class ModelArtifact(Artifact):
     """ML Model artifact
@@ -148,8 +163,32 @@ class ModelArtifact(Artifact):
         feature_weights=None,
         extra_data=None,
         model_dir=None,
+        model_url: Optional[str] = None,
+        default_config: Optional[dict] = None,
         **kwargs,
     ):
+        """
+        :param key:             Artifact key or artifact class ()
+        :param body:            Will use the body as the artifact content
+        :param format:          Optional, format to use (e.g. csv, parquet, ..)
+        :param model_file:      Path to the local model file we upload (see also model_dir)
+                                or to a model file data url (e.g. `http://host/path/model.pkl`)
+        :param metrics:         The key/value dict of model metrics
+        :param target_path:     Absolute target path (instead of using artifact_path + local_path)
+        :param parameters:      Key/value dict of model parameters
+        :param inputs:          Ordered list of model input features (name, type, ..)
+        :param outputs:         Ordered list of model output/result elements (name, type, ..)
+        :param framework:       Name of the ML framework
+        :param algorithm:       Training algorithm name
+        :param feature_vector:  Feature store feature vector uri (store://feature-vectors/<project>/<name>[:tag])
+        :param feature_weights: List of feature weights, one per input column
+        :param extra_data:      Extra artifacts and files to log with the model.
+        :param model_dir:       Path to the local dir holding the model file and extra files
+        :param model_url:       Remote model url.
+        :param default_config:  Default configuration for client building
+                                Saved as a sub-dictionary under the parameter.
+        :param kwargs:
+        """
         if key or body or format or target_path:
             warnings.warn(
                 "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
@@ -158,10 +197,18 @@ class ModelArtifact(Artifact):
             )
         super().__init__(key, body, format=format, target_path=target_path, **kwargs)
         model_file = str(model_file or "")
+        if model_file and model_url:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Arguments 'model_file' and 'model_dir' cannot be"
+                " used together with 'model_url'."
+            )
         if model_file and "/" in model_file:
+            if model_dir:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "'model_file' cannot contain '/' (i.e., be a full path) when 'model_dir' is also specified"
+                )
             model_dir = path.dirname(model_file)
             model_file = path.basename(model_file)
-
         self.spec.model_file = model_file
         self.spec.src_path = model_dir
         self.spec.parameters = parameters or {}
@@ -174,6 +221,8 @@ class ModelArtifact(Artifact):
         self.spec.feature_vector = feature_vector
         self.spec.feature_weights = feature_weights
         self.spec.feature_stats = None
+        self.spec.model_url = model_url
+        self.default_config = default_config
 
     @property
     def spec(self) -> ModelArtifactSpec:
@@ -211,6 +260,14 @@ class ModelArtifact(Artifact):
     def model_file(self, model_file):
         self.spec.model_file = model_file
 
+    @property
+    def model_url(self):
+        return self.spec.model_url
+
+    @model_url.setter
+    def model_url(self, model_url):
+        self.spec.model_url = model_url
+
     @property
     def parameters(self):
         return self.spec.parameters
@@ -219,6 +276,14 @@ class ModelArtifact(Artifact):
     def parameters(self, parameters):
         self.spec.parameters = parameters
 
+    @property
+    def default_config(self):
+        return self.spec.default_config
+
+    @default_config.setter
+    def default_config(self, default_config):
+        self.spec.default_config = default_config
+
     @property
     def metrics(self):
         return self.spec.metrics
@@ -299,8 +364,10 @@ class ModelArtifact(Artifact):
         return True
 
     def before_log(self):
-        if not self.spec.model_file:
-            raise ValueError("model_file attr must be specified")
+        if not self.spec.model_file and not self.spec.model_url:
+            raise ValueError(
+                "ModelArtifact must have either model_file or model_url attributes"
+            )
 
         super().before_log()
 
@@ -406,8 +473,13 @@ class ModelArtifact(Artifact):
         return mlrun.get_dataitem(target_model_path).get()
 
 
-def get_model(model_dir, suffix=""):
-    """return model file, model spec object, and list of extra data items
+def get_model(
+    model_dir: Optional[
+        Union[str, ModelArtifact, "mlrun.datastore.base.DataItem"]
+    ] = None,
+    suffix="",
+) -> (str, ModelArtifact, dict):
+    """return model file, model spec object, and dictionary of extra data items
 
     this function will get the model file, metadata, and extra data
     the returned model file is always local, when using remote urls
@@ -428,6 +500,7 @@ def get_model(model_dir, suffix=""):
     :returns: model filename, model artifact object, extra data dict
 
     """
+    # TODO support LLMPromptArtifact
     model_file = ""
     model_spec = None
     extra_dataitems = {}
@@ -435,18 +508,25 @@ def get_model(model_dir, suffix=""):
 
     if hasattr(model_dir, "artifact_url"):
         model_dir = model_dir.artifact_url
-
     alternative_suffix = next(
         (
             optional_suffix
             for optional_suffix in MODEL_OPTIONAL_SUFFIXES
-            if model_dir.lower().endswith(optional_suffix)
+            if isinstance(model_dir, str)
+            and model_dir.lower().endswith(optional_suffix)
         ),
         None,
     )
-
-    if mlrun.datastore.is_store_uri(model_dir):
-        model_spec, target = mlrun.datastore.store_manager.get_store_artifact(model_dir)
+    is_store_uri = isinstance(model_dir, str) and mlrun.datastore.is_store_uri(
+        model_dir
+    )
+    if is_store_uri or isinstance(model_dir, ModelArtifact):
+        if is_store_uri:
+            model_spec, target = mlrun.datastore.store_manager.get_store_artifact(
+                model_dir
+            )
+        else:
+            model_spec, target = model_dir, model_dir.get_target_path()
         if not model_spec or model_spec.kind != "model":
             raise ValueError(f"store artifact ({model_dir}) is not model kind")
         # in case model_target_file is specified, use it, because that means that the actual model target path

mlrun/common/formatters/artifact.py

@@ -40,6 +40,7 @@ class ArtifactFormat(ObjectFormat, mlrun.common.types.StrEnum):
                     "spec.algorithm",
                     "spec.metrics",
                     "spec.target_path",
+                    "spec.parent_uri",
                 ]
             ),
         }[_format]

mlrun/common/runtimes/constants.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import enum
 import typing
 
 import mlrun.common.constants as mlrun_constants
@@ -237,19 +236,6 @@ class RunStates:
         }[pipeline_run_status]
 
 
-# TODO: remove this class in 1.10.0 - use only MlrunInternalLabels
-class RunLabels(enum.Enum):
-    owner = mlrun_constants.MLRunInternalLabels.owner
-    v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user
-
-    @staticmethod
-    def all():
-        return [
-            RunLabels.owner,
-            RunLabels.v3io_user,
-        ]
-
-
 class SparkApplicationStates:
     """
     https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go#L321

mlrun/common/schemas/artifact.py

@@ -15,7 +15,6 @@
 import typing
 
 import pydantic.v1
-from deprecated import deprecated
 
 import mlrun.common.types
 
@@ -26,6 +25,7 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
     model = "model"
     dataset = "dataset"
     document = "document"
+    llm_prompt = "llm-prompt"
     other = "other"
 
     # we define the link as a category to prevent import cycles, but it's not a real category
@@ -41,19 +41,27 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
             return [ArtifactCategories.dataset.value, link_kind], False
         if self.value == ArtifactCategories.document.value:
             return [ArtifactCategories.document.value, link_kind], False
+        if self.value == ArtifactCategories.llm_prompt.value:
+            return [ArtifactCategories.llm_prompt.value, link_kind], False
         if self.value == ArtifactCategories.other.value:
             return (
                 [
                     ArtifactCategories.model.value,
                     ArtifactCategories.dataset.value,
                     ArtifactCategories.document.value,
+                    ArtifactCategories.llm_prompt.value,
                 ],
                 True,
             )
 
     @classmethod
     def from_kind(cls, kind: str) -> "ArtifactCategories":
-        if kind in [cls.model.value, cls.dataset.value, cls.document.value]:
+        if kind in [
+            cls.model.value,
+            cls.dataset.value,
+            cls.document.value,
+            cls.llm_prompt.value,
+        ]:
             return cls(kind)
         return cls.other
 
@@ -64,6 +72,7 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
             ArtifactCategories.model,
             ArtifactCategories.dataset,
             ArtifactCategories.document,
+            ArtifactCategories.llm_prompt,
         ]
 
 
@@ -78,16 +87,6 @@ class ArtifactIdentifier(pydantic.v1.BaseModel):
     # hash: typing.Optional[str]
 
 
-@deprecated(
-    version="1.7.0",
-    reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.10.0. "
-    "Use mlrun.common.formatters.ArtifactFormat instead.",
-    category=FutureWarning,
-)
-class ArtifactsFormat(mlrun.common.types.StrEnum):
-    full = "full"
-
-
 class ArtifactMetadata(pydantic.v1.BaseModel):
     key: str
     project: str
@@ -108,6 +107,7 @@ class ArtifactSpec(pydantic.v1.BaseModel):
     db_key: typing.Optional[str]
     extra_data: typing.Optional[dict[str, typing.Any]]
     unpackaging_instructions: typing.Optional[dict[str, typing.Any]]
+    parent_uri: typing.Optional[str]
 
     class Config:
         extra = pydantic.v1.Extra.allow

mlrun/common/schemas/pipeline.py

@@ -15,22 +15,6 @@
 import typing
 
 import pydantic.v1
-from deprecated import deprecated
-
-import mlrun.common.types
-
-
-@deprecated(
-    version="1.7.0",
-    reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.10.0. "
-    "Use mlrun.common.formatters.PipelineFormat instead.",
-    category=FutureWarning,
-)
-class PipelinesFormat(mlrun.common.types.StrEnum):
-    full = "full"
-    metadata_only = "metadata_only"
-    summary = "summary"
-    name_only = "name_only"
 
 
 class PipelinesPagination(str):