mlrun 1.7.1rc10__py3-none-any.whl → 1.8.0rc11__py3-none-any.whl

mlrun/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 __all__ = [
     "get_version",
     "set_environment",
@@ -29,9 +27,9 @@ __all__ = [
 ]
 
 from os import environ, path
+from typing import Optional
 
 import dotenv
-import mlrun_pipelines
 
 from .config import config as mlconf
 from .datastore import DataItem, store_manager
@@ -41,6 +39,7 @@ from .execution import MLClientCtx
 from .model import RunObject, RunTemplate, new_task
 from .package import ArtifactType, DefaultPackager, Packager, handler
 from .projects import (
+    MlrunProject,
     ProjectMetadata,
     build_function,
     deploy_function,
@@ -61,18 +60,19 @@ from .run import (
     get_pipeline,
     import_function,
     new_function,
+    retry_pipeline,
     wait_for_pipeline_completion,
 )
-from .runtimes import new_model_server
+from .runtimes import mounts, new_model_server
 from .secrets import get_secret_or_env
 from .utils.version import Version
 
 __version__ = Version().get()["version"]
 
-VolumeMount = mlrun_pipelines.common.mounts.VolumeMount
-mount_v3io = mlrun_pipelines.mounts.mount_v3io
-v3io_cred = mlrun_pipelines.mounts.v3io_cred
-auto_mount = mlrun_pipelines.mounts.auto_mount
+VolumeMount = mounts.VolumeMount
+mount_v3io = mounts.mount_v3io
+v3io_cred = mounts.v3io_cred
+auto_mount = mounts.auto_mount
 
 
 def get_version():
@@ -88,12 +88,12 @@ if "IGZ_NAMESPACE_DOMAIN" in environ:
 
 
 def set_environment(
-    api_path: str = None,
+    api_path: Optional[str] = None,
     artifact_path: str = "",
-    access_key: str = None,
-    username: str = None,
-    env_file: str = None,
-    mock_functions: str = None,
+    access_key: Optional[str] = None,
+    username: Optional[str] = None,
+    env_file: Optional[str] = None,
+    mock_functions: Optional[str] = None,
 ):
     """set and test default config for: api path, artifact_path and project
 
@@ -136,15 +136,16 @@ def set_environment(
     if not mlconf.dbpath:
         raise ValueError("DB/API path was not detected, please specify its address")
 
-    if mock_functions is not None:
-        mock_functions = "1" if mock_functions is True else mock_functions
-        mlconf.force_run_local = mock_functions
-        mlconf.mock_nuclio_deployment = mock_functions
-
     # check connectivity and load remote defaults
     get_run_db()
     if api_path:
         environ["MLRUN_DBPATH"] = mlconf.dbpath
+    mlconf.reload()
+
+    if mock_functions is not None:
+        mock_functions = "1" if mock_functions is True else mock_functions
+        mlconf.force_run_local = mock_functions
+        mlconf.mock_nuclio_deployment = mock_functions
 
     if not mlconf.artifact_path and not artifact_path:
         raise ValueError(
@@ -159,13 +160,14 @@ def set_environment(
                 "artifact_path must refer to an absolute path" " or a valid url"
             )
         mlconf.artifact_path = artifact_path
+
     return mlconf.default_project, mlconf.artifact_path
 
 
-def get_current_project(silent=False):
+def get_current_project(silent: bool = False) -> Optional[MlrunProject]:
     if not pipeline_context.project and not silent:
         raise MLRunInvalidArgumentError(
-            "current project is not initialized, use new, get or load project methods first"
+            "No current project is initialized. Use new, get or load project functions first."
         )
     return pipeline_context.project
 
@@ -182,7 +184,7 @@ def get_sample_path(subpath=""):
     return samples_path
 
 
-def set_env_from_file(env_file: str, return_dict: bool = False):
+def set_env_from_file(env_file: str, return_dict: bool = False) -> Optional[dict]:
     """Read and set and/or return environment variables from a file
     the env file should have lines in the form KEY=VALUE, comment line start with "#"
 

mlrun/__main__.py

@@ -27,13 +27,13 @@ import click
 import dotenv
 import pandas as pd
 import yaml
-from mlrun_pipelines.mounts import auto_mount as auto_mount_modifier
 from tabulate import tabulate
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 from mlrun.common.helpers import parse_versioned_object_uri
+from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
 
 from .config import config as mlconf
 from .db import get_run_db
@@ -1248,10 +1248,10 @@ def show_or_set_config(
         }
         for key, value in env_dict.items():
             if value:
-                dotenv.set_key(filename, key, value, quote_mode="")
+                dotenv.set_key(filename, key, value, quote_mode="always")
         if env_vars:
             for key, value in list2dict(env_vars).items():
-                dotenv.set_key(filename, key, value, quote_mode="")
+                dotenv.set_key(filename, key, value, quote_mode="always")
         if env_file:
             # if its not the default file print the usage details
             print(

mlrun/alerts/alert.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from typing import Union
+from datetime import datetime
+from typing import Optional, Union
 
 import mlrun
 import mlrun.common.schemas.alert as alert_objects
@@ -30,6 +30,7 @@ class AlertConfig(ModelObj):
         "state",
         "count",
         "created",
+        "updated",
     ]
     _fields_to_serialize = ModelObj._fields_to_serialize + [
         "entities",
@@ -40,21 +41,22 @@ class AlertConfig(ModelObj):
 
     def __init__(
         self,
-        project: str = None,
-        name: str = None,
+        project: Optional[str] = None,
+        name: Optional[str] = None,
         template: Union[alert_objects.AlertTemplate, str] = None,
-        description: str = None,
-        summary: str = None,
+        description: Optional[str] = None,
+        summary: Optional[str] = None,
         severity: alert_objects.AlertSeverity = None,
         trigger: alert_objects.AlertTrigger = None,
         criteria: alert_objects.AlertCriteria = None,
         reset_policy: alert_objects.ResetPolicy = None,
-        notifications: list[alert_objects.AlertNotification] = None,
+        notifications: Optional[list[alert_objects.AlertNotification]] = None,
         entities: alert_objects.EventEntities = None,
-        id: int = None,
+        id: Optional[int] = None,
         state: alert_objects.AlertActiveState = None,
-        created: str = None,
-        count: int = None,
+        created: Optional[str] = None,
+        count: Optional[int] = None,
+        updated: Optional[str] = None,
     ):
         """Alert config object
 
@@ -118,6 +120,7 @@ class AlertConfig(ModelObj):
         :param state:          State of the alert, may be active/inactive (user should not supply it)
         :param created:        When the alert is created (user should not supply it)
         :param count:          Internal counter of the alert (user should not supply it)
+        :param updated:        The last update time of the alert (user should not supply it)
         """
         self.project = project
         self.name = name
@@ -131,18 +134,45 @@ class AlertConfig(ModelObj):
         self.entities = entities
         self.id = id
         self.state = state
-        self.created = created
+        self._created = created
         self.count = count
+        self._updated = updated
 
         if template:
             self._apply_template(template)
 
+    @property
+    def created(self) -> datetime:
+        """
+        Get the `created` field as a datetime object.
+        """
+        if isinstance(self._created, str):
+            return datetime.fromisoformat(self._created)
+        return self._created
+
+    @created.setter
+    def created(self, created):
+        self._created = created
+
+    @property
+    def updated(self) -> datetime:
+        """
+        Get the `updated` field as a datetime object.
+        """
+        if isinstance(self._updated, str):
+            return datetime.fromisoformat(self._updated)
+        return self._updated
+
+    @updated.setter
+    def updated(self, updated):
+        self._updated = updated
+
     def validate_required_fields(self):
         if not self.name:
             raise mlrun.errors.MLRunInvalidArgumentError("Alert name must be provided")
 
     def _serialize_field(
-        self, struct: dict, field_name: str = None, strip: bool = False
+        self, struct: dict, field_name: Optional[str] = None, strip: bool = False
     ):
         if field_name == "entities":
             if self.entities:
@@ -179,7 +209,12 @@ class AlertConfig(ModelObj):
             return None
         return super()._serialize_field(struct, field_name, strip)
 
-    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+    def to_dict(
+        self,
+        fields: Optional[list] = None,
+        exclude: Optional[list] = None,
+        strip: bool = False,
+    ):
         if self.entities is None:
             raise mlrun.errors.MLRunBadRequestError("Alert entity field is missing")
         if not self.notifications:
@@ -189,7 +224,9 @@ class AlertConfig(ModelObj):
         return super().to_dict(self._dict_fields)
 
     @classmethod
-    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+    def from_dict(
+        cls, struct=None, fields=None, deprecated_fields: Optional[dict] = None
+    ):
         new_obj = super().from_dict(struct, fields=fields)
 
         entity_data = struct.get("entities")
@@ -246,3 +283,100 @@ class AlertConfig(ModelObj):
         self.criteria = self.criteria or template.criteria
         self.trigger = self.trigger or template.trigger
         self.reset_policy = self.reset_policy or template.reset_policy
+
+    def list_activations(
+        self,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        from_last_update: bool = False,
+    ) -> list[mlrun.common.schemas.alert.AlertActivation]:
+        """
+        Retrieve a list of all alert activations.
+
+        :param since: Filters for alert activations occurring after this timestamp.
+        :param until: Filters for alert activations occurring before this timestamp.
+        :param from_last_update: If set to True, retrieves alert activations since the alert's last update time.
+                                 if both since and from_last_update=True are provided, from_last_update takes precedence
+                                 and the since value will be overridden by the alert's last update timestamp.
+
+        :returns: A list of alert activations matching the provided filters.
+        """
+        db = mlrun.get_run_db()
+        if from_last_update and self._updated:
+            since = self.updated
+
+        return db.list_alert_activations(
+            project=self.project,
+            name=self.name,
+            since=since,
+            until=until,
+        )
+
+    def paginated_list_activations(
+        self,
+        *args,
+        page: Optional[int] = None,
+        page_size: Optional[int] = None,
+        page_token: Optional[str] = None,
+        from_last_update: bool = False,
+        **kwargs,
+    ) -> tuple[mlrun.common.schemas.alert.AlertActivation, Optional[str]]:
+        """
+        List alerts activations with support for pagination and various filtering options.
+
+        This method retrieves a paginated list of alert activations based on the specified filter parameters.
+        Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
+        will return a list of alert activations that match the filtering criteria provided.
+
+        For detailed information about the parameters, refer to the list_activations method:
+            See :py:func:`~list_activations` for more details.
+
+        Examples::
+
+            # Fetch first page of alert activations with page size of 5
+            alert_activations, token = alert_config.paginated_list_activations(page_size=5)
+            # Fetch next page using the pagination token from the previous response
+            alert_activations, token = alert_config.paginated_list_activations(
+                page_token=token
+            )
+            # Fetch alert activations for a specific page (e.g., page 3)
+            alert_activations, token = alert_config.paginated_list_activations(
+                page=3, page_size=5
+            )
+
+            # Automatically iterate over all pages without explicitly specifying the page number
+            alert_activations = []
+            token = None
+            while True:
+                page_alert_activations, token = alert_config.paginated_list_activations(
+                    page_token=token, page_size=5
+                )
+                alert_activations.extend(page_alert_activations)
+
+                # If token is None and page_alert_activations is empty, we've reached the end (no more activations).
+                # If token is None and page_alert_activations is not empty, we've fetched the last page of activations.
+                if not token:
+                    break
+            print(f"Total alert activations retrieved: {len(alert_activations)}")
+
+        :param page: The page number to retrieve. If not provided, the next page will be retrieved.
+        :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
+        :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
+            for the first request.
+        :param from_last_update: If set to True, retrieves alert activations since the alert's last update time.
+
+        :returns: A tuple containing the list of alert activations and an optional `page_token` for pagination.
+        """
+        if from_last_update and self._updated:
+            kwargs["since"] = self.updated
+
+        db = mlrun.get_run_db()
+        return db.paginated_list_alert_activations(
+            *args,
+            project=self.project,
+            name=self.name,
+            page=page,
+            page_size=page_size,
+            page_token=page_token,
+            **kwargs,
+        )

mlrun/artifacts/__init__.py

@@ -12,10 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 # Don't remove this, used by sphinx documentation
-__all__ = ["get_model", "update_model"]
+__all__ = ["get_model", "update_model", "DocumentLoaderSpec", "MLRunLoader"]
 
 from .base import (
     Artifact,
@@ -25,6 +23,7 @@ from .base import (
     get_artifact_meta,
 )
 from .dataset import DatasetArtifact, TableArtifact, update_dataset_meta
+from .document import DocumentArtifact, DocumentLoaderSpec, MLRunLoader
 from .manager import (
     ArtifactManager,
     ArtifactProducer,

mlrun/artifacts/base.py

@@ -36,7 +36,16 @@ from ..utils import (
 
 
 class ArtifactMetadata(ModelObj):
-    _dict_fields = ["key", "project", "iter", "tree", "description", "hash", "tag"]
+    _dict_fields = [
+        "key",
+        "project",
+        "iter",
+        "tree",
+        "description",
+        "hash",
+        "tag",
+        "uid",
+    ]
     _extra_fields = ["updated", "labels"]
 
     def __init__(
@@ -48,6 +57,7 @@ class ArtifactMetadata(ModelObj):
         description=None,
         hash=None,
         tag=None,
+        uid=None,
     ):
         self.key = key
         self.project = project
@@ -58,18 +68,26 @@ class ArtifactMetadata(ModelObj):
         self.labels = {}
         self.updated = None
         self.tag = tag  # temp store of the tag
+        self.uid = uid
 
     def base_dict(self):
         return super().to_dict()
 
-    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+    def to_dict(
+        self,
+        fields: typing.Optional[list] = None,
+        exclude: typing.Optional[list] = None,
+        strip: bool = False,
+    ):
         """return long dict form of the artifact"""
         return super().to_dict(
             self._dict_fields + self._extra_fields, exclude=exclude, strip=strip
         )
 
     @classmethod
-    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+    def from_dict(
+        cls, struct=None, fields=None, deprecated_fields: typing.Optional[dict] = None
+    ):
         fields = fields or cls._dict_fields + cls._extra_fields
         return super().from_dict(
             struct, fields=fields, deprecated_fields=deprecated_fields
@@ -109,7 +127,7 @@ class ArtifactSpec(ModelObj):
         db_key=None,
         extra_data=None,
         body=None,
-        unpackaging_instructions: dict = None,
+        unpackaging_instructions: typing.Optional[dict] = None,
     ):
         self.src_path = src_path
         self.target_path = target_path
@@ -131,14 +149,21 @@ class ArtifactSpec(ModelObj):
     def base_dict(self):
         return super().to_dict()
 
-    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+    def to_dict(
+        self,
+        fields: typing.Optional[list] = None,
+        exclude: typing.Optional[list] = None,
+        strip: bool = False,
+    ):
         """return long dict form of the artifact"""
         return super().to_dict(
             self._dict_fields + self._extra_fields, exclude=exclude, strip=strip
         )
 
     @classmethod
-    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+    def from_dict(
+        cls, struct=None, fields=None, deprecated_fields: typing.Optional[dict] = None
+    ):
         fields = fields or cls._dict_fields + cls._extra_fields
         return super().from_dict(
             struct, fields=fields, deprecated_fields=deprecated_fields
@@ -192,7 +217,7 @@ class Artifact(ModelObj):
         size=None,
         target_path=None,
         project=None,
-        src_path: str = None,
+        src_path: typing.Optional[str] = None,
         # All params up until here are legacy params for compatibility with legacy artifacts.
         # TODO: remove them in 1.9.0.
         metadata: ArtifactMetadata = None,
@@ -354,6 +379,7 @@ class Artifact(ModelObj):
             iter=self.metadata.iter,
             tree=tree,
             tag=tag,
+            uid=self.uid,
         )
         return mlrun.datastore.get_store_uri(self._store_prefix, uri)
 
@@ -366,7 +392,7 @@ class Artifact(ModelObj):
                 struct[field] = val.base_dict()
         return struct
 
-    def upload(self, artifact_path: str = None):
+    def upload(self, artifact_path: typing.Optional[str] = None):
         """
         internal, upload to target store
         :param artifact_path: required only for when generating target_path from artifact hash
@@ -379,7 +405,9 @@ class Artifact(ModelObj):
             if src_path and os.path.isfile(src_path):
                 self._upload_file(source_path=src_path, artifact_path=artifact_path)
 
-    def _upload_body(self, body, target=None, artifact_path: str = None):
+    def _upload_body(
+        self, body, target=None, artifact_path: typing.Optional[str] = None
+    ):
         body_hash = None
         if not target and not self.spec.target_path:
             if not mlrun.mlconf.artifacts.generate_target_path_from_artifact_hash:
@@ -400,7 +428,10 @@ class Artifact(ModelObj):
         )
 
     def _upload_file(
-        self, source_path: str, target_path: str = None, artifact_path: str = None
+        self,
+        source_path: str,
+        target_path: typing.Optional[str] = None,
+        artifact_path: typing.Optional[str] = None,
     ):
         file_hash = None
         if not target_path and not self.spec.target_path:
@@ -623,6 +654,14 @@ class Artifact(ModelObj):
     def hash(self, hash):
         self.metadata.hash = hash
 
+    @property
+    def uid(self):
+        return self.metadata.uid
+
+    @uid.setter
+    def uid(self, uid):
+        self.metadata.uid = uid
+
     def generate_target_path(self, artifact_path, producer):
         return generate_target_path(self, artifact_path, producer)
 
@@ -651,7 +690,7 @@ class DirArtifact(Artifact):
     def is_dir(self):
         return True
 
-    def upload(self, artifact_path: str = None):
+    def upload(self, artifact_path: typing.Optional[str] = None):
         """
         internal, upload to target store
         :param artifact_path: required only for when generating target_path from artifact hash
@@ -758,13 +797,17 @@ def upload_extra_data(
     extra_data: dict,
     prefix="",
     update_spec=False,
-    artifact_path: str = None,
+    artifact_path: typing.Optional[str] = None,
 ):
     """upload extra data to the artifact store"""
     if not extra_data:
         return
     target_path = artifact.target_path
     for key, item in extra_data.items():
+        if item is ...:
+            # Skip future links (packagers feature for linking artifacts before they are logged)
+            continue
+
         if isinstance(item, bytes):
             if target_path:
                 target = os.path.join(target_path, prefix + key)

mlrun/artifacts/dataset.py

@@ -149,16 +149,16 @@ class DatasetArtifact(Artifact):
 
     def __init__(
         self,
-        key: str = None,
+        key: Optional[str] = None,
         df=None,
-        preview: int = None,
+        preview: Optional[int] = None,
         format: str = "",  # TODO: should be changed to 'fmt'.
-        stats: bool = None,
-        target_path: str = None,
-        extra_data: dict = None,
-        column_metadata: dict = None,
+        stats: Optional[bool] = None,
+        target_path: Optional[str] = None,
+        extra_data: Optional[dict] = None,
+        column_metadata: Optional[dict] = None,
         ignore_preview_limits: bool = False,
-        label_column: str = None,
+        label_column: Optional[str] = None,
         **kwargs,
     ):
         if key or format or target_path:
@@ -207,7 +207,7 @@ class DatasetArtifact(Artifact):
     def spec(self, spec):
         self._spec = self._verify_dict(spec, "spec", DatasetArtifactSpec)
 
-    def upload(self, artifact_path: str = None):
+    def upload(self, artifact_path: Optional[str] = None):
         """
         internal, upload to target store
         :param artifact_path: required only for when generating target_path from artifact hash
@@ -396,13 +396,13 @@ def get_df_stats(df):
 def update_dataset_meta(
     artifact,
     from_df=None,
-    schema: dict = None,
-    header: list = None,
-    preview: list = None,
-    stats: dict = None,
-    extra_data: dict = None,
-    column_metadata: dict = None,
-    labels: dict = None,
+    schema: Optional[dict] = None,
+    header: Optional[list] = None,
+    preview: Optional[list] = None,
+    stats: Optional[dict] = None,
+    extra_data: Optional[dict] = None,
+    column_metadata: Optional[dict] = None,
+    labels: Optional[dict] = None,
     ignore_preview_limits: bool = False,
 ):
     """Update dataset object attributes/metadata
@@ -495,4 +495,4 @@ def upload_dataframe(
         size = target_class(path=target_path).write_dataframe(df, **kw)
         return size, None
 
-    raise mlrun.errors.MLRunInvalidArgumentError(f"format {format} not implemented yet")
+    raise mlrun.errors.MLRunInvalidArgumentError(f"Format {format} not implemented yet")