mlrun 1.7.2rc4__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 __all__ = [
     "get_version",
     "set_environment",
@@ -29,9 +27,9 @@ __all__ = [
 ]
 
 from os import environ, path
+from typing import Optional
 
 import dotenv
-import mlrun_pipelines
 
 from .config import config as mlconf
 from .datastore import DataItem, store_manager
@@ -41,6 +39,7 @@ from .execution import MLClientCtx
 from .model import RunObject, RunTemplate, new_task
 from .package import ArtifactType, DefaultPackager, Packager, handler
 from .projects import (
+    MlrunProject,
     ProjectMetadata,
     build_function,
     deploy_function,
@@ -61,18 +60,19 @@ from .run import (
     get_pipeline,
     import_function,
     new_function,
+    retry_pipeline,
     wait_for_pipeline_completion,
 )
-from .runtimes import new_model_server
+from .runtimes import mounts, new_model_server
 from .secrets import get_secret_or_env
 from .utils.version import Version
 
 __version__ = Version().get()["version"]
 
-VolumeMount = mlrun_pipelines.common.mounts.VolumeMount
-mount_v3io = mlrun_pipelines.mounts.mount_v3io
-v3io_cred = mlrun_pipelines.mounts.v3io_cred
-auto_mount = mlrun_pipelines.mounts.auto_mount
+VolumeMount = mounts.VolumeMount
+mount_v3io = mounts.mount_v3io
+v3io_cred = mounts.v3io_cred
+auto_mount = mounts.auto_mount
 
 
 def get_version():
@@ -88,12 +88,12 @@ if "IGZ_NAMESPACE_DOMAIN" in environ:
 
 
 def set_environment(
-    api_path: str = None,
+    api_path: Optional[str] = None,
     artifact_path: str = "",
-    access_key: str = None,
-    username: str = None,
-    env_file: str = None,
-    mock_functions: str = None,
+    access_key: Optional[str] = None,
+    username: Optional[str] = None,
+    env_file: Optional[str] = None,
+    mock_functions: Optional[str] = None,
 ):
     """set and test default config for: api path, artifact_path and project
 
@@ -136,15 +136,16 @@ def set_environment(
     if not mlconf.dbpath:
         raise ValueError("DB/API path was not detected, please specify its address")
 
-    if mock_functions is not None:
-        mock_functions = "1" if mock_functions is True else mock_functions
-        mlconf.force_run_local = mock_functions
-        mlconf.mock_nuclio_deployment = mock_functions
-
     # check connectivity and load remote defaults
     get_run_db()
     if api_path:
         environ["MLRUN_DBPATH"] = mlconf.dbpath
+    mlconf.reload()
+
+    if mock_functions is not None:
+        mock_functions = "1" if mock_functions is True else mock_functions
+        mlconf.force_run_local = mock_functions
+        mlconf.mock_nuclio_deployment = mock_functions
 
     if not mlconf.artifact_path and not artifact_path:
         raise ValueError(
@@ -159,13 +160,14 @@ def set_environment(
                 "artifact_path must refer to an absolute path" " or a valid url"
             )
         mlconf.artifact_path = artifact_path
+
     return mlconf.default_project, mlconf.artifact_path
 
 
-def get_current_project(silent=False):
+def get_current_project(silent: bool = False) -> Optional[MlrunProject]:
     if not pipeline_context.project and not silent:
         raise MLRunInvalidArgumentError(
-            "current project is not initialized, use new, get or load project methods first"
+            "No current project is initialized. Use new, get or load project functions first."
         )
     return pipeline_context.project
 
@@ -182,7 +184,7 @@ def get_sample_path(subpath=""):
     return samples_path
 
 
-def set_env_from_file(env_file: str, return_dict: bool = False):
+def set_env_from_file(env_file: str, return_dict: bool = False) -> Optional[dict]:
     """Read and set and/or return environment variables from a file
     the env file should have lines in the form KEY=VALUE, comment line start with "#"
 
@@ -211,7 +213,9 @@ def set_env_from_file(env_file: str, return_dict: bool = False):
     env_vars = dotenv.dotenv_values(env_file)
     if None in env_vars.values():
         raise MLRunInvalidArgumentError("env file lines must be in the form key=value")
+
     for key, value in env_vars.items():
-        environ[key] = value  # Load to local environ
+        environ[key] = value
+
     mlconf.reload()  # reload mlrun configuration
     return env_vars if return_dict else None

mlrun/__main__.py

@@ -17,9 +17,8 @@ import json
 import pathlib
 import socket
 import traceback
-import warnings
 from ast import literal_eval
-from base64 import b64decode, b64encode
+from base64 import b64decode
 from os import environ, path, remove
 from pprint import pprint
 
@@ -27,13 +26,14 @@ import click
 import dotenv
 import pandas as pd
 import yaml
-from mlrun_pipelines.mounts import auto_mount as auto_mount_modifier
 from tabulate import tabulate
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
+import mlrun.utils.helpers
 from mlrun.common.helpers import parse_versioned_object_uri
+from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
 
 from .config import config as mlconf
 from .db import get_run_db
@@ -297,13 +297,14 @@ def run(
             if url_file and path.isfile(url_file):
                 with open(url_file) as fp:
                     body = fp.read()
-                based = b64encode(body.encode("utf-8")).decode("utf-8")
+                based = mlrun.utils.helpers.encode_user_code(body)
                 logger.info(f"packing code at {url_file}")
                 update_in(runtime, "spec.build.functionSourceCode", based)
                 url = f"main{pathlib.Path(url_file).suffix} {url_args}"
                 update_in(runtime, "spec.build.code_origin", url_file)
     elif runtime:
         runtime = py_eval(runtime)
+        runtime = mlrun.utils.helpers.as_dict(runtime)
         if not isinstance(runtime, dict):
             print(f"Runtime parameter must be a dict, not {type(runtime)}")
             exit(1)
@@ -515,6 +516,7 @@ def build(
 
     if runtime:
         runtime = py_eval(runtime)
+        runtime = mlrun.utils.helpers.as_dict(runtime)
         if not isinstance(runtime, dict):
             print(f"Runtime parameter must be a dict, not {type(runtime)}")
             exit(1)
@@ -554,7 +556,7 @@ def build(
             exit(1)
         with open(source) as fp:
             body = fp.read()
-        based = b64encode(body.encode("utf-8")).decode("utf-8")
+        based = mlrun.utils.helpers.encode_user_code(body)
         logger.info(f"Packing code at {source}")
         b.functionSourceCode = based
         func.spec.command = ""
@@ -662,6 +664,8 @@ def deploy(
         runtime = py_eval(spec)
     else:
         runtime = {}
+
+    runtime = mlrun.utils.helpers.as_dict(runtime)
     if not isinstance(runtime, dict):
         print(f"Runtime parameter must be a dict, not {type(runtime)}")
         exit(1)
@@ -767,10 +771,11 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
 
         runs = run_db.list_runs(uid=uid, project=project, labels=selector)
         df = runs.to_df()[
-            ["name", "uid", "iter", "start", "state", "parameters", "results"]
+            ["name", "uid", "iter", "start", "end", "state", "parameters", "results"]
         ]
         # df['uid'] = df['uid'].apply(lambda x: f'..{x[-6:]}')
-        df["start"] = df["start"].apply(time_str)
+        for time_column in ["start", "end"]:
+            df[time_column] = df[time_column].apply(time_str)
         df["parameters"] = df["parameters"].apply(dict_to_str)
         df["results"] = df["results"].apply(dict_to_str)
         print(tabulate(df, headers="keys"))
@@ -858,14 +863,8 @@ def version():
 )
 @click.option("--offset", type=int, default=0, help="byte offset")
 @click.option("--db", help="api and db service path/url")
-@click.option("--watch", "-w", is_flag=True, help="Deprecated. not in use")
-def logs(uid, project, offset, db, watch):
+def logs(uid, project, offset, db):
     """Get or watch task logs"""
-    if watch:
-        warnings.warn(
-            "'--watch' is deprecated in 1.6.0, and will be removed in 1.8.0, "
-            # TODO: Remove in 1.8.0
-        )
     mldb = get_run_db(db or mlconf.dbpath)
     if mldb.kind == "http":
         state, _ = mldb.watch_log(uid, project, watch=False, offset=offset)
@@ -1248,10 +1247,10 @@ def show_or_set_config(
         }
         for key, value in env_dict.items():
             if value:
-                dotenv.set_key(filename, key, value, quote_mode="")
+                dotenv.set_key(filename, key, value, quote_mode="always")
         if env_vars:
             for key, value in list2dict(env_vars).items():
-                dotenv.set_key(filename, key, value, quote_mode="")
+                dotenv.set_key(filename, key, value, quote_mode="always")
         if env_file:
             # if its not the default file print the usage details
             print(

mlrun/alerts/alert.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from typing import Union
+from datetime import datetime
+from typing import Optional, Union
 
 import mlrun
 import mlrun.common.schemas.alert as alert_objects
@@ -30,6 +30,7 @@ class AlertConfig(ModelObj):
         "state",
         "count",
         "created",
+        "updated",
     ]
     _fields_to_serialize = ModelObj._fields_to_serialize + [
         "entities",
@@ -40,21 +41,23 @@ class AlertConfig(ModelObj):
 
     def __init__(
         self,
-        project: str = None,
-        name: str = None,
+        project: Optional[str] = None,
+        name: Optional[str] = None,
         template: Union[alert_objects.AlertTemplate, str] = None,
-        description: str = None,
-        summary: str = None,
+        description: Optional[str] = None,
+        summary: Optional[str] = None,
         severity: alert_objects.AlertSeverity = None,
         trigger: alert_objects.AlertTrigger = None,
         criteria: alert_objects.AlertCriteria = None,
         reset_policy: alert_objects.ResetPolicy = None,
-        notifications: list[alert_objects.AlertNotification] = None,
+        notifications: Optional[list[alert_objects.AlertNotification]] = None,
         entities: alert_objects.EventEntities = None,
-        id: int = None,
+        id: Optional[int] = None,
         state: alert_objects.AlertActiveState = None,
-        created: str = None,
-        count: int = None,
+        created: Optional[str] = None,
+        count: Optional[int] = None,
+        updated: Optional[str] = None,
+        **kwargs,
     ):
         """Alert config object
 
@@ -109,7 +112,7 @@ class AlertConfig(ModelObj):
                                complex trigger which is based on a prometheus alert
         :param criteria:       When the alert will be triggered based on the specified number of events within the
                                defined time period.
-        :param reset_policy:   When to clear the alert. May be "manual" for manual reset of the alert, or
+        :param reset_policy:   When to clear the alert. Either "manual" for manual reset of the alert, or
                                "auto" if the criteria contains a time period
         :param notifications:  List of notifications to invoke once the alert is triggered
         :param entities:       Entities that the event relates to. The entity object will contain fields that
@@ -118,6 +121,7 @@ class AlertConfig(ModelObj):
         :param state:          State of the alert, may be active/inactive (user should not supply it)
         :param created:        When the alert is created (user should not supply it)
         :param count:          Internal counter of the alert (user should not supply it)
+        :param updated:        The last update time of the alert (user should not supply it)
         """
         self.project = project
         self.name = name
@@ -131,18 +135,45 @@ class AlertConfig(ModelObj):
         self.entities = entities
         self.id = id
         self.state = state
-        self.created = created
+        self._created = created
         self.count = count
+        self._updated = updated
 
         if template:
             self._apply_template(template)
 
+    @property
+    def created(self) -> datetime:
+        """
+        Get the `created` field as a datetime object.
+        """
+        if isinstance(self._created, str):
+            return datetime.fromisoformat(self._created)
+        return self._created
+
+    @created.setter
+    def created(self, created):
+        self._created = created
+
+    @property
+    def updated(self) -> datetime:
+        """
+        Get the `updated` field as a datetime object.
+        """
+        if isinstance(self._updated, str):
+            return datetime.fromisoformat(self._updated)
+        return self._updated
+
+    @updated.setter
+    def updated(self, updated):
+        self._updated = updated
+
     def validate_required_fields(self):
         if not self.name:
             raise mlrun.errors.MLRunInvalidArgumentError("Alert name must be provided")
 
     def _serialize_field(
-        self, struct: dict, field_name: str = None, strip: bool = False
+        self, struct: dict, field_name: Optional[str] = None, strip: bool = False
     ):
         if field_name == "entities":
             if self.entities:
@@ -179,7 +210,12 @@ class AlertConfig(ModelObj):
             return None
         return super()._serialize_field(struct, field_name, strip)
 
-    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+    def to_dict(
+        self,
+        fields: Optional[list] = None,
+        exclude: Optional[list] = None,
+        strip: bool = False,
+    ):
         if self.entities is None:
             raise mlrun.errors.MLRunBadRequestError("Alert entity field is missing")
         if not self.notifications:
@@ -189,7 +225,9 @@ class AlertConfig(ModelObj):
         return super().to_dict(self._dict_fields)
 
     @classmethod
-    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+    def from_dict(
+        cls, struct=None, fields=None, deprecated_fields: Optional[dict] = None
+    ):
         new_obj = super().from_dict(struct, fields=fields)
 
         entity_data = struct.get("entities")
@@ -246,3 +284,100 @@ class AlertConfig(ModelObj):
         self.criteria = self.criteria or template.criteria
         self.trigger = self.trigger or template.trigger
         self.reset_policy = self.reset_policy or template.reset_policy
+
+    def list_activations(
+        self,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        from_last_update: bool = False,
+    ) -> list[mlrun.common.schemas.alert.AlertActivation]:
+        """
+        Retrieve a list of all alert activations.
+
+        :param since: Filters for alert activations occurring after this timestamp.
+        :param until: Filters for alert activations occurring before this timestamp.
+        :param from_last_update: If set to True, retrieves alert activations since the alert's last update time.
+                                 if both since and from_last_update=True are provided, from_last_update takes precedence
+                                 and the since value will be overridden by the alert's last update timestamp.
+
+        :returns: A list of alert activations matching the provided filters.
+        """
+        db = mlrun.get_run_db()
+        if from_last_update and self._updated:
+            since = self.updated
+
+        return db.list_alert_activations(
+            project=self.project,
+            name=self.name,
+            since=since,
+            until=until,
+        )
+
+    def paginated_list_activations(
+        self,
+        *args,
+        page: Optional[int] = None,
+        page_size: Optional[int] = None,
+        page_token: Optional[str] = None,
+        from_last_update: bool = False,
+        **kwargs,
+    ) -> tuple[mlrun.common.schemas.alert.AlertActivation, Optional[str]]:
+        """
+        List alerts activations with support for pagination and various filtering options.
+
+        This method retrieves a paginated list of alert activations based on the specified filter parameters.
+        Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
+        will return a list of alert activations that match the filtering criteria provided.
+
+        For detailed information about the parameters, refer to the list_activations method:
+            See :py:func:`~list_activations` for more details.
+
+        Examples::
+
+            # Fetch first page of alert activations with page size of 5
+            alert_activations, token = alert_config.paginated_list_activations(page_size=5)
+            # Fetch next page using the pagination token from the previous response
+            alert_activations, token = alert_config.paginated_list_activations(
+                page_token=token
+            )
+            # Fetch alert activations for a specific page (e.g., page 3)
+            alert_activations, token = alert_config.paginated_list_activations(
+                page=3, page_size=5
+            )
+
+            # Automatically iterate over all pages without explicitly specifying the page number
+            alert_activations = []
+            token = None
+            while True:
+                page_alert_activations, token = alert_config.paginated_list_activations(
+                    page_token=token, page_size=5
+                )
+                alert_activations.extend(page_alert_activations)
+
+                # If token is None and page_alert_activations is empty, we've reached the end (no more activations).
+                # If token is None and page_alert_activations is not empty, we've fetched the last page of activations.
+                if not token:
+                    break
+            print(f"Total alert activations retrieved: {len(alert_activations)}")
+
+        :param page: The page number to retrieve. If not provided, the next page will be retrieved.
+        :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
+        :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
+            for the first request.
+        :param from_last_update: If set to True, retrieves alert activations since the alert's last update time.
+
+        :returns: A tuple containing the list of alert activations and an optional `page_token` for pagination.
+        """
+        if from_last_update and self._updated:
+            kwargs["since"] = self.updated
+
+        db = mlrun.get_run_db()
+        return db.paginated_list_alert_activations(
+            *args,
+            project=self.project,
+            name=self.name,
+            page=page,
+            page_size=page_size,
+            page_token=page_token,
+            **kwargs,
+        )

mlrun/api/schemas/__init__.py

@@ -193,9 +193,7 @@ FeatureValues = DeprecationHelper(mlrun.common.schemas.FeatureValues)
 GrafanaColumn = DeprecationHelper(
     mlrun.common.schemas.model_monitoring.grafana.GrafanaColumn
 )
-GrafanaDataPoint = DeprecationHelper(
-    mlrun.common.schemas.model_monitoring.grafana.GrafanaDataPoint
-)
+
 GrafanaNumberColumn = DeprecationHelper(
     mlrun.common.schemas.model_monitoring.grafana.GrafanaNumberColumn
 )
@@ -205,17 +203,11 @@ GrafanaStringColumn = DeprecationHelper(
 GrafanaTable = DeprecationHelper(
     mlrun.common.schemas.model_monitoring.grafana.GrafanaTable
 )
-GrafanaTimeSeriesTarget = DeprecationHelper(
-    mlrun.common.schemas.model_monitoring.grafana.GrafanaTimeSeriesTarget
-)
 ModelEndpoint = DeprecationHelper(mlrun.common.schemas.ModelEndpoint)
 ModelEndpointList = DeprecationHelper(mlrun.common.schemas.ModelEndpointList)
 ModelEndpointMetadata = DeprecationHelper(mlrun.common.schemas.ModelEndpointMetadata)
 ModelEndpointSpec = DeprecationHelper(mlrun.common.schemas.ModelEndpointSpec)
 ModelEndpointStatus = DeprecationHelper(mlrun.common.schemas.ModelEndpointStatus)
-ModelMonitoringStoreKinds = DeprecationHelper(
-    mlrun.common.schemas.ModelMonitoringStoreKinds
-)
 NotificationSeverity = DeprecationHelper(mlrun.common.schemas.NotificationSeverity)
 NotificationStatus = DeprecationHelper(mlrun.common.schemas.NotificationStatus)
 ObjectKind = DeprecationHelper(mlrun.common.schemas.ObjectKind)

mlrun/artifacts/__init__.py

@@ -12,10 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 # Don't remove this, used by sphinx documentation
-__all__ = ["get_model", "update_model"]
+__all__ = ["get_model", "update_model", "DocumentLoaderSpec", "MLRunLoader"]
 
 from .base import (
     Artifact,
@@ -25,6 +23,7 @@ from .base import (
     get_artifact_meta,
 )
 from .dataset import DatasetArtifact, TableArtifact, update_dataset_meta
+from .document import DocumentArtifact, DocumentLoaderSpec, MLRunLoader
 from .manager import (
     ArtifactManager,
     ArtifactProducer,