mlrun 1.7.0rc4__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/__init__.py

@@ -22,11 +22,16 @@ __all__ = [
     "handler",
     "ArtifactType",
     "get_secret_or_env",
+    "mount_v3io",
+    "v3io_cred",
+    "auto_mount",
+    "VolumeMount",
 ]
 
 from os import environ, path
 
 import dotenv
+import mlrun_pipelines
 
 from .config import config as mlconf
 from .datastore import DataItem, store_manager
@@ -35,7 +40,6 @@ from .errors import MLRunInvalidArgumentError, MLRunNotFoundError
 from .execution import MLClientCtx
 from .model import RunObject, RunTemplate, new_task
 from .package import ArtifactType, DefaultPackager, Packager, handler
-from .platforms import VolumeMount, auto_mount, mount_v3io, v3io_cred
 from .projects import (
     ProjectMetadata,
     build_function,
@@ -65,6 +69,11 @@ from .utils.version import Version
 
 __version__ = Version().get()["version"]
 
+VolumeMount = mlrun_pipelines.common.mounts.VolumeMount
+mount_v3io = mlrun_pipelines.mounts.mount_v3io
+v3io_cred = mlrun_pipelines.mounts.v3io_cred
+auto_mount = mlrun_pipelines.mounts.auto_mount
+
 
 def get_version():
     """get current mlrun version"""
@@ -97,6 +106,7 @@ def set_environment(
     example::
 
         from os import path
+
         project_name, artifact_path = set_environment()
         set_environment("http://localhost:8080", artifact_path="./")
         set_environment(env_file="mlrun.env")

mlrun/__main__.py

@@ -22,17 +22,16 @@ from ast import literal_eval
 from base64 import b64decode, b64encode
 from os import environ, path, remove
 from pprint import pprint
-from subprocess import Popen
-from sys import executable
-from urllib.parse import urlparse
 
 import click
 import dotenv
 import pandas as pd
 import yaml
+from mlrun_pipelines.mounts import auto_mount as auto_mount_modifier
 from tabulate import tabulate
 
 import mlrun
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 from mlrun.common.helpers import parse_versioned_object_uri
 
@@ -40,7 +39,6 @@ from .config import config as mlconf
 from .db import get_run_db
 from .errors import err_to_str
 from .model import RunTemplate
-from .platforms import auto_mount as auto_mount_modifier
 from .projects import load_project
 from .run import (
     get_object,
@@ -52,12 +50,12 @@ from .run import (
 from .runtimes import RemoteRuntime, RunError, RuntimeKinds, ServingRuntime
 from .secrets import SecretsStore
 from .utils import (
+    RunKeys,
     dict_to_yaml,
     get_in,
     is_relative_path,
     list2dict,
     logger,
-    run_keys,
     update_in,
 )
 from .utils.version import Version
@@ -104,7 +102,9 @@ def main():
 )
 @click.option("--uid", help="unique run ID")
 @click.option("--name", help="run name")
-@click.option("--workflow", help="workflow name/id")
+@click.option(
+    "--workflow", help="sets the run labels to match the given workflow name/id"
+)
 @click.option("--project", help="project name/id")
 @click.option("--db", default="", help="save run results to path or DB url")
 @click.option(
@@ -259,8 +259,10 @@ def run(
             runobj.metadata.labels[k] = v
 
     if workflow:
-        runobj.metadata.labels["workflow"] = workflow
-        runobj.metadata.labels["mlrun/runner-pod"] = socket.gethostname()
+        runobj.metadata.labels[mlrun_constants.MLRunInternalLabels.workflow] = workflow
+        runobj.metadata.labels[mlrun_constants.MLRunInternalLabels.runner_pod] = (
+            socket.gethostname()
+        )
 
     if db:
         mlconf.dbpath = db
@@ -378,15 +380,15 @@ def run(
     set_item(runobj.spec.hyper_param_options, hyper_param_strategy, "strategy")
     set_item(runobj.spec.hyper_param_options, selector, "selector")
 
-    set_item(runobj.spec, inputs, run_keys.inputs, list2dict(inputs))
+    set_item(runobj.spec, inputs, RunKeys.inputs, list2dict(inputs))
     set_item(
-        runobj.spec, returns, run_keys.returns, [py_eval(value) for value in returns]
+        runobj.spec, returns, RunKeys.returns, [py_eval(value) for value in returns]
     )
-    set_item(runobj.spec, in_path, run_keys.input_path)
-    set_item(runobj.spec, out_path, run_keys.output_path)
-    set_item(runobj.spec, outputs, run_keys.outputs, list(outputs))
+    set_item(runobj.spec, in_path, RunKeys.input_path)
+    set_item(runobj.spec, out_path, RunKeys.output_path)
+    set_item(runobj.spec, outputs, RunKeys.outputs, list(outputs))
     set_item(
-        runobj.spec, secrets, run_keys.secrets, line2keylist(secrets, "kind", "source")
+        runobj.spec, secrets, RunKeys.secrets, line2keylist(secrets, "kind", "source")
     )
     set_item(runobj.spec, verbose, "verbose")
     set_item(runobj.spec, scrape_metrics, "scrape_metrics")
@@ -469,6 +471,17 @@ def run(
     is_flag=True,
     help="ensure the project exists, if not, create project",
 )
+@click.option(
+    "--state-file-path", default="/tmp/state", help="path to file with state data"
+)
+@click.option(
+    "--image-file-path", default="/tmp/image", help="path to file with image data"
+)
+@click.option(
+    "--full-image-file-path",
+    default="/tmp/fullimage",
+    help="path to file with full image data",
+)
 def build(
     func_url,
     name,
@@ -488,6 +501,9 @@ def build(
     skip,
     env_file,
     ensure_project,
+    state_file_path,
+    image_file_path,
+    full_image_file_path,
 ):
     """Build a container image from code and requirements."""
 
@@ -505,6 +521,8 @@ def build(
         if kfp:
             print("Runtime:")
             pprint(runtime)
+        # use kind = "job" by default if not specified
+        runtime.setdefault("kind", "job")
         func = new_function(runtime=runtime)
 
     elif func_url:
@@ -575,12 +593,12 @@ def build(
         state = func.status.state
         image = func.spec.image
         if kfp:
-            with open("/tmp/state", "w") as fp:
+            with open(state_file_path, "w") as fp:
                 fp.write(state or "none")
             full_image = func.full_image_path(image) or ""
-            with open("/tmp/image", "w") as fp:
+            with open(image_file_path, "w") as fp:
                 fp.write(image)
-            with open("/tmp/fullimage", "w") as fp:
+            with open(full_image_file_path, "w") as fp:
                 fp.write(full_image)
             print("Full image path = ", full_image)
 
@@ -716,9 +734,11 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
     if db:
         mlconf.dbpath = db
     if not project:
-        print("Warning, project parameter was not specified using default !")
+        logger.warning(
+            "Project parameter was not specified. Defaulting to 'default' project"
+        )
     if kind.startswith("po"):
-        print("Unsupported, use 'get runtimes' instead")
+        logger.warning("Unsupported, use 'get runtimes' instead")
         return
 
     elif kind.startswith("runtime"):
@@ -825,108 +845,6 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
         )
 
 
-@main.command(deprecated=True)
-@click.option("--port", "-p", help="port to listen on", type=int)
-@click.option("--dirpath", "-d", help="database directory (dirpath)")
-@click.option("--dsn", "-s", help="database dsn, e.g. sqlite:///db/mlrun.db")
-@click.option("--logs-path", "-l", help="logs directory path")
-@click.option("--data-volume", "-v", help="path prefix to the location of artifacts")
-@click.option("--verbose", is_flag=True, help="verbose log")
-@click.option("--background", "-b", is_flag=True, help="run in background process")
-@click.option("--artifact-path", "-a", help="default artifact path")
-@click.option(
-    "--update-env",
-    default="",
-    is_flag=False,
-    flag_value=mlrun.config.default_env_file,
-    help=f"update the specified mlrun .env file (if TEXT not provided defaults to {mlrun.config.default_env_file})",
-)
-def db(
-    port,
-    dirpath,
-    dsn,
-    logs_path,
-    data_volume,
-    verbose,
-    background,
-    artifact_path,
-    update_env,
-):
-    """Run HTTP api/database server"""
-    warnings.warn(
-        "The `mlrun db` command is deprecated in 1.5.0 and will be removed in 1.7.0, it is for internal use only.",
-        FutureWarning,
-    )
-    env = environ.copy()
-    # ignore client side .env file (so import mlrun in server will not try to connect to local/remote DB)
-    env["MLRUN_IGNORE_ENV_FILE"] = "true"
-    env["MLRUN_DBPATH"] = ""
-
-    if port is not None:
-        env["MLRUN_HTTPDB__PORT"] = str(port)
-    if dirpath is not None:
-        env["MLRUN_HTTPDB__DIRPATH"] = dirpath
-    if dsn is not None:
-        if dsn.startswith("sqlite://") and "check_same_thread=" not in dsn:
-            dsn += "?check_same_thread=false"
-        env["MLRUN_HTTPDB__DSN"] = dsn
-    if logs_path is not None:
-        env["MLRUN_HTTPDB__LOGS_PATH"] = logs_path
-    if data_volume is not None:
-        env["MLRUN_HTTPDB__DATA_VOLUME"] = data_volume
-    if verbose:
-        env["MLRUN_LOG_LEVEL"] = "DEBUG"
-    if artifact_path or "MLRUN_ARTIFACT_PATH" not in env:
-        if not artifact_path:
-            artifact_path = (
-                env.get("MLRUN_HTTPDB__DATA_VOLUME", "./artifacts").rstrip("/")
-                + "/{{project}}"
-            )
-        env["MLRUN_ARTIFACT_PATH"] = path.realpath(path.expanduser(artifact_path))
-
-    env["MLRUN_IS_API_SERVER"] = "true"
-
-    # create the DB dir if needed
-    dsn = dsn or mlconf.httpdb.dsn
-    if dsn and dsn.startswith("sqlite:///"):
-        parsed = urlparse(dsn)
-        p = pathlib.Path(parsed.path[1:]).parent
-        p.mkdir(parents=True, exist_ok=True)
-
-    cmd = [executable, "-m", "server.api.main"]
-    pid = None
-    if background:
-        print("Starting MLRun API service in the background...")
-        child = Popen(
-            cmd,
-            env=env,
-            stdout=open("mlrun-stdout.log", "w"),
-            stderr=open("mlrun-stderr.log", "w"),
-            start_new_session=True,
-        )
-        pid = child.pid
-        print(
-            f"Background pid: {pid}, logs written to mlrun-stdout.log and mlrun-stderr.log, use:\n"
-            f"`kill {pid}` (linux/mac) or `taskkill /pid {pid} /t /f` (windows), to kill the mlrun service process"
-        )
-    else:
-        child = Popen(cmd, env=env)
-        returncode = child.wait()
-        if returncode != 0:
-            raise SystemExit(returncode)
-    if update_env:
-        # update mlrun client env file with the API path, so client will use the new DB
-        # update and PID, allow killing the correct process in a config script
-        filename = path.expanduser(update_env)
-        dotenv.set_key(
-            filename, "MLRUN_DBPATH", f"http://localhost:{port or 8080}", quote_mode=""
-        )
-        dotenv.set_key(filename, "MLRUN_MOCK_NUCLIO_DEPLOYMENT", "auto", quote_mode="")
-        if pid:
-            dotenv.set_key(filename, "MLRUN_SERVICE_PID", str(pid), quote_mode="")
-        print(f"Updated configuration in {update_env} .env file")
-
-
 @main.command()
 def version():
     """get mlrun version"""

mlrun/{datastore/helpers.py → alerts/__init__.py}

@@ -1,4 +1,4 @@
-# Copyright 2023 Iguazio
+# Copyright 2024 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,8 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-
 
-ONE_GB = 1024 * 1024 * 1024
-ONE_MB = 1024 * 1024
+from .alert import AlertConfig

mlrun/alerts/alert.py

@@ -0,0 +1,248 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Union
+
+import mlrun
+import mlrun.common.schemas.alert as alert_objects
+from mlrun.model import ModelObj
+
+
+class AlertConfig(ModelObj):
+    _dict_fields = [
+        "project",
+        "name",
+        "description",
+        "summary",
+        "severity",
+        "reset_policy",
+        "state",
+        "count",
+        "created",
+    ]
+    _fields_to_serialize = ModelObj._fields_to_serialize + [
+        "entities",
+        "notifications",
+        "trigger",
+        "criteria",
+    ]
+
+    def __init__(
+        self,
+        project: str = None,
+        name: str = None,
+        template: Union[alert_objects.AlertTemplate, str] = None,
+        description: str = None,
+        summary: str = None,
+        severity: alert_objects.AlertSeverity = None,
+        trigger: alert_objects.AlertTrigger = None,
+        criteria: alert_objects.AlertCriteria = None,
+        reset_policy: alert_objects.ResetPolicy = None,
+        notifications: list[alert_objects.AlertNotification] = None,
+        entities: alert_objects.EventEntities = None,
+        id: int = None,
+        state: alert_objects.AlertActiveState = None,
+        created: str = None,
+        count: int = None,
+    ):
+        """Alert config object
+
+        Example::
+
+            # create an alert on endpoint_id, which will be triggered to slack if there is a "data_drift_detected" event
+            # 3 times in the next hour.
+
+            from mlrun.alerts import AlertConfig
+            import mlrun.common.schemas.alert as alert_objects
+
+            entity_kind = alert_objects.EventEntityKind.MODEL_ENDPOINT_RESULT
+            entity_id = get_default_result_instance_fqn(endpoint_id)
+            event_name = alert_objects.EventKind.DATA_DRIFT_DETECTED
+            notification = mlrun.model.Notification(
+                kind="slack",
+                name="slack_notification",
+                message="drift was detected",
+                severity="warning",
+                when=["now"],
+                condition="failed",
+                secret_params={
+                    "webhook": "https://hooks.slack.com/",
+                },
+            ).to_dict()
+
+            alert_data = AlertConfig(
+                project="my-project",
+                name="drift-alert",
+                summary="a drift was detected",
+                severity=alert_objects.AlertSeverity.LOW,
+                entities=alert_objects.EventEntities(
+                    kind=entity_kind, project="my-project", ids=[entity_id]
+                ),
+                trigger=alert_objects.AlertTrigger(events=[event_name]),
+                criteria=alert_objects.AlertCriteria(count=3, period="1h"),
+                notifications=[alert_objects.AlertNotification(notification=notification)],
+            )
+            project.store_alert_config(alert_data)
+
+        :param project:        Name of the project to associate the alert with
+        :param name:           Name of the alert
+        :param template:       Optional parameter that allows creating an alert based on a predefined template.
+                               You can pass either an AlertTemplate object or a string (the template name).
+                               If a template is used, many fields of the alert will be auto-generated based on the
+                               template.However, you still need to provide the following fields:
+                               `name`, `project`, `entity`, `notifications`
+        :param description:    Description of the alert
+        :param summary:        Summary of the alert, will be sent in the generated notifications
+        :param severity:       Severity of the alert
+        :param trigger:        The events that will trigger this alert, may be a simple trigger based on events or
+                               complex trigger which is based on a prometheus alert
+        :param criteria:       When the alert will be triggered based on the specified number of events within the
+                               defined time period.
+        :param reset_policy:   When to clear the alert. May be "manual" for manual reset of the alert, or
+                               "auto" if the criteria contains a time period
+        :param notifications:  List of notifications to invoke once the alert is triggered
+        :param entities:       Entities that the event relates to. The entity object will contain fields that
+                               uniquely identify a given entity in the system
+        :param id:             Internal id of the alert (user should not supply it)
+        :param state:          State of the alert, may be active/inactive (user should not supply it)
+        :param created:        When the alert is created (user should not supply it)
+        :param count:          Internal counter of the alert (user should not supply it)
+        """
+        self.project = project
+        self.name = name
+        self.description = description
+        self.summary = summary
+        self.severity = severity
+        self.trigger = trigger
+        self.criteria = criteria
+        self.reset_policy = reset_policy
+        self.notifications = notifications or []
+        self.entities = entities
+        self.id = id
+        self.state = state
+        self.created = created
+        self.count = count
+
+        if template:
+            self._apply_template(template)
+
+    def validate_required_fields(self):
+        if not self.name:
+            raise mlrun.errors.MLRunInvalidArgumentError("Alert name must be provided")
+
+    def _serialize_field(
+        self, struct: dict, field_name: str = None, strip: bool = False
+    ):
+        if field_name == "entities":
+            if self.entities:
+                return (
+                    self.entities.dict()
+                    if not isinstance(self.entities, dict)
+                    else self.entities
+                )
+            return None
+        if field_name == "notifications":
+            if self.notifications:
+                return [
+                    notification_data.dict()
+                    if not isinstance(notification_data, dict)
+                    else notification_data
+                    for notification_data in self.notifications
+                ]
+            return None
+        if field_name == "trigger":
+            if self.trigger:
+                return (
+                    self.trigger.dict()
+                    if not isinstance(self.trigger, dict)
+                    else self.trigger
+                )
+            return None
+        if field_name == "criteria":
+            if self.criteria:
+                return (
+                    self.criteria.dict()
+                    if not isinstance(self.criteria, dict)
+                    else self.criteria
+                )
+            return None
+        return super()._serialize_field(struct, field_name, strip)
+
+    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+        if self.entities is None:
+            raise mlrun.errors.MLRunBadRequestError("Alert entity field is missing")
+        if not self.notifications:
+            raise mlrun.errors.MLRunBadRequestError(
+                "Alert must have at least one notification"
+            )
+        return super().to_dict(self._dict_fields)
+
+    @classmethod
+    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+        new_obj = super().from_dict(struct, fields=fields)
+
+        entity_data = struct.get("entities")
+        if entity_data:
+            entity_obj = alert_objects.EventEntities.parse_obj(entity_data)
+            new_obj.entities = entity_obj
+
+        notifications_data = struct.get("notifications")
+        if notifications_data:
+            notifications_objs = [
+                alert_objects.AlertNotification.parse_obj(notification)
+                for notification in notifications_data
+            ]
+            new_obj.notifications = notifications_objs
+
+        trigger_data = struct.get("trigger")
+        if trigger_data:
+            trigger_obj = alert_objects.AlertTrigger.parse_obj(trigger_data)
+            new_obj.trigger = trigger_obj
+
+        criteria_data = struct.get("criteria")
+        if criteria_data:
+            criteria_obj = alert_objects.AlertCriteria.parse_obj(criteria_data)
+            new_obj.criteria = criteria_obj
+        return new_obj
+
+    def with_notifications(self, notifications: list[alert_objects.AlertNotification]):
+        if not isinstance(notifications, list) or not all(
+            isinstance(item, alert_objects.AlertNotification) for item in notifications
+        ):
+            raise ValueError(
+                "Notifications parameter must be a list of AlertNotification"
+            )
+        for notification_data in notifications:
+            self.notifications.append(notification_data)
+        return self
+
+    def with_entities(self, entities: alert_objects.EventEntities):
+        if not isinstance(entities, alert_objects.EventEntities):
+            raise ValueError("Entities parameter must be of type: EventEntities")
+        self.entities = entities
+        return self
+
+    def _apply_template(self, template):
+        if isinstance(template, str):
+            db = mlrun.get_run_db()
+            template = db.get_alert_template(template)
+
+        # Apply parameters from the template to the AlertConfig object only if they are not already specified by the
+        # user in the current configuration.
+        # User-provided parameters will take precedence over corresponding template values
+        self.summary = self.summary or template.summary
+        self.severity = self.severity or template.severity
+        self.criteria = self.criteria or template.criteria
+        self.trigger = self.trigger or template.trigger
+        self.reset_policy = self.reset_policy or template.reset_policy

mlrun/api/schemas/__init__.py

@@ -25,6 +25,7 @@ schema.
 import sys
 import warnings
 
+import mlrun.common.formatters
 import mlrun.common.schemas
 import mlrun.common.schemas.artifact as old_artifact
 import mlrun.common.schemas.auth as old_auth
@@ -96,7 +97,7 @@ sys.modules["mlrun.api.schemas.tag"] = old_tag
 # and return the new schema. This is done for backwards compatibility with mlrun.api.schemas.
 ArtifactCategories = DeprecationHelper(mlrun.common.schemas.ArtifactCategories)
 ArtifactIdentifier = DeprecationHelper(mlrun.common.schemas.ArtifactIdentifier)
-ArtifactsFormat = DeprecationHelper(mlrun.common.schemas.ArtifactsFormat)
+ArtifactsFormat = DeprecationHelper(mlrun.common.formatters.ArtifactFormat)
 AuthInfo = DeprecationHelper(mlrun.common.schemas.AuthInfo)
 AuthorizationAction = DeprecationHelper(mlrun.common.schemas.AuthorizationAction)
 AuthorizationResourceTypes = DeprecationHelper(
@@ -221,7 +222,7 @@ ObjectKind = DeprecationHelper(mlrun.common.schemas.ObjectKind)
 ObjectMetadata = DeprecationHelper(mlrun.common.schemas.ObjectMetadata)
 ObjectSpec = DeprecationHelper(mlrun.common.schemas.ObjectSpec)
 ObjectStatus = DeprecationHelper(mlrun.common.schemas.ObjectStatus)
-PipelinesFormat = DeprecationHelper(mlrun.common.schemas.PipelinesFormat)
+PipelinesFormat = DeprecationHelper(mlrun.common.formatters.PipelineFormat)
 PipelinesOutput = DeprecationHelper(mlrun.common.schemas.PipelinesOutput)
 PipelinesPagination = DeprecationHelper(mlrun.common.schemas.PipelinesPagination)
 IguazioProject = DeprecationHelper(mlrun.common.schemas.IguazioProject)
@@ -229,7 +230,7 @@ Project = DeprecationHelper(mlrun.common.schemas.Project)
 ProjectDesiredState = DeprecationHelper(mlrun.common.schemas.ProjectDesiredState)
 ProjectMetadata = DeprecationHelper(mlrun.common.schemas.ProjectMetadata)
 ProjectOwner = DeprecationHelper(mlrun.common.schemas.ProjectOwner)
-ProjectsFormat = DeprecationHelper(mlrun.common.schemas.ProjectsFormat)
+ProjectsFormat = DeprecationHelper(mlrun.common.formatters.ProjectFormat)
 ProjectsOutput = DeprecationHelper(mlrun.common.schemas.ProjectsOutput)
 ProjectSpec = DeprecationHelper(mlrun.common.schemas.ProjectSpec)
 ProjectState = DeprecationHelper(mlrun.common.schemas.ProjectState)

mlrun/artifacts/__init__.py

@@ -17,14 +17,19 @@
 # Don't remove this, used by sphinx documentation
 __all__ = ["get_model", "update_model"]
 
-from .base import Artifact, ArtifactMetadata, ArtifactSpec, get_artifact_meta
+from .base import (
+    Artifact,
+    ArtifactMetadata,
+    ArtifactSpec,
+    DirArtifact,
+    get_artifact_meta,
+)
 from .dataset import DatasetArtifact, TableArtifact, update_dataset_meta
 from .manager import (
     ArtifactManager,
     ArtifactProducer,
     artifact_types,
     dict_to_artifact,
-    legacy_artifact_types,
 )
 from .model import ModelArtifact, get_model, update_model
-from .plots import BokehArtifact, ChartArtifact, PlotArtifact, PlotlyArtifact
+from .plots import PlotArtifact, PlotlyArtifact