mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/__init__.py

@@ -22,11 +22,16 @@ __all__ = [
     "handler",
     "ArtifactType",
     "get_secret_or_env",
+    "mount_v3io",
+    "v3io_cred",
+    "auto_mount",
+    "VolumeMount",
 ]
 
 from os import environ, path
 
 import dotenv
+import mlrun_pipelines
 
 from .config import config as mlconf
 from .datastore import DataItem, store_manager
@@ -35,7 +40,6 @@ from .errors import MLRunInvalidArgumentError, MLRunNotFoundError
 from .execution import MLClientCtx
 from .model import RunObject, RunTemplate, new_task
 from .package import ArtifactType, DefaultPackager, Packager, handler
-from .platforms import VolumeMount, auto_mount, mount_v3io, v3io_cred
 from .projects import (
     ProjectMetadata,
     build_function,
@@ -65,6 +69,11 @@ from .utils.version import Version
 
 __version__ = Version().get()["version"]
 
+VolumeMount = mlrun_pipelines.common.mounts.VolumeMount
+mount_v3io = mlrun_pipelines.mounts.mount_v3io
+v3io_cred = mlrun_pipelines.mounts.v3io_cred
+auto_mount = mlrun_pipelines.mounts.auto_mount
+
 
 def get_version():
     """get current mlrun version"""
@@ -97,6 +106,7 @@ def set_environment(
     example::
 
         from os import path
+
         project_name, artifact_path = set_environment()
         set_environment("http://localhost:8080", artifact_path="./")
         set_environment(env_file="mlrun.env")

mlrun/__main__.py

@@ -22,17 +22,16 @@ from ast import literal_eval
 from base64 import b64decode, b64encode
 from os import environ, path, remove
 from pprint import pprint
-from subprocess import Popen
-from sys import executable
-from urllib.parse import urlparse
 
 import click
 import dotenv
 import pandas as pd
 import yaml
+from mlrun_pipelines.mounts import auto_mount as auto_mount_modifier
 from tabulate import tabulate
 
 import mlrun
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 from mlrun.common.helpers import parse_versioned_object_uri
 
@@ -40,7 +39,6 @@ from .config import config as mlconf
 from .db import get_run_db
 from .errors import err_to_str
 from .model import RunTemplate
-from .platforms import auto_mount as auto_mount_modifier
 from .projects import load_project
 from .run import (
     get_object,
@@ -259,8 +257,10 @@ def run(
             runobj.metadata.labels[k] = v
 
     if workflow:
-        runobj.metadata.labels["workflow"] = workflow
-        runobj.metadata.labels["mlrun/runner-pod"] = socket.gethostname()
+        runobj.metadata.labels[mlrun_constants.MLRunInternalLabels.workflow] = workflow
+        runobj.metadata.labels[mlrun_constants.MLRunInternalLabels.runner_pod] = (
+            socket.gethostname()
+        )
 
     if db:
         mlconf.dbpath = db
@@ -469,6 +469,17 @@ def run(
     is_flag=True,
     help="ensure the project exists, if not, create project",
 )
+@click.option(
+    "--state-file-path", default="/tmp/state", help="path to file with state data"
+)
+@click.option(
+    "--image-file-path", default="/tmp/image", help="path to file with image data"
+)
+@click.option(
+    "--full-image-file-path",
+    default="/tmp/fullimage",
+    help="path to file with full image data",
+)
 def build(
     func_url,
     name,
@@ -488,6 +499,9 @@ def build(
     skip,
     env_file,
     ensure_project,
+    state_file_path,
+    image_file_path,
+    full_image_file_path,
 ):
     """Build a container image from code and requirements."""
 
@@ -505,6 +519,8 @@ def build(
         if kfp:
             print("Runtime:")
             pprint(runtime)
+        # use kind = "job" by default if not specified
+        runtime.setdefault("kind", "job")
         func = new_function(runtime=runtime)
 
     elif func_url:
@@ -575,12 +591,12 @@ def build(
         state = func.status.state
         image = func.spec.image
         if kfp:
-            with open("/tmp/state", "w") as fp:
+            with open(state_file_path, "w") as fp:
                 fp.write(state or "none")
             full_image = func.full_image_path(image) or ""
-            with open("/tmp/image", "w") as fp:
+            with open(image_file_path, "w") as fp:
                 fp.write(image)
-            with open("/tmp/fullimage", "w") as fp:
+            with open(full_image_file_path, "w") as fp:
                 fp.write(full_image)
             print("Full image path = ", full_image)
 
@@ -825,108 +841,6 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
         )
 
 
-@main.command(deprecated=True)
-@click.option("--port", "-p", help="port to listen on", type=int)
-@click.option("--dirpath", "-d", help="database directory (dirpath)")
-@click.option("--dsn", "-s", help="database dsn, e.g. sqlite:///db/mlrun.db")
-@click.option("--logs-path", "-l", help="logs directory path")
-@click.option("--data-volume", "-v", help="path prefix to the location of artifacts")
-@click.option("--verbose", is_flag=True, help="verbose log")
-@click.option("--background", "-b", is_flag=True, help="run in background process")
-@click.option("--artifact-path", "-a", help="default artifact path")
-@click.option(
-    "--update-env",
-    default="",
-    is_flag=False,
-    flag_value=mlrun.config.default_env_file,
-    help=f"update the specified mlrun .env file (if TEXT not provided defaults to {mlrun.config.default_env_file})",
-)
-def db(
-    port,
-    dirpath,
-    dsn,
-    logs_path,
-    data_volume,
-    verbose,
-    background,
-    artifact_path,
-    update_env,
-):
-    """Run HTTP api/database server"""
-    warnings.warn(
-        "The `mlrun db` command is deprecated in 1.5.0 and will be removed in 1.7.0, it is for internal use only.",
-        FutureWarning,
-    )
-    env = environ.copy()
-    # ignore client side .env file (so import mlrun in server will not try to connect to local/remote DB)
-    env["MLRUN_IGNORE_ENV_FILE"] = "true"
-    env["MLRUN_DBPATH"] = ""
-
-    if port is not None:
-        env["MLRUN_httpdb__port"] = str(port)
-    if dirpath is not None:
-        env["MLRUN_httpdb__dirpath"] = dirpath
-    if dsn is not None:
-        if dsn.startswith("sqlite://") and "check_same_thread=" not in dsn:
-            dsn += "?check_same_thread=false"
-        env["MLRUN_HTTPDB__DSN"] = dsn
-    if logs_path is not None:
-        env["MLRUN_HTTPDB__LOGS_PATH"] = logs_path
-    if data_volume is not None:
-        env["MLRUN_HTTPDB__DATA_VOLUME"] = data_volume
-    if verbose:
-        env["MLRUN_LOG_LEVEL"] = "DEBUG"
-    if artifact_path or "MLRUN_ARTIFACT_PATH" not in env:
-        if not artifact_path:
-            artifact_path = (
-                env.get("MLRUN_HTTPDB__DATA_VOLUME", "./artifacts").rstrip("/")
-                + "/{{project}}"
-            )
-        env["MLRUN_ARTIFACT_PATH"] = path.realpath(path.expanduser(artifact_path))
-
-    env["MLRUN_IS_API_SERVER"] = "true"
-
-    # create the DB dir if needed
-    dsn = dsn or mlconf.httpdb.dsn
-    if dsn and dsn.startswith("sqlite:///"):
-        parsed = urlparse(dsn)
-        p = pathlib.Path(parsed.path[1:]).parent
-        p.mkdir(parents=True, exist_ok=True)
-
-    cmd = [executable, "-m", "server.api.main"]
-    pid = None
-    if background:
-        print("Starting MLRun API service in the background...")
-        child = Popen(
-            cmd,
-            env=env,
-            stdout=open("mlrun-stdout.log", "w"),
-            stderr=open("mlrun-stderr.log", "w"),
-            start_new_session=True,
-        )
-        pid = child.pid
-        print(
-            f"Background pid: {pid}, logs written to mlrun-stdout.log and mlrun-stderr.log, use:\n"
-            f"`kill {pid}` (linux/mac) or `taskkill /pid {pid} /t /f` (windows), to kill the mlrun service process"
-        )
-    else:
-        child = Popen(cmd, env=env)
-        returncode = child.wait()
-        if returncode != 0:
-            raise SystemExit(returncode)
-    if update_env:
-        # update mlrun client env file with the API path, so client will use the new DB
-        # update and PID, allow killing the correct process in a config script
-        filename = path.expanduser(update_env)
-        dotenv.set_key(
-            filename, "MLRUN_DBPATH", f"http://localhost:{port or 8080}", quote_mode=""
-        )
-        dotenv.set_key(filename, "MLRUN_MOCK_NUCLIO_DEPLOYMENT", "auto", quote_mode="")
-        if pid:
-            dotenv.set_key(filename, "MLRUN_SERVICE_PID", str(pid), quote_mode="")
-        print(f"Updated configuration in {update_env} .env file")
-
-
 @main.command()
 def version():
     """get mlrun version"""
@@ -1452,7 +1366,7 @@ def load_notification(notifications: str, project: mlrun.projects.MlrunProject):
     for notification in notifications:
         if notification.startswith("file="):
             file_path = notification.split("=")[-1]
-            notification = open(file_path, "r")
+            notification = open(file_path)
             notification = json.load(notification)
         else:
             notification = json.loads(notification)

mlrun/alerts/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .alert import AlertConfig

mlrun/alerts/alert.py

@@ -0,0 +1,144 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Union
+
+import mlrun
+import mlrun.common.schemas.alert as alert_objects
+from mlrun.model import ModelObj
+
+
+class AlertConfig(ModelObj):
+    _dict_fields = [
+        "project",
+        "name",
+        "description",
+        "summary",
+        "severity",
+        "criteria",
+        "reset_policy",
+        "state",
+    ]
+
+    def __init__(
+        self,
+        project: str = None,
+        name: str = None,
+        template: Union[alert_objects.AlertTemplate, str] = None,
+        description: str = None,
+        summary: str = None,
+        severity: alert_objects.AlertSeverity = None,
+        trigger: alert_objects.AlertTrigger = None,
+        criteria: alert_objects.AlertCriteria = None,
+        reset_policy: alert_objects.ResetPolicy = None,
+        notifications: list[alert_objects.AlertNotification] = None,
+        entities: alert_objects.EventEntities = None,
+        id: int = None,
+        state: alert_objects.AlertActiveState = None,
+        created: str = None,
+        count: int = None,
+    ):
+        self.project = project
+        self.name = name
+        self.description = description
+        self.summary = summary
+        self.severity = severity
+        self.trigger = trigger
+        self.criteria = criteria
+        self.reset_policy = reset_policy
+        self.notifications = notifications or []
+        self.entities = entities
+        self.id = id
+        self.state = state
+        self.created = created
+        self.count = count
+
+        if template:
+            self._apply_template(template)
+
+    def validate_required_fields(self):
+        if not self.project or not self.name:
+            raise mlrun.errors.MLRunBadRequestError("Project and name must be provided")
+
+    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
+        data = super().to_dict(self._dict_fields)
+
+        data["entities"] = (
+            self.entities.dict()
+            if not isinstance(self.entities, dict)
+            else self.entities
+        )
+        data["notifications"] = [
+            notification_data.dict()
+            if not isinstance(notification_data, dict)
+            else notification_data
+            for notification_data in self.notifications
+        ]
+        data["trigger"] = (
+            self.trigger.dict() if not isinstance(self.trigger, dict) else self.trigger
+        )
+        return data
+
+    @classmethod
+    def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
+        new_obj = super().from_dict(struct, fields=fields)
+
+        entity_data = struct.get("entities")
+        if entity_data:
+            entity_obj = alert_objects.EventEntities.parse_obj(entity_data)
+            new_obj.entities = entity_obj
+
+        notifications_data = struct.get("notifications")
+        if notifications_data:
+            notifications_objs = [
+                alert_objects.AlertNotification.parse_obj(notification)
+                for notification in notifications_data
+            ]
+            new_obj.notifications = notifications_objs
+
+        trigger_data = struct.get("trigger")
+        if trigger_data:
+            trigger_obj = alert_objects.AlertTrigger.parse_obj(trigger_data)
+            new_obj.trigger = trigger_obj
+
+        return new_obj
+
+    def with_notifications(self, notifications: list[alert_objects.AlertNotification]):
+        if not isinstance(notifications, list) or not all(
+            isinstance(item, alert_objects.AlertNotification) for item in notifications
+        ):
+            raise ValueError(
+                "Notifications parameter must be a list of AlertNotification"
+            )
+        for notification_data in notifications:
+            self.notifications.append(notification_data)
+        return self
+
+    def with_entities(self, entities: alert_objects.EventEntities):
+        if not isinstance(entities, alert_objects.EventEntities):
+            raise ValueError("Entities parameter must be of type: EventEntities")
+        self.entities = entities
+        return self
+
+    def _apply_template(self, template):
+        if isinstance(template, str):
+            db = mlrun.get_run_db()
+            template = db.get_alert_template(template)
+
+        # Extract parameters from the template and apply them to the AlertConfig object
+        self.summary = template.summary
+        self.severity = template.severity
+        self.criteria = template.criteria
+        self.trigger = template.trigger
+        self.reset_policy = template.reset_policy

mlrun/api/schemas/__init__.py

@@ -25,6 +25,7 @@ schema.
 import sys
 import warnings
 
+import mlrun.common.formatters
 import mlrun.common.schemas
 import mlrun.common.schemas.artifact as old_artifact
 import mlrun.common.schemas.auth as old_auth
@@ -48,7 +49,7 @@ import mlrun.common.schemas.secret as old_secret
 import mlrun.common.schemas.tag as old_tag
 
 
-class DeprecationHelper(object):
+class DeprecationHelper:
     """A helper class to deprecate old schemas"""
 
     def __init__(self, new_target, version="1.4.0"):
@@ -96,7 +97,7 @@ sys.modules["mlrun.api.schemas.tag"] = old_tag
 # and return the new schema. This is done for backwards compatibility with mlrun.api.schemas.
 ArtifactCategories = DeprecationHelper(mlrun.common.schemas.ArtifactCategories)
 ArtifactIdentifier = DeprecationHelper(mlrun.common.schemas.ArtifactIdentifier)
-ArtifactsFormat = DeprecationHelper(mlrun.common.schemas.ArtifactsFormat)
+ArtifactsFormat = DeprecationHelper(mlrun.common.formatters.ArtifactsFormat)
 AuthInfo = DeprecationHelper(mlrun.common.schemas.AuthInfo)
 AuthorizationAction = DeprecationHelper(mlrun.common.schemas.AuthorizationAction)
 AuthorizationResourceTypes = DeprecationHelper(
@@ -221,7 +222,7 @@ ObjectKind = DeprecationHelper(mlrun.common.schemas.ObjectKind)
 ObjectMetadata = DeprecationHelper(mlrun.common.schemas.ObjectMetadata)
 ObjectSpec = DeprecationHelper(mlrun.common.schemas.ObjectSpec)
 ObjectStatus = DeprecationHelper(mlrun.common.schemas.ObjectStatus)
-PipelinesFormat = DeprecationHelper(mlrun.common.schemas.PipelinesFormat)
+PipelinesFormat = DeprecationHelper(mlrun.common.formatters.PipelinesFormat)
 PipelinesOutput = DeprecationHelper(mlrun.common.schemas.PipelinesOutput)
 PipelinesPagination = DeprecationHelper(mlrun.common.schemas.PipelinesPagination)
 IguazioProject = DeprecationHelper(mlrun.common.schemas.IguazioProject)
@@ -229,7 +230,7 @@ Project = DeprecationHelper(mlrun.common.schemas.Project)
 ProjectDesiredState = DeprecationHelper(mlrun.common.schemas.ProjectDesiredState)
 ProjectMetadata = DeprecationHelper(mlrun.common.schemas.ProjectMetadata)
 ProjectOwner = DeprecationHelper(mlrun.common.schemas.ProjectOwner)
-ProjectsFormat = DeprecationHelper(mlrun.common.schemas.ProjectsFormat)
+ProjectsFormat = DeprecationHelper(mlrun.common.formatters.ProjectsFormat)
 ProjectsOutput = DeprecationHelper(mlrun.common.schemas.ProjectsOutput)
 ProjectSpec = DeprecationHelper(mlrun.common.schemas.ProjectSpec)
 ProjectState = DeprecationHelper(mlrun.common.schemas.ProjectState)

mlrun/artifacts/__init__.py

@@ -17,14 +17,19 @@
 # Don't remove this, used by sphinx documentation
 __all__ = ["get_model", "update_model"]
 
-from .base import Artifact, ArtifactMetadata, ArtifactSpec, get_artifact_meta
+from .base import (
+    Artifact,
+    ArtifactMetadata,
+    ArtifactSpec,
+    DirArtifact,
+    get_artifact_meta,
+)
 from .dataset import DatasetArtifact, TableArtifact, update_dataset_meta
 from .manager import (
     ArtifactManager,
     ArtifactProducer,
     artifact_types,
     dict_to_artifact,
-    legacy_artifact_types,
 )
 from .model import ModelArtifact, get_model, update_model
-from .plots import BokehArtifact, ChartArtifact, PlotArtifact, PlotlyArtifact
+from .plots import PlotArtifact, PlotlyArtifact