mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/db/httpdb.py

@@ -18,19 +18,19 @@ import tempfile
 import time
 import traceback
 import typing
-import warnings
 from datetime import datetime
 from os import path, remove
 from typing import Dict, List, Optional, Union
+from urllib.parse import urlparse
 
 import kfp
 import requests
 import semver
 
 import mlrun
-import mlrun.api.utils.helpers
 import mlrun.common.schemas
 import mlrun.model_monitoring.model_endpoint
+import mlrun.platforms
 import mlrun.projects
 from mlrun.errors import MLRunInvalidArgumentError, err_to_str
 
@@ -107,11 +107,7 @@ class HTTPRunDB(RunDBInterface):
         r"\/?run\/.+\/.+",
     ]
 
-    def __init__(self, base_url, user="", password="", token=""):
-        self.base_url = base_url
-        self.user = user
-        self.password = password
-        self.token = token
+    def __init__(self, url):
         self.server_version = ""
         self.session = None
         self._wait_for_project_terminal_state_retry_interval = 3
@@ -120,6 +116,33 @@ class HTTPRunDB(RunDBInterface):
         self.client_version = version.Version().get()["version"]
         self.python_version = str(version.Version().get_python_version())
 
+        self._enrich_and_validate(url)
+
+    def _enrich_and_validate(self, url):
+        parsed_url = urlparse(url)
+        scheme = parsed_url.scheme.lower()
+        if scheme not in ("http", "https"):
+            raise ValueError(
+                f"Invalid URL scheme {scheme} for HTTPRunDB, only http(s) is supported"
+            )
+
+        endpoint = parsed_url.hostname
+        if parsed_url.port:
+            endpoint += f":{parsed_url.port}"
+        base_url = f"{parsed_url.scheme}://{endpoint}{parsed_url.path}"
+
+        username = parsed_url.username or config.httpdb.user
+        password = parsed_url.password or config.httpdb.password
+
+        username, password, token = mlrun.platforms.add_or_refresh_credentials(
+            parsed_url.hostname, username, password, config.httpdb.token
+        )
+
+        self.base_url = base_url
+        self.user = username
+        self.password = password
+        self.token = token
+
     def __repr__(self):
         cls = self.__class__.__name__
         return f"{cls}({self.base_url!r})"
@@ -416,6 +439,10 @@ class HTTPRunDB(RunDBInterface):
             )
             config.function = server_cfg.get("function") or config.function
             config.httpdb.logs = server_cfg.get("logs") or config.httpdb.logs
+            config.model_endpoint_monitoring.store_type = (
+                server_cfg.get("model_endpoint_monitoring_store_type")
+                or config.model_endpoint_monitoring.store_type
+            )
 
         except Exception as exc:
             logger.warning(
@@ -1353,8 +1380,6 @@ class HTTPRunDB(RunDBInterface):
         namespace=None,
         artifact_path=None,
         ops=None,
-        # TODO: deprecated, remove in 1.5.0
-        ttl=None,
         cleanup_ttl=None,
     ):
         """Submit a KFP pipeline for execution.
@@ -1367,27 +1392,17 @@ class HTTPRunDB(RunDBInterface):
         :param namespace: Kubernetes namespace to execute the pipeline in.
         :param artifact_path: A path to artifacts used by this pipeline.
         :param ops: Transformers to apply on all ops in the pipeline.
-        :param ttl: pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the workflow
-                    and all its resources are deleted) (deprecated, use cleanup_ttl instead)
         :param cleanup_ttl: pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
                             workflow and all its resources are deleted)
         """
 
-        if ttl:
-            warnings.warn(
-                "'ttl' is deprecated, use 'cleanup_ttl' instead. "
-                "This will be removed in 1.5.0",
-                # TODO: Remove this in 1.5.0
-                FutureWarning,
-            )
-
         if isinstance(pipeline, str):
             pipe_file = pipeline
         else:
             pipe_file = tempfile.NamedTemporaryFile(suffix=".yaml", delete=False).name
             conf = new_pipe_metadata(
                 artifact_path=artifact_path,
-                cleanup_ttl=cleanup_ttl or ttl,
+                cleanup_ttl=cleanup_ttl,
                 op_transformers=ops,
             )
             kfp.compiler.Compiler().compile(
@@ -3085,6 +3100,146 @@ class HTTPRunDB(RunDBInterface):
             },
         )
 
+    def store_run_notifications(
+        self,
+        notification_objects: typing.List[mlrun.model.Notification],
+        run_uid: str,
+        project: str = None,
+        mask_params: bool = True,
+    ):
+        """
+        For internal use.
+        The notification mechanism may run "locally" for certain runtimes.
+        However, the updates occur in the API so nothing to do here.
+        """
+        pass
+
+    def submit_workflow(
+        self,
+        project: str,
+        name: str,
+        workflow_spec: Union[
+            mlrun.projects.pipelines.WorkflowSpec,
+            mlrun.common.schemas.WorkflowSpec,
+            dict,
+        ],
+        arguments: Optional[Dict] = None,
+        artifact_path: Optional[str] = None,
+        source: Optional[str] = None,
+        run_name: Optional[str] = None,
+        namespace: Optional[str] = None,
+    ):
+        """
+        Submitting workflow for a remote execution.
+
+        :param project:         project name
+        :param name:            workflow name
+        :param workflow_spec:   the workflow spec to execute
+        :param arguments:       arguments for the workflow
+        :param artifact_path:   artifact target path of the workflow
+        :param source:          source url of the project
+        :param run_name:        run name to override the default: 'workflow-runner-<workflow name>'
+        :param namespace:       kubernetes namespace if other than default
+
+        :returns:    :py:class:`~mlrun.common.schemas.WorkflowResponse`.
+        """
+        image = (
+            workflow_spec.image
+            if hasattr(workflow_spec, "image")
+            else workflow_spec.get("image", None)
+        )
+        req = {
+            "arguments": arguments,
+            "artifact_path": artifact_path,
+            "source": source,
+            "run_name": run_name,
+            "namespace": namespace,
+        }
+        if isinstance(workflow_spec, mlrun.common.schemas.WorkflowSpec):
+            req["spec"] = workflow_spec.dict()
+        elif isinstance(workflow_spec, mlrun.projects.pipelines.WorkflowSpec):
+            req["spec"] = workflow_spec.to_dict()
+        else:
+            req["spec"] = workflow_spec
+        req["spec"]["image"] = image
+        response = self.api_call(
+            "POST",
+            f"projects/{project}/workflows/{name}/submit",
+            json=req,
+        )
+        return mlrun.common.schemas.WorkflowResponse(**response.json())
+
+    def get_workflow_id(
+        self,
+        project: str,
+        name: str,
+        run_id: str,
+        engine: str = "",
+    ):
+        """
+        Retrieve workflow id from the uid of the workflow runner.
+
+        :param project: project name
+        :param name:    workflow name
+        :param run_id:  the id of the workflow runner - the job that runs the workflow
+        :param engine:  pipeline runner
+
+        :returns:   :py:class:`~mlrun.common.schemas.GetWorkflowResponse`.
+        """
+        params = {}
+        if engine:
+            params["engine"] = engine
+        response = self.api_call(
+            "GET",
+            f"projects/{project}/workflows/{name}/runs/{run_id}",
+            params=params,
+        )
+        return mlrun.common.schemas.GetWorkflowResponse(**response.json())
+
+    def load_project(
+        self,
+        name: str,
+        url: str,
+        secrets: Optional[Dict] = None,
+        save_secrets: bool = True,
+    ) -> str:
+        """
+        Loading a project remotely from the given source.
+        :param name:            project name
+        :param url:             git or tar.gz or .zip sources archive path e.g.:
+                                git://github.com/mlrun/demo-xgb-project.git
+                                http://mysite/archived-project.zip
+                                The git project should include the project yaml file.
+        :param secrets:         Secrets to store in project in order to load it from the provided url.
+                                For more information see :py:func:`mlrun.load_project` function.
+        :param save_secrets:    Whether to store secrets in the loaded project.
+                                Setting to False will cause waiting for the process completion.
+
+        :returns:               The terminal state of load project process.
+        """
+        params = {"url": url}
+        body = None
+        if secrets:
+            provider = mlrun.common.schemas.SecretProviderName.kubernetes
+            secrets_input = mlrun.common.schemas.SecretsData(
+                provider=provider, secrets=secrets
+            )
+            body = secrets_input.dict()
+        response = self.api_call(
+            "POST", f"projects/{name}/load", params=params, body=dict_to_json(body)
+        )
+        response = response.json()
+        run = mlrun.RunObject.from_dict(response["data"])
+        state, _ = run.logs()
+
+        if secrets and not save_secrets:
+            self.delete_project_secrets(project=name, secrets=list(secrets.keys()))
+            if state != "completed":
+                logger.error("Load project task failed, deleting project")
+                self.delete_project(name, mlrun.common.schemas.DeletionStrategy.cascade)
+
+        return state
+
 
 def _as_json(obj):
     fn = getattr(obj, "to_json", None)

mlrun/db/nopdb.py

@@ -31,7 +31,7 @@ class NopDB(RunDBInterface):
     def __getattribute__(self, attr):
         def nop(*args, **kwargs):
             env_var_message = (
-                "MLRUN_DBPATH is not set. Set this environment variable to the URL of the API "
+                "MLRUN_DBPATH is misconfigured. Set this environment variable to the URL of the API "
                 "server in order to connect"
             )
             if config.httpdb.nop_db.raise_error:
@@ -45,7 +45,8 @@ class NopDB(RunDBInterface):
 
             return
 
-        if attr == "connect":
+        # ignore __class__ because __getattribute__ overrides the parent class's method and it spams logs
+        if attr in ["connect", "__class__"]:
             return super().__getattribute__(attr)
         else:
             nop()
@@ -93,6 +94,7 @@ class NopDB(RunDBInterface):
             mlrun.common.schemas.OrderType, str
         ] = mlrun.common.schemas.OrderType.desc,
         max_partitions: int = 0,
+        with_notifications: bool = False,
     ):
         pass
 

mlrun/execution.py

@@ -1038,8 +1038,10 @@ class MLClientCtx(object):
 
     def _update_run(self, commit=False, message=""):
         """
-        update the required fields in the run object (using mlrun.utils.helpers.update_in)
-        instead of overwriting existing
+        update the required fields in the run object instead of overwriting existing values with empty ones
+
+        :param commit:  commit the changes to the DB if autocommit is not set or update the tmpfile alone
+        :param message: commit message
         """
         self._merge_tmpfile()
         if commit or self._autocommit:

mlrun/feature_store/__init__.py

@@ -53,6 +53,7 @@ from .feature_set import FeatureSet
 from .feature_vector import (
     FeatureVector,
     FixedWindowType,
+    JoinGraph,
     OfflineVectorResponse,
     OnlineVectorService,
 )

mlrun/feature_store/api.py

@@ -35,7 +35,6 @@ from ..datastore.targets import (
     validate_target_list,
     validate_target_paths_for_engine,
 )
-from ..db import RunDBError
 from ..model import DataSource, DataTargetBase
 from ..runtimes import RuntimeKinds
 from ..runtimes.function_reference import FunctionReference
@@ -417,7 +416,7 @@ def ingest(
             _, stripped_name = parse_store_uri(featureset)
             try:
                 featureset = get_feature_set_by_uri(stripped_name)
-            except RunDBError as exc:
+            except mlrun.db.RunDBError as exc:
                 # TODO: this handling is needed because the generic httpdb error handling doesn't raise the correct
                 #  error class and doesn't propagate the correct message, until it solved we're manually handling this
                 #  case to give better user experience, remove this when the error handling is fixed.

mlrun/feature_store/common.py

@@ -20,8 +20,9 @@ from mlrun.common.schemas import AuthorizationVerificationInput
 from mlrun.runtimes import BaseRuntime
 from mlrun.runtimes.function_reference import FunctionReference
 from mlrun.runtimes.utils import enrich_function_from_dict
-from mlrun.utils import StorePrefix, logger, parse_versioned_object_uri
+from mlrun.utils import StorePrefix, logger
 
+from ..common.helpers import parse_versioned_object_uri
 from ..config import config
 
 project_separator = "/"

mlrun/feature_store/feature_set.py

@@ -273,7 +273,7 @@ class FeatureSetStatus(ModelObj):
         :param run_uri: last run used for ingestion
         """
 
-        self.state = state or "created"
+        self.state = state or mlrun.common.schemas.object.ObjectStatusState.CREATED
         self._targets: ObjectList = None
         self.targets = targets or []
         self.stats = stats or {}
@@ -418,16 +418,6 @@ class FeatureSet(ModelObj):
             fullname += ":" + self._metadata.tag
         return fullname
 
-    def _override_run_db(
-        self,
-        session,
-    ):
-        # Import here, since this method only runs in API context. If this import was global, client would need
-        # API requirements and would fail.
-        from ..api.api.utils import get_run_db_instance
-
-        self._run_db = get_run_db_instance(session)
-
     def _get_run_db(self):
         if self._run_db:
             return self._run_db