mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/db/sqldb.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,8 @@
 import datetime
 from typing import List, Optional, Union
 
-import mlrun.api.schemas
+import mlrun.common.schemas
+import mlrun.model_monitoring.model_endpoint
 from mlrun.api.db.base import DBError
 from mlrun.api.db.sqldb.db import SQLDB as SQLAPIDB
 from mlrun.api.db.sqldb.session import create_session
@@ -28,8 +29,6 @@ from mlrun.api.db.sqldb.session import create_session
 # service, in order to prevent the api from calling itself several times for each submission request (since the runDB
 # will be httpdb to that same api service) we have this class which is kind of a proxy between the RunDB interface to
 # the api service's DB interface
-from ..api import schemas
-from ..api.schemas import ModelEndpoint
 from .base import RunDBError, RunDBInterface
 
 
@@ -95,7 +94,7 @@ class SQLDB(RunDBInterface):
             updates,
         )
 
-    def abort_run(self, uid, project="", iter=0):
+    def abort_run(self, uid, project="", iter=0, timeout=45):
         raise NotImplementedError()
 
     def read_run(self, uid, project=None, iter=None):
@@ -123,11 +122,14 @@ class SQLDB(RunDBInterface):
         start_time_to: datetime.datetime = None,
         last_update_time_from: datetime.datetime = None,
         last_update_time_to: datetime.datetime = None,
-        partition_by: Union[schemas.RunPartitionByField, str] = None,
+        partition_by: Union[mlrun.common.schemas.RunPartitionByField, str] = None,
         rows_per_partition: int = 1,
-        partition_sort_by: Union[schemas.SortField, str] = None,
-        partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
+        partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
         max_partitions: int = 0,
+        with_notifications: bool = False,
     ):
         import mlrun.api.crud
 
@@ -151,6 +153,7 @@ class SQLDB(RunDBInterface):
             partition_sort_by,
             partition_order,
             max_partitions,
+            with_notifications,
         )
 
     def del_run(self, uid, project=None, iter=None):
@@ -214,12 +217,12 @@ class SQLDB(RunDBInterface):
         iter: int = None,
         best_iteration: bool = False,
         kind: str = None,
-        category: Union[str, schemas.ArtifactCategories] = None,
+        category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
     ):
         import mlrun.api.crud
 
         if category and isinstance(category, str):
-            category = schemas.ArtifactCategories(category)
+            category = mlrun.common.schemas.ArtifactCategories(category)
 
         return self._transform_db_error(
             mlrun.api.crud.Artifacts().list_artifacts,
@@ -307,7 +310,9 @@ class SQLDB(RunDBInterface):
         )
 
     def list_artifact_tags(
-        self, project=None, category: Union[str, schemas.ArtifactCategories] = None
+        self,
+        project=None,
+        category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
     ):
         return self._transform_db_error(
             self.db.list_artifact_tags, self.session, project
@@ -317,7 +322,7 @@ class SQLDB(RunDBInterface):
         self,
         project: str,
         tag_name: str,
-        tag_objects: mlrun.api.schemas.TagObjects,
+        tag_objects: mlrun.common.schemas.TagObjects,
         replace: bool = False,
     ):
         import mlrun.api.crud
@@ -343,7 +348,7 @@ class SQLDB(RunDBInterface):
         self,
         project: str,
         tag_name: str,
-        tag_objects: mlrun.api.schemas.TagObjects,
+        tag_objects: mlrun.common.schemas.TagObjects,
     ):
         import mlrun.api.crud
 
@@ -392,34 +397,65 @@ class SQLDB(RunDBInterface):
     def store_project(
         self,
         name: str,
-        project: mlrun.api.schemas.Project,
-    ) -> mlrun.api.schemas.Project:
-        raise NotImplementedError()
+        project: mlrun.common.schemas.Project,
+    ) -> mlrun.common.schemas.Project:
+        import mlrun.api.crud
+
+        if isinstance(project, dict):
+            project = mlrun.common.schemas.Project(**project)
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().store_project,
+            self.session,
+            name=name,
+            project=project,
+        )
 
     def patch_project(
         self,
         name: str,
         project: dict,
-        patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
-    ) -> mlrun.api.schemas.Project:
-        raise NotImplementedError()
+        patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
+    ) -> mlrun.common.schemas.Project:
+        import mlrun.api.crud
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().patch_project,
+            self.session,
+            name=name,
+            project=project,
+            patch_mode=patch_mode,
+        )
 
     def create_project(
         self,
-        project: mlrun.api.schemas.Project,
-    ) -> mlrun.api.schemas.Project:
-        raise NotImplementedError()
+        project: mlrun.common.schemas.Project,
+    ) -> mlrun.common.schemas.Project:
+        import mlrun.api.crud
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().create_project,
+            self.session,
+            project=project,
+        )
 
     def delete_project(
         self,
         name: str,
-        deletion_strategy: mlrun.api.schemas.DeletionStrategy = mlrun.api.schemas.DeletionStrategy.default(),
+        deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
     ):
-        raise NotImplementedError()
+        import mlrun.api.crud
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().delete_project,
+            self.session,
+            name=name,
+            deletion_strategy=deletion_strategy,
+        )
 
     def get_project(
         self, name: str = None, project_id: int = None
-    ) -> mlrun.api.schemas.Project:
+    ) -> mlrun.common.schemas.Project:
         import mlrun.api.crud
 
         return self._transform_db_error(
@@ -431,11 +467,20 @@ class SQLDB(RunDBInterface):
     def list_projects(
         self,
         owner: str = None,
-        format_: mlrun.api.schemas.ProjectsFormat = mlrun.api.schemas.ProjectsFormat.full,
+        format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
         labels: List[str] = None,
-        state: mlrun.api.schemas.ProjectState = None,
-    ) -> mlrun.api.schemas.ProjectsOutput:
-        raise NotImplementedError()
+        state: mlrun.common.schemas.ProjectState = None,
+    ) -> mlrun.common.schemas.ProjectsOutput:
+        import mlrun.api.crud
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().list_projects,
+            self.session,
+            owner=owner,
+            format_=format_,
+            labels=labels,
+            state=state,
+        )
 
     @staticmethod
     def _transform_db_error(func, *args, **kwargs):
@@ -517,10 +562,10 @@ class SQLDB(RunDBInterface):
         entities: List[str] = None,
         features: List[str] = None,
         labels: List[str] = None,
-        partition_by: mlrun.api.schemas.FeatureStorePartitionByField = None,
+        partition_by: mlrun.common.schemas.FeatureStorePartitionByField = None,
         rows_per_partition: int = 1,
-        partition_sort_by: mlrun.api.schemas.SortField = None,
-        partition_order: mlrun.api.schemas.OrderType = mlrun.api.schemas.OrderType.desc,
+        partition_sort_by: mlrun.common.schemas.SortField = None,
+        partition_order: mlrun.common.schemas.OrderType = mlrun.common.schemas.OrderType.desc,
     ):
         import mlrun.api.crud
 
@@ -542,7 +587,7 @@ class SQLDB(RunDBInterface):
 
     def store_feature_set(
         self,
-        feature_set: Union[dict, mlrun.api.schemas.FeatureSet],
+        feature_set: Union[dict, mlrun.common.schemas.FeatureSet],
         name=None,
         project="",
         tag=None,
@@ -552,7 +597,7 @@ class SQLDB(RunDBInterface):
         import mlrun.api.crud
 
         if isinstance(feature_set, dict):
-            feature_set = mlrun.api.schemas.FeatureSet(**feature_set)
+            feature_set = mlrun.common.schemas.FeatureSet(**feature_set)
 
         name = name or feature_set.metadata.name
         project = project or feature_set.metadata.project
@@ -627,10 +672,10 @@ class SQLDB(RunDBInterface):
         tag: str = None,
         state: str = None,
         labels: List[str] = None,
-        partition_by: mlrun.api.schemas.FeatureStorePartitionByField = None,
+        partition_by: mlrun.common.schemas.FeatureStorePartitionByField = None,
         rows_per_partition: int = 1,
-        partition_sort_by: mlrun.api.schemas.SortField = None,
-        partition_order: mlrun.api.schemas.OrderType = mlrun.api.schemas.OrderType.desc,
+        partition_sort_by: mlrun.common.schemas.SortField = None,
+        partition_order: mlrun.common.schemas.OrderType = mlrun.common.schemas.OrderType.desc,
     ):
         import mlrun.api.crud
 
@@ -712,18 +757,18 @@ class SQLDB(RunDBInterface):
         page_token: str = "",
         filter_: str = "",
         format_: Union[
-            str, mlrun.api.schemas.PipelinesFormat
-        ] = mlrun.api.schemas.PipelinesFormat.metadata_only,
+            str, mlrun.common.schemas.PipelinesFormat
+        ] = mlrun.common.schemas.PipelinesFormat.metadata_only,
         page_size: int = None,
-    ) -> mlrun.api.schemas.PipelinesOutput:
+    ) -> mlrun.common.schemas.PipelinesOutput:
         raise NotImplementedError()
 
     def create_project_secrets(
         self,
         project: str,
         provider: Union[
-            str, mlrun.api.schemas.SecretProviderName
-        ] = mlrun.api.schemas.SecretProviderName.kubernetes,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.kubernetes,
         secrets: dict = None,
     ):
         raise NotImplementedError()
@@ -733,28 +778,28 @@ class SQLDB(RunDBInterface):
         project: str,
         token: str,
         provider: Union[
-            str, mlrun.api.schemas.SecretProviderName
-        ] = mlrun.api.schemas.SecretProviderName.kubernetes,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.kubernetes,
         secrets: List[str] = None,
-    ) -> mlrun.api.schemas.SecretsData:
+    ) -> mlrun.common.schemas.SecretsData:
         raise NotImplementedError()
 
     def list_project_secret_keys(
         self,
         project: str,
         provider: Union[
-            str, mlrun.api.schemas.SecretProviderName
-        ] = mlrun.api.schemas.SecretProviderName.kubernetes,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.kubernetes,
         token: str = None,
-    ) -> mlrun.api.schemas.SecretKeysData:
+    ) -> mlrun.common.schemas.SecretKeysData:
         raise NotImplementedError()
 
     def delete_project_secrets(
         self,
         project: str,
         provider: Union[
-            str, mlrun.api.schemas.SecretProviderName
-        ] = mlrun.api.schemas.SecretProviderName.kubernetes,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.kubernetes,
         secrets: List[str] = None,
     ):
         raise NotImplementedError()
@@ -763,8 +808,8 @@ class SQLDB(RunDBInterface):
         self,
         user: str,
         provider: Union[
-            str, mlrun.api.schemas.SecretProviderName
-        ] = mlrun.api.schemas.SecretProviderName.vault,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.vault,
         secrets: dict = None,
     ):
         raise NotImplementedError()
@@ -773,7 +818,9 @@ class SQLDB(RunDBInterface):
         self,
         project: str,
         endpoint_id: str,
-        model_endpoint: ModelEndpoint,
+        model_endpoint: Union[
+            mlrun.model_monitoring.model_endpoint.ModelEndpoint, dict
+        ],
     ):
         raise NotImplementedError()
 
@@ -815,40 +862,40 @@ class SQLDB(RunDBInterface):
     ):
         raise NotImplementedError()
 
-    def create_marketplace_source(
-        self, source: Union[dict, schemas.IndexedMarketplaceSource]
+    def create_hub_source(
+        self, source: Union[dict, mlrun.common.schemas.IndexedHubSource]
     ):
         raise NotImplementedError()
 
-    def store_marketplace_source(
-        self, source_name: str, source: Union[dict, schemas.IndexedMarketplaceSource]
+    def store_hub_source(
+        self,
+        source_name: str,
+        source: Union[dict, mlrun.common.schemas.IndexedHubSource],
     ):
         raise NotImplementedError()
 
-    def list_marketplace_sources(self):
+    def list_hub_sources(self):
         raise NotImplementedError()
 
-    def get_marketplace_source(self, source_name: str):
+    def get_hub_source(self, source_name: str):
         raise NotImplementedError()
 
-    def delete_marketplace_source(self, source_name: str):
+    def delete_hub_source(self, source_name: str):
         raise NotImplementedError()
 
-    def get_marketplace_catalog(
+    def get_hub_catalog(
         self,
         source_name: str,
-        channel: str = None,
         version: str = None,
         tag: str = None,
         force_refresh: bool = False,
     ):
         raise NotImplementedError()
 
-    def get_marketplace_item(
+    def get_hub_item(
         self,
         source_name: str,
         item_name: str,
-        channel: str = "development",
         version: str = None,
         tag: str = "latest",
         force_refresh: bool = False,
@@ -857,7 +904,7 @@ class SQLDB(RunDBInterface):
 
     def verify_authorization(
         self,
-        authorization_verification_input: mlrun.api.schemas.AuthorizationVerificationInput,
+        authorization_verification_input: mlrun.common.schemas.AuthorizationVerificationInput,
     ):
         # on server side authorization is done in endpoint anyway, so for server side we can "pass" on check
         # done from ingest()

mlrun/errors.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -179,6 +179,10 @@ class MLRunInternalServerError(MLRunHTTPStatusError):
     error_status_code = HTTPStatus.INTERNAL_SERVER_ERROR.value
 
 
+class MLRunServiceUnavailableError(MLRunHTTPStatusError):
+    error_status_code = HTTPStatus.SERVICE_UNAVAILABLE.value
+
+
 class MLRunRuntimeError(MLRunHTTPStatusError, RuntimeError):
     error_status_code = HTTPStatus.INTERNAL_SERVER_ERROR.value
 
@@ -213,4 +217,5 @@ STATUS_ERRORS = {
     HTTPStatus.CONFLICT.value: MLRunConflictError,
     HTTPStatus.PRECONDITION_FAILED.value: MLRunPreconditionFailedError,
     HTTPStatus.INTERNAL_SERVER_ERROR.value: MLRunInternalServerError,
+    HTTPStatus.SERVICE_UNAVAILABLE.value: MLRunServiceUnavailableError,
 }

mlrun/execution.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ from typing import List, Union
 
 import numpy as np
 import yaml
+from dateutil import parser
 
 import mlrun
 from mlrun.artifacts import ModelArtifact
@@ -79,6 +80,7 @@ class MLClientCtx(object):
         self._log_level = "info"
         self._matrics_db = None
         self._autocommit = autocommit
+        self._notifications = []
 
         self._labels = {}
         self._annotations = {}
@@ -298,11 +300,12 @@ class MLClientCtx(object):
             self.artifact_path = spec.get(run_keys.output_path, self.artifact_path)
             self._in_path = spec.get(run_keys.input_path, self._in_path)
             inputs = spec.get(run_keys.inputs)
+            self._notifications = spec.get("notifications", self._notifications)
 
         self._init_dbs(rundb)
 
-        if spec and not is_api:
-            # init data related objects (require DB & Secrets to be set first), skip when running in the api service
+        if spec:
+            # init data related objects (require DB & Secrets to be set first)
             self._data_stores.from_dict(spec)
             if inputs and isinstance(inputs, dict):
                 for k, v in inputs.items():
@@ -314,6 +317,7 @@ class MLClientCtx(object):
 
         start = get_in(attrs, "status.start_time")
         if start:
+            start = parser.parse(start) if isinstance(start, str) else start
             self._start_time = start
         self._state = "running"
         if store_run:
@@ -376,7 +380,7 @@ class MLClientCtx(object):
 
     @property
     def inputs(self):
-        """dictionary of input data items (read-only)"""
+        """dictionary of input data item urls (read-only)"""
         return self._inputs
 
     @property
@@ -459,16 +463,26 @@ class MLClientCtx(object):
     def _load_project_object(self):
         if not self._project_object:
             if not self._project:
-                self.logger.warning("get_project_param called without a project name")
+                self.logger.warning(
+                    "Project cannot be loaded without a project name set in the context"
+                )
                 return None
             if not self._rundb:
                 self.logger.warning(
-                    "cannot retrieve project parameters - MLRun DB is not accessible"
+                    "Cannot retrieve project data - MLRun DB is not accessible"
                 )
                 return None
             self._project_object = self._rundb.get_project(self._project)
         return self._project_object
 
+    def get_project_object(self):
+        """
+        Get the MLRun project object by the project name set in the context.
+
+        :return: The project object or None if it couldn't be retrieved.
+        """
+        return self._load_project_object()
+
     def get_project_param(self, key: str, default=None):
         """get a parameter from the run's project's parameters"""
         if not self._load_project_object():
@@ -493,27 +507,34 @@ class MLClientCtx(object):
             url = key
         if self.in_path and is_relative_path(url):
             url = os.path.join(self._in_path, url)
-        obj = self._data_stores.object(
-            url,
-            key,
-            project=self._project,
-            allow_empty_resources=self._allow_empty_resources,
-        )
-        self._inputs[key] = obj
-        return obj
+        self._inputs[key] = url
 
     def get_input(self, key: str, url: str = ""):
-        """get an input :py:class:`~mlrun.DataItem` object, data objects have methods such as
-        .get(), .download(), .url, .. to access the actual data
+        """
+        Get an input :py:class:`~mlrun.DataItem` object,
+        data objects have methods such as .get(), .download(), .url, .. to access the actual data.
+        Requires access to the data store secrets if configured.
 
-        example::
+        Example::
 
             data = context.get_input("my_data").get()
+
+        :param key:  The key name for the input url entry.
+        :param url:  The url of the input data (file, stream, ..) - optional, saved in the inputs dictionary
+                     if the key is not already present.
+
+        :return:     :py:class:`~mlrun.datastore.base.DataItem` object
         """
         if key not in self._inputs:
-            return self._set_input(key, url)
-        else:
-            return self._inputs[key]
+            self._set_input(key, url)
+
+        url = self._inputs[key]
+        return self._data_stores.object(
+            url,
+            key,
+            project=self._project,
+            allow_empty_resources=self._allow_empty_resources,
+        )
 
     def log_result(self, key: str, value, commit=False):
         """log a scalar result value
@@ -941,7 +962,8 @@ class MLClientCtx(object):
                 "handler": self._handler,
                 "outputs": self._outputs,
                 run_keys.output_path: self.artifact_path,
-                run_keys.inputs: {k: v.artifact_url for k, v in self._inputs.items()},
+                run_keys.inputs: self._inputs,
+                "notifications": self._notifications,
             },
             "status": {
                 "results": self._results,
@@ -977,7 +999,7 @@ class MLClientCtx(object):
             "metadata.annotations": self._annotations,
             "spec.parameters": self._parameters,
             "spec.outputs": self._outputs,
-            "spec.inputs": {k: v.artifact_url for k, v in self._inputs.items()},
+            "spec.inputs": self._inputs,
             "status.results": self._results,
             "status.start_time": to_date_str(self._start_time),
             "status.last_update": to_date_str(self._last_update),

mlrun/feature_store/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.