mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/db/httpdb.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
 # limitations under the License.
 import enum
 import http
+import re
 import tempfile
 import time
 import traceback
@@ -27,11 +28,12 @@ import requests
 import semver
 
 import mlrun
+import mlrun.api.utils.helpers
+import mlrun.common.schemas
+import mlrun.model_monitoring.model_endpoint
 import mlrun.projects
-from mlrun.api import schemas
 from mlrun.errors import MLRunInvalidArgumentError, err_to_str
 
-from ..api.schemas import ModelEndpoint
 from ..artifacts import Artifact
 from ..config import config
 from ..feature_store import FeatureSet, FeatureVector
@@ -98,6 +100,12 @@ class HTTPRunDB(RunDBInterface):
     """
 
     kind = "http"
+    # by default we don't retry on POST request as they are usually not idempotent
+    # here is a list of identified POST requests that are idempotent ('store' vs 'create') and can be retried
+    RETRIABLE_POST_PATHS = [
+        r"\/?projects\/.+\/artifacts\/.+\/.+",
+        r"\/?run\/.+\/.+",
+    ]
 
     def __init__(self, base_url, user="", password="", token=""):
         self.base_url = base_url
@@ -192,13 +200,13 @@ class HTTPRunDB(RunDBInterface):
                 if "Authorization" not in kw.setdefault("headers", {}):
                     kw["headers"].update({"Authorization": "Bearer " + self.token})
 
-        if mlrun.api.schemas.HeaderNames.client_version not in kw.setdefault(
+        if mlrun.common.schemas.HeaderNames.client_version not in kw.setdefault(
             "headers", {}
         ):
             kw["headers"].update(
                 {
-                    mlrun.api.schemas.HeaderNames.client_version: self.client_version,
-                    mlrun.api.schemas.HeaderNames.python_version: self.python_version,
+                    mlrun.common.schemas.HeaderNames.client_version: self.client_version,
+                    mlrun.common.schemas.HeaderNames.python_version: self.python_version,
                 }
             )
 
@@ -210,8 +218,10 @@ class HTTPRunDB(RunDBInterface):
                     if isinstance(dict_[key], enum.Enum):
                         dict_[key] = dict_[key].value
 
-        if not self.session:
-            self.session = self._init_session()
+        # if the method is POST, we need to update the session with the appropriate retry policy
+        if not self.session or method == "POST":
+            retry_on_post = self._is_retry_on_post_allowed(method, path)
+            self.session = self._init_session(retry_on_post)
 
         try:
             response = self.session.request(
@@ -239,16 +249,29 @@ class HTTPRunDB(RunDBInterface):
 
         return response
 
-    def _init_session(self):
+    def _init_session(self, retry_on_post: bool = False):
         return mlrun.utils.HTTPSessionWithRetry(
             retry_on_exception=config.httpdb.retry_api_call_on_exception
-            == mlrun.api.schemas.HTTPSessionRetryMode.enabled.value
+            == mlrun.common.schemas.HTTPSessionRetryMode.enabled.value,
+            retry_on_post=retry_on_post,
         )
 
     def _path_of(self, prefix, project, uid):
         project = project or config.default_project
         return f"{prefix}/{project}/{uid}"
 
+    def _is_retry_on_post_allowed(self, method, path: str):
+        """
+        Check if the given path is allowed to be retried on POST method
+        :param method:  used to verify that the method is POST since if there is no session initialized there is no
+                        need to initialize it with retry policy for POST when the method is not POST
+        :param path:    the path to check
+        :return:        True if the path is allowed to be retried on POST method and method is POST, False otherwise
+        """
+        return method == "POST" and any(
+            re.match(regex, path) for regex in self.RETRIABLE_POST_PATHS
+        )
+
     def connect(self, secrets=None):
         """Connect to the MLRun API server. Must be called prior to executing any other method.
         The code utilizes the URL for the API server from the configuration - ``mlconf.dbpath``.
@@ -499,16 +522,16 @@ class HTTPRunDB(RunDBInterface):
         body = _as_json(struct)
         self.api_call("POST", path, error, params=params, body=body)
 
-    def update_run(self, updates: dict, uid, project="", iter=0):
+    def update_run(self, updates: dict, uid, project="", iter=0, timeout=45):
         """Update the details of a stored run in the DB."""
 
         path = self._path_of("run", project, uid)
         params = {"iter": iter}
         error = f"update run {project}/{uid}"
         body = _as_json(updates)
-        self.api_call("PATCH", path, error, params=params, body=body)
+        self.api_call("PATCH", path, error, params=params, body=body, timeout=timeout)
 
-    def abort_run(self, uid, project="", iter=0):
+    def abort_run(self, uid, project="", iter=0, timeout=45):
         """
         Abort a running run - will remove the run's runtime resources and mark its state as aborted
         """
@@ -517,6 +540,7 @@ class HTTPRunDB(RunDBInterface):
             uid,
             project,
             iter,
+            timeout,
         )
 
     def read_run(self, uid, project="", iter=0):
@@ -548,28 +572,33 @@ class HTTPRunDB(RunDBInterface):
 
     def list_runs(
         self,
-        name=None,
+        name: Optional[str] = None,
         uid: Optional[Union[str, List[str]]] = None,
-        project=None,
-        labels=None,
-        state=None,
-        sort=True,
-        last=0,
-        iter=False,
-        start_time_from: datetime = None,
-        start_time_to: datetime = None,
-        last_update_time_from: datetime = None,
-        last_update_time_to: datetime = None,
-        partition_by: Union[schemas.RunPartitionByField, str] = None,
+        project: Optional[str] = None,
+        labels: Optional[Union[str, List[str]]] = None,
+        state: Optional[str] = None,
+        sort: bool = True,
+        last: int = 0,
+        iter: bool = False,
+        start_time_from: Optional[datetime] = None,
+        start_time_to: Optional[datetime] = None,
+        last_update_time_from: Optional[datetime] = None,
+        last_update_time_to: Optional[datetime] = None,
+        partition_by: Optional[
+            Union[mlrun.common.schemas.RunPartitionByField, str]
+        ] = None,
         rows_per_partition: int = 1,
-        partition_sort_by: Union[schemas.SortField, str] = None,
-        partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
+        partition_sort_by: Optional[Union[mlrun.common.schemas.SortField, str]] = None,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
         max_partitions: int = 0,
+        with_notifications: bool = False,
     ) -> RunList:
         """Retrieve a list of runs, filtered by various options.
         Example::
 
-            runs = db.list_runs(name='download', project='iris', labels='owner=admin')
+            runs = db.list_runs(name='download', project='iris', labels=['owner=admin', 'kind=job'])
             # If running in Jupyter, can use the .show() function to display the results
             db.list_runs(name='', project=project_name).show()
 
@@ -577,8 +606,8 @@ class HTTPRunDB(RunDBInterface):
         :param name: Name of the run to retrieve.
         :param uid: Unique ID of the run, or a list of run UIDs.
         :param project: Project that the runs belongs to.
-        :param labels: List runs that have a specific label assigned. Currently only a single label filter can be
-            applied, otherwise result will be empty.
+        :param labels: List runs that have specific labels assigned. a single or multi label filter can be
+            applied.
         :param state: List only runs whose state is specified.
         :param sort: Whether to sort the result according to their start time. Otherwise, results will be
             returned by their internal order in the DB (order will not be guaranteed).
@@ -598,6 +627,7 @@ class HTTPRunDB(RunDBInterface):
         :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
         :param max_partitions: Maximal number of partitions to include in the result. Default is `0` which means no
             limit.
+        :param with_notifications: Return runs with notifications, and join them to the response. Default is `False`.
         """
 
         project = project or config.default_project
@@ -613,12 +643,13 @@ class HTTPRunDB(RunDBInterface):
             "start_time_to": datetime_to_iso(start_time_to),
             "last_update_time_from": datetime_to_iso(last_update_time_from),
             "last_update_time_to": datetime_to_iso(last_update_time_to),
+            "with-notifications": with_notifications,
         }
 
         if partition_by:
             params.update(
                 self._generate_partition_by_params(
-                    schemas.RunPartitionByField,
+                    mlrun.common.schemas.RunPartitionByField,
                     partition_by,
                     rows_per_partition,
                     partition_sort_by,
@@ -687,7 +718,7 @@ class HTTPRunDB(RunDBInterface):
         endpoint_path = f"projects/{project}/artifacts/{key}?tag={tag}"
         error = f"read artifact {project}/{key}"
         # explicitly set artifacts format to 'full' since old servers may default to 'legacy'
-        params = {"format": schemas.ArtifactsFormat.full.value}
+        params = {"format": mlrun.common.schemas.ArtifactsFormat.full.value}
         if iter:
             params["iter"] = str(iter)
         resp = self.api_call("GET", endpoint_path, error, params=params)
@@ -715,7 +746,7 @@ class HTTPRunDB(RunDBInterface):
         iter: int = None,
         best_iteration: bool = False,
         kind: str = None,
-        category: Union[str, schemas.ArtifactCategories] = None,
+        category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
     ) -> ArtifactList:
         """List artifacts filtered by various parameters.
 
@@ -728,8 +759,9 @@ class HTTPRunDB(RunDBInterface):
             # Show artifacts with label filters - both uploaded and of binary type
             result_labels = db.list_artifacts('results', tag='*', project='iris', labels=['uploaded', 'type=binary'])
 
-        :param name: Name of artifacts to retrieve. Name is used as a like query, and is not case-sensitive. This means
-            that querying for ``name`` may return artifacts named ``my_Name_1`` or ``surname``.
+        :param name: Name of artifacts to retrieve. Name with '~' prefix is used as a like query, and is not
+            case-sensitive. This means that querying for ``~name`` may return artifacts named
+            ``my_Name_1`` or ``surname``.
         :param project: Project name.
         :param tag: Return artifacts assigned this tag.
         :param labels: Return artifacts that have these labels. Labels can either be a dictionary {"label": "value"} or
@@ -759,7 +791,7 @@ class HTTPRunDB(RunDBInterface):
             "best-iteration": best_iteration,
             "kind": kind,
             "category": category,
-            "format": schemas.ArtifactsFormat.full.value,
+            "format": mlrun.common.schemas.ArtifactsFormat.full.value,
         }
         error = "list artifacts"
         endpoint_path = f"projects/{project}/artifacts"
@@ -792,7 +824,7 @@ class HTTPRunDB(RunDBInterface):
     def list_artifact_tags(
         self,
         project=None,
-        category: Union[str, schemas.ArtifactCategories] = None,
+        category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
     ) -> List[str]:
         """Return a list of all the tags assigned to artifacts in the scope of the given project."""
 
@@ -820,7 +852,7 @@ class HTTPRunDB(RunDBInterface):
 
         params = {"tag": tag, "versioned": versioned}
         project = project or config.default_project
-        path = self._path_of("func", project, name)
+        path = f"projects/{project}/functions/{name}"
 
         error = f"store function {project}/{name}"
         resp = self.api_call(
@@ -835,7 +867,7 @@ class HTTPRunDB(RunDBInterface):
 
         params = {"tag": tag, "hash_key": hash_key}
         project = project or config.default_project
-        path = self._path_of("func", project, name)
+        path = f"projects/{project}/functions/{name}"
         error = f"get function {project}/{name}"
         resp = self.api_call("GET", path, error, params=params)
         return resp.json()["func"]
@@ -857,15 +889,15 @@ class HTTPRunDB(RunDBInterface):
         :param labels: Return functions that have specific labels assigned to them.
         :returns: List of function objects (as dictionary).
         """
-
+        project = project or config.default_project
         params = {
-            "project": project or config.default_project,
             "name": name,
             "tag": tag,
             "label": labels or [],
         }
         error = "list functions"
-        resp = self.api_call("GET", "funcs", error, params=params)
+        path = f"projects/{project}/functions"
+        resp = self.api_call("GET", path, error, params=params)
         return resp.json()["funcs"]
 
     def list_runtime_resources(
@@ -874,11 +906,13 @@ class HTTPRunDB(RunDBInterface):
         label_selector: Optional[str] = None,
         kind: Optional[str] = None,
         object_id: Optional[str] = None,
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ) -> Union[
-        mlrun.api.schemas.RuntimeResourcesOutput,
-        mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-        mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+        mlrun.common.schemas.RuntimeResourcesOutput,
+        mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+        mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
     ]:
         """List current runtime resources, which are usually (but not limited to) Kubernetes pods or CRDs.
         Function applies for runs of type `['dask', 'job', 'spark', 'remote-spark', 'mpijob']`, and will return per
@@ -907,25 +941,25 @@ class HTTPRunDB(RunDBInterface):
         )
         if group_by is None:
             structured_list = [
-                mlrun.api.schemas.KindRuntimeResources(**kind_runtime_resources)
+                mlrun.common.schemas.KindRuntimeResources(**kind_runtime_resources)
                 for kind_runtime_resources in response.json()
             ]
             return structured_list
-        elif group_by == mlrun.api.schemas.ListRuntimeResourcesGroupByField.job:
+        elif group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.job:
             structured_dict = {}
             for project, job_runtime_resources_map in response.json().items():
                 for job_id, runtime_resources in job_runtime_resources_map.items():
                     structured_dict.setdefault(project, {})[
                         job_id
-                    ] = mlrun.api.schemas.RuntimeResources(**runtime_resources)
+                    ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
             return structured_dict
-        elif group_by == mlrun.api.schemas.ListRuntimeResourcesGroupByField.project:
+        elif group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.project:
             structured_dict = {}
             for project, kind_runtime_resources_map in response.json().items():
                 for kind, runtime_resources in kind_runtime_resources_map.items():
                     structured_dict.setdefault(project, {})[
                         kind
-                    ] = mlrun.api.schemas.RuntimeResources(**runtime_resources)
+                    ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
             return structured_dict
         else:
             raise NotImplementedError(
@@ -940,7 +974,7 @@ class HTTPRunDB(RunDBInterface):
         object_id: Optional[str] = None,
         force: bool = False,
         grace_period: int = None,
-    ) -> mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput:
+    ) -> mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput:
         """Delete all runtime resources which are in terminal state.
 
         :param project: Delete only runtime resources of a specific project, by default None, which will delete only
@@ -955,7 +989,7 @@ class HTTPRunDB(RunDBInterface):
         :param grace_period: Grace period given to the runtime resource before they are actually removed, counted from
             the moment they moved to terminal state (defaults to mlrun.mlconf.runtime_resources_deletion_grace_period).
 
-        :returns: :py:class:`~mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput` listing the runtime resources
+        :returns: :py:class:`~mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput` listing the runtime resources
             that were removed.
         """
         if grace_period is None:
@@ -985,10 +1019,12 @@ class HTTPRunDB(RunDBInterface):
             for kind, runtime_resources in kind_runtime_resources_map.items():
                 structured_dict.setdefault(project, {})[
                     kind
-                ] = mlrun.api.schemas.RuntimeResources(**runtime_resources)
+                ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
         return structured_dict
 
-    def create_schedule(self, project: str, schedule: schemas.ScheduleInput):
+    def create_schedule(
+        self, project: str, schedule: mlrun.common.schemas.ScheduleInput
+    ):
         """Create a new schedule on the given project. The details on the actual object to schedule as well as the
         schedule itself are within the schedule object provided.
         The :py:class:`~ScheduleCronTrigger` follows the guidelines in
@@ -1000,7 +1036,7 @@ class HTTPRunDB(RunDBInterface):
 
         Example::
 
-            from mlrun.api import schemas
+            from mlrun.common import schemas
 
             # Execute the get_data_func function every Tuesday at 15:30
             schedule = schemas.ScheduleInput(
@@ -1019,7 +1055,7 @@ class HTTPRunDB(RunDBInterface):
         self.api_call("POST", path, error_message, body=dict_to_json(schedule.dict()))
 
     def update_schedule(
-        self, project: str, name: str, schedule: schemas.ScheduleUpdate
+        self, project: str, name: str, schedule: mlrun.common.schemas.ScheduleUpdate
     ):
         """Update an existing schedule, replace it with the details contained in the schedule object."""
 
@@ -1031,7 +1067,7 @@ class HTTPRunDB(RunDBInterface):
 
     def get_schedule(
         self, project: str, name: str, include_last_run: bool = False
-    ) -> schemas.ScheduleOutput:
+    ) -> mlrun.common.schemas.ScheduleOutput:
         """Retrieve details of the schedule in question. Besides returning the details of the schedule object itself,
         this function also returns the next scheduled run for this specific schedule, as well as potentially the
         results of the last run executed through this schedule.
@@ -1047,15 +1083,15 @@ class HTTPRunDB(RunDBInterface):
         resp = self.api_call(
             "GET", path, error_message, params={"include_last_run": include_last_run}
         )
-        return schemas.ScheduleOutput(**resp.json())
+        return mlrun.common.schemas.ScheduleOutput(**resp.json())
 
     def list_schedules(
         self,
         project: str,
         name: str = None,
-        kind: schemas.ScheduleKinds = None,
+        kind: mlrun.common.schemas.ScheduleKinds = None,
         include_last_run: bool = False,
-    ) -> schemas.SchedulesOutput:
+    ) -> mlrun.common.schemas.SchedulesOutput:
         """Retrieve list of schedules of specific name or kind.
 
         :param project: Project name.
@@ -1070,7 +1106,7 @@ class HTTPRunDB(RunDBInterface):
         path = f"projects/{project}/schedules"
         error_message = f"Failed listing schedules for {project} ? {kind} {name}"
         resp = self.api_call("GET", path, error_message, params=params)
-        return schemas.SchedulesOutput(**resp.json())
+        return mlrun.common.schemas.SchedulesOutput(**resp.json())
 
     def delete_schedule(self, project: str, name: str):
         """Delete a specific schedule by name."""
@@ -1133,19 +1169,20 @@ class HTTPRunDB(RunDBInterface):
     def get_builder_status(
         self,
         func: BaseRuntime,
-        offset=0,
-        logs=True,
-        last_log_timestamp=0,
-        verbose=False,
+        offset: int = 0,
+        logs: bool = True,
+        last_log_timestamp: float = 0.0,
+        verbose: bool = False,
     ):
         """Retrieve the status of a build operation currently in progress.
 
-        :param func: Function object that is being built.
-        :param offset: Offset into the build logs to retrieve logs from.
-        :param logs: Should build logs be retrieved.
-        :param last_log_timestamp: Last timestamp of logs that were already retrieved. Function will return only logs
-            later than this parameter.
-        :param verbose: Add verbose logs into the output.
+        :param func:                Function object that is being built.
+        :param offset:              Offset into the build logs to retrieve logs from.
+        :param logs:                Should build logs be retrieved.
+        :param last_log_timestamp:  Last timestamp of logs that were already retrieved. Function will return only logs
+                                    later than this parameter.
+        :param verbose:             Add verbose logs into the output.
+
         :returns: The following parameters:
 
             - Text of builder logs.
@@ -1199,7 +1236,7 @@ class HTTPRunDB(RunDBInterface):
             text = resp.content.decode()
         return text, last_log_timestamp
 
-    def remote_start(self, func_url) -> schemas.BackgroundTask:
+    def remote_start(self, func_url) -> mlrun.common.schemas.BackgroundTask:
         """Execute a function remotely, Used for ``dask`` functions.
 
         :param func_url: URL to the function to be executed.
@@ -1222,13 +1259,13 @@ class HTTPRunDB(RunDBInterface):
             logger.error(f"bad resp!!\n{resp.text}")
             raise ValueError("bad function start response")
 
-        return schemas.BackgroundTask(**resp.json())
+        return mlrun.common.schemas.BackgroundTask(**resp.json())
 
     def get_project_background_task(
         self,
         project: str,
         name: str,
-    ) -> schemas.BackgroundTask:
+    ) -> mlrun.common.schemas.BackgroundTask:
         """Retrieve updated information on a project background task being executed."""
 
         project = project or config.default_project
@@ -1237,15 +1274,15 @@ class HTTPRunDB(RunDBInterface):
             f"Failed getting project background task. project={project}, name={name}"
         )
         response = self.api_call("GET", path, error_message)
-        return schemas.BackgroundTask(**response.json())
+        return mlrun.common.schemas.BackgroundTask(**response.json())
 
-    def get_background_task(self, name: str) -> schemas.BackgroundTask:
+    def get_background_task(self, name: str) -> mlrun.common.schemas.BackgroundTask:
         """Retrieve updated information on a background task being executed."""
 
         path = f"background-tasks/{name}"
         error_message = f"Failed getting background task. name={name}"
         response = self.api_call("GET", path, error_message)
-        return schemas.BackgroundTask(**response.json())
+        return mlrun.common.schemas.BackgroundTask(**response.json())
 
     def remote_status(self, project, name, kind, selector):
         """Retrieve status of a function being executed remotely (relevant to ``dask`` functions).
@@ -1270,7 +1307,9 @@ class HTTPRunDB(RunDBInterface):
         return resp.json()["data"]
 
     def submit_job(
-        self, runspec, schedule: Union[str, schemas.ScheduleCronTrigger] = None
+        self,
+        runspec,
+        schedule: Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
     ):
         """Submit a job for remote execution.
 
@@ -1282,7 +1321,7 @@ class HTTPRunDB(RunDBInterface):
         try:
             req = {"task": runspec.to_dict()}
             if schedule:
-                if isinstance(schedule, schemas.ScheduleCronTrigger):
+                if isinstance(schedule, mlrun.common.schemas.ScheduleCronTrigger):
                     schedule = schedule.dict()
                 req["schedule"] = schedule
             timeout = (int(config.submit_timeout) or 120) + 20
@@ -1364,7 +1403,9 @@ class HTTPRunDB(RunDBInterface):
         if arguments:
             if not isinstance(arguments, dict):
                 raise ValueError("arguments must be dict type")
-            headers[schemas.HeaderNames.pipeline_arguments] = str(arguments)
+            headers[mlrun.common.schemas.HeaderNames.pipeline_arguments] = str(
+                arguments
+            )
 
         if not path.isfile(pipe_file):
             raise OSError(f"file {pipe_file} doesnt exist")
@@ -1403,10 +1444,10 @@ class HTTPRunDB(RunDBInterface):
         page_token: str = "",
         filter_: str = "",
         format_: Union[
-            str, mlrun.api.schemas.PipelinesFormat
-        ] = mlrun.api.schemas.PipelinesFormat.metadata_only,
+            str, mlrun.common.schemas.PipelinesFormat
+        ] = mlrun.common.schemas.PipelinesFormat.metadata_only,
         page_size: int = None,
-    ) -> mlrun.api.schemas.PipelinesOutput:
+    ) -> mlrun.common.schemas.PipelinesOutput:
         """Retrieve a list of KFP pipelines. This function can be invoked to get all pipelines from all projects,
         by specifying ``project=*``, in which case pagination can be used and the various sorting and pagination
         properties can be applied. If a specific project is requested, then the pagination options cannot be
@@ -1425,9 +1466,9 @@ class HTTPRunDB(RunDBInterface):
         :param page_size: Size of a single page when applying pagination.
         """
 
-        if project != "*" and (page_token or page_size or sort_by):
+        if project != "*" and (page_token or page_size):
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "Filtering by project can not be used together with pagination, or sorting"
+                "Filtering by project can not be used together with pagination"
             )
         params = {
             "namespace": namespace,
@@ -1442,7 +1483,7 @@ class HTTPRunDB(RunDBInterface):
         response = self.api_call(
             "GET", f"projects/{project}/pipelines", error_message, params=params
         )
-        return mlrun.api.schemas.PipelinesOutput(**response.json())
+        return mlrun.common.schemas.PipelinesOutput(**response.json())
 
     def get_pipeline(
         self,
@@ -1450,27 +1491,23 @@ class HTTPRunDB(RunDBInterface):
         namespace: str = None,
         timeout: int = 10,
         format_: Union[
-            str, mlrun.api.schemas.PipelinesFormat
-        ] = mlrun.api.schemas.PipelinesFormat.summary,
+            str, mlrun.common.schemas.PipelinesFormat
+        ] = mlrun.common.schemas.PipelinesFormat.summary,
         project: str = None,
     ):
         """Retrieve details of a specific pipeline using its run ID (as provided when the pipeline was executed)."""
 
-        try:
-            params = {}
-            if namespace:
-                params["namespace"] = namespace
-            params["format"] = format_
-            project_path = project if project else "*"
-            resp = self.api_call(
-                "GET",
-                f"projects/{project_path}/pipelines/{run_id}",
-                params=params,
-                timeout=timeout,
-            )
-        except OSError as err:
-            logger.error(f"error cannot get pipeline: {err_to_str(err)}")
-            raise OSError(f"error: cannot get pipeline, {err_to_str(err)}")
+        params = {}
+        if namespace:
+            params["namespace"] = namespace
+        params["format"] = format_
+        project_path = project if project else "*"
+        resp = self.api_call(
+            "GET",
+            f"projects/{project_path}/pipelines/{run_id}",
+            params=params,
+            timeout=timeout,
+        )
 
         if not resp.ok:
             logger.error(f"bad resp!!\n{resp.text}")
@@ -1486,7 +1523,7 @@ class HTTPRunDB(RunDBInterface):
 
     def create_feature_set(
         self,
-        feature_set: Union[dict, schemas.FeatureSet, FeatureSet],
+        feature_set: Union[dict, mlrun.common.schemas.FeatureSet, FeatureSet],
         project="",
         versioned=True,
     ) -> dict:
@@ -1499,7 +1536,7 @@ class HTTPRunDB(RunDBInterface):
             will be kept in the DB and can be retrieved until explicitly deleted.
         :returns: The :py:class:`~mlrun.feature_store.FeatureSet` object (as dict).
         """
-        if isinstance(feature_set, schemas.FeatureSet):
+        if isinstance(feature_set, mlrun.common.schemas.FeatureSet):
             feature_set = feature_set.dict()
         elif isinstance(feature_set, FeatureSet):
             feature_set = feature_set.to_dict()
@@ -1633,10 +1670,14 @@ class HTTPRunDB(RunDBInterface):
         entities: List[str] = None,
         features: List[str] = None,
         labels: List[str] = None,
-        partition_by: Union[schemas.FeatureStorePartitionByField, str] = None,
+        partition_by: Union[
+            mlrun.common.schemas.FeatureStorePartitionByField, str
+        ] = None,
         rows_per_partition: int = 1,
-        partition_sort_by: Union[schemas.SortField, str] = None,
-        partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
+        partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
     ) -> List[FeatureSet]:
         """Retrieve a list of feature-sets matching the criteria provided.
 
@@ -1670,7 +1711,7 @@ class HTTPRunDB(RunDBInterface):
         if partition_by:
             params.update(
                 self._generate_partition_by_params(
-                    schemas.FeatureStorePartitionByField,
+                    mlrun.common.schemas.FeatureStorePartitionByField,
                     partition_by,
                     rows_per_partition,
                     partition_sort_by,
@@ -1690,7 +1731,7 @@ class HTTPRunDB(RunDBInterface):
 
     def store_feature_set(
         self,
-        feature_set: Union[dict, schemas.FeatureSet, FeatureSet],
+        feature_set: Union[dict, mlrun.common.schemas.FeatureSet, FeatureSet],
         name=None,
         project="",
         tag=None,
@@ -1715,7 +1756,7 @@ class HTTPRunDB(RunDBInterface):
         reference = self._resolve_reference(tag, uid)
         params = {"versioned": versioned}
 
-        if isinstance(feature_set, schemas.FeatureSet):
+        if isinstance(feature_set, mlrun.common.schemas.FeatureSet):
             feature_set = feature_set.dict()
         elif isinstance(feature_set, FeatureSet):
             feature_set = feature_set.to_dict()
@@ -1738,7 +1779,9 @@ class HTTPRunDB(RunDBInterface):
         project="",
         tag=None,
         uid=None,
-        patch_mode: Union[str, schemas.PatchMode] = schemas.PatchMode.replace,
+        patch_mode: Union[
+            str, mlrun.common.schemas.PatchMode
+        ] = mlrun.common.schemas.PatchMode.replace,
     ):
         """Modify (patch) an existing :py:class:`~mlrun.feature_store.FeatureSet` object.
         The object is identified by its name (and project it belongs to), as well as optionally a ``tag`` or its
@@ -1761,7 +1804,7 @@ class HTTPRunDB(RunDBInterface):
         """
         project = project or config.default_project
         reference = self._resolve_reference(tag, uid)
-        headers = {schemas.HeaderNames.patch_mode: patch_mode}
+        headers = {mlrun.common.schemas.HeaderNames.patch_mode: patch_mode}
         path = f"projects/{project}/feature-sets/{name}/references/{reference}"
         error_message = f"Failed updating feature-set {project}/{name}"
         self.api_call(
@@ -1790,7 +1833,7 @@ class HTTPRunDB(RunDBInterface):
 
     def create_feature_vector(
         self,
-        feature_vector: Union[dict, schemas.FeatureVector, FeatureVector],
+        feature_vector: Union[dict, mlrun.common.schemas.FeatureVector, FeatureVector],
         project="",
         versioned=True,
     ) -> dict:
@@ -1802,7 +1845,7 @@ class HTTPRunDB(RunDBInterface):
             will be kept in the DB and can be retrieved until explicitly deleted.
         :returns: The :py:class:`~mlrun.feature_store.FeatureVector` object (as dict).
         """
-        if isinstance(feature_vector, schemas.FeatureVector):
+        if isinstance(feature_vector, mlrun.common.schemas.FeatureVector):
             feature_vector = feature_vector.dict()
         elif isinstance(feature_vector, FeatureVector):
             feature_vector = feature_vector.to_dict()
@@ -1846,10 +1889,14 @@ class HTTPRunDB(RunDBInterface):
         tag: str = None,
         state: str = None,
         labels: List[str] = None,
-        partition_by: Union[schemas.FeatureStorePartitionByField, str] = None,
+        partition_by: Union[
+            mlrun.common.schemas.FeatureStorePartitionByField, str
+        ] = None,
         rows_per_partition: int = 1,
-        partition_sort_by: Union[schemas.SortField, str] = None,
-        partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
+        partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
     ) -> List[FeatureVector]:
         """Retrieve a list of feature-vectors matching the criteria provided.
 
@@ -1879,7 +1926,7 @@ class HTTPRunDB(RunDBInterface):
         if partition_by:
             params.update(
                 self._generate_partition_by_params(
-                    schemas.FeatureStorePartitionByField,
+                    mlrun.common.schemas.FeatureStorePartitionByField,
                     partition_by,
                     rows_per_partition,
                     partition_sort_by,
@@ -1899,7 +1946,7 @@ class HTTPRunDB(RunDBInterface):
 
     def store_feature_vector(
         self,
-        feature_vector: Union[dict, schemas.FeatureVector, FeatureVector],
+        feature_vector: Union[dict, mlrun.common.schemas.FeatureVector, FeatureVector],
         name=None,
         project="",
         tag=None,
@@ -1924,7 +1971,7 @@ class HTTPRunDB(RunDBInterface):
         reference = self._resolve_reference(tag, uid)
         params = {"versioned": versioned}
 
-        if isinstance(feature_vector, schemas.FeatureVector):
+        if isinstance(feature_vector, mlrun.common.schemas.FeatureVector):
             feature_vector = feature_vector.dict()
         elif isinstance(feature_vector, FeatureVector):
             feature_vector = feature_vector.to_dict()
@@ -1949,7 +1996,9 @@ class HTTPRunDB(RunDBInterface):
         project="",
         tag=None,
         uid=None,
-        patch_mode: Union[str, schemas.PatchMode] = schemas.PatchMode.replace,
+        patch_mode: Union[
+            str, mlrun.common.schemas.PatchMode
+        ] = mlrun.common.schemas.PatchMode.replace,
     ):
         """Modify (patch) an existing :py:class:`~mlrun.feature_store.FeatureVector` object.
         The object is identified by its name (and project it belongs to), as well as optionally a ``tag`` or its
@@ -1967,7 +2016,7 @@ class HTTPRunDB(RunDBInterface):
         """
         reference = self._resolve_reference(tag, uid)
         project = project or config.default_project
-        headers = {schemas.HeaderNames.patch_mode: patch_mode}
+        headers = {mlrun.common.schemas.HeaderNames.patch_mode: patch_mode}
         path = f"projects/{project}/feature-vectors/{name}/references/{reference}"
         error_message = f"Failed updating feature-vector {project}/{name}"
         self.api_call(
@@ -1997,7 +2046,7 @@ class HTTPRunDB(RunDBInterface):
         self,
         project: str,
         tag_name: str,
-        objects: Union[mlrun.api.schemas.TagObjects, dict],
+        objects: Union[mlrun.common.schemas.TagObjects, dict],
         replace: bool = False,
     ):
         """Tag a list of objects.
@@ -2017,7 +2066,7 @@ class HTTPRunDB(RunDBInterface):
             error_message,
             body=dict_to_json(
                 objects.dict()
-                if isinstance(objects, mlrun.api.schemas.TagObjects)
+                if isinstance(objects, mlrun.common.schemas.TagObjects)
                 else objects
             ),
         )
@@ -2026,7 +2075,7 @@ class HTTPRunDB(RunDBInterface):
         self,
         project: str,
         tag_name: str,
-        tag_objects: Union[mlrun.api.schemas.TagObjects, dict],
+        tag_objects: Union[mlrun.common.schemas.TagObjects, dict],
     ):
         """Delete a tag from a list of objects.
 
@@ -2043,7 +2092,7 @@ class HTTPRunDB(RunDBInterface):
             error_message,
             body=dict_to_json(
                 tag_objects.dict()
-                if isinstance(tag_objects, mlrun.api.schemas.TagObjects)
+                if isinstance(tag_objects, mlrun.common.schemas.TagObjects)
                 else tag_objects
             ),
         )
@@ -2086,10 +2135,10 @@ class HTTPRunDB(RunDBInterface):
         self,
         owner: str = None,
         format_: Union[
-            str, mlrun.api.schemas.ProjectsFormat
-        ] = mlrun.api.schemas.ProjectsFormat.full,
+            str, mlrun.common.schemas.ProjectsFormat
+        ] = mlrun.common.schemas.ProjectsFormat.full,
         labels: List[str] = None,
-        state: Union[str, mlrun.api.schemas.ProjectState] = None,
+        state: Union[str, mlrun.common.schemas.ProjectState] = None,
     ) -> List[Union[mlrun.projects.MlrunProject, str]]:
         """Return a list of the existing projects, potentially filtered by specific criteria.
 
@@ -2113,20 +2162,17 @@ class HTTPRunDB(RunDBInterface):
 
         error_message = f"Failed listing projects, query: {params}"
         response = self.api_call("GET", "projects", error_message, params=params)
-        if format_ == mlrun.api.schemas.ProjectsFormat.name_only:
+        if format_ == mlrun.common.schemas.ProjectsFormat.name_only:
+
+            # projects is just a list of strings
             return response.json()["projects"]
-        elif format_ in [
-            mlrun.api.schemas.ProjectsFormat.full,
-            mlrun.api.schemas.ProjectsFormat.minimal,
-        ]:
-            return [
-                mlrun.projects.MlrunProject.from_dict(project_dict)
-                for project_dict in response.json()["projects"]
-            ]
-        else:
-            raise NotImplementedError(
-                f"Provided format is not supported. format={format_}"
-            )
+
+        # forwards compatibility - we want to be able to handle new formats that might be added in the future
+        # if format is not known to the api, it is up to the server to return either an error or a default format
+        return [
+            mlrun.projects.MlrunProject.from_dict(project_dict)
+            for project_dict in response.json()["projects"]
+        ]
 
     def get_project(self, name: str) -> mlrun.projects.MlrunProject:
         """Get details for a specific project."""
@@ -2143,8 +2189,8 @@ class HTTPRunDB(RunDBInterface):
         self,
         name: str,
         deletion_strategy: Union[
-            str, mlrun.api.schemas.DeletionStrategy
-        ] = mlrun.api.schemas.DeletionStrategy.default(),
+            str, mlrun.common.schemas.DeletionStrategy
+        ] = mlrun.common.schemas.DeletionStrategy.default(),
     ):
         """Delete a project.
 
@@ -2157,7 +2203,9 @@ class HTTPRunDB(RunDBInterface):
         """
 
         path = f"projects/{name}"
-        headers = {schemas.HeaderNames.deletion_strategy: deletion_strategy}
+        headers = {
+            mlrun.common.schemas.HeaderNames.deletion_strategy: deletion_strategy
+        }
         error_message = f"Failed deleting project {name}"
         response = self.api_call("DELETE", path, error_message, headers=headers)
         if response.status_code == http.HTTPStatus.ACCEPTED:
@@ -2166,13 +2214,13 @@ class HTTPRunDB(RunDBInterface):
     def store_project(
         self,
         name: str,
-        project: Union[dict, mlrun.projects.MlrunProject, mlrun.api.schemas.Project],
+        project: Union[dict, mlrun.projects.MlrunProject, mlrun.common.schemas.Project],
     ) -> mlrun.projects.MlrunProject:
         """Store a project in the DB. This operation will overwrite existing project of the same name if exists."""
 
         path = f"projects/{name}"
         error_message = f"Failed storing project {name}"
-        if isinstance(project, mlrun.api.schemas.Project):
+        if isinstance(project, mlrun.common.schemas.Project):
             project = project.dict()
         elif isinstance(project, mlrun.projects.MlrunProject):
             project = project.to_dict()
@@ -2190,7 +2238,9 @@ class HTTPRunDB(RunDBInterface):
         self,
         name: str,
         project: dict,
-        patch_mode: Union[str, schemas.PatchMode] = schemas.PatchMode.replace,
+        patch_mode: Union[
+            str, mlrun.common.schemas.PatchMode
+        ] = mlrun.common.schemas.PatchMode.replace,
     ) -> mlrun.projects.MlrunProject:
         """Patch an existing project object.
 
@@ -2201,7 +2251,7 @@ class HTTPRunDB(RunDBInterface):
         """
 
         path = f"projects/{name}"
-        headers = {schemas.HeaderNames.patch_mode: patch_mode}
+        headers = {mlrun.common.schemas.HeaderNames.patch_mode: patch_mode}
         error_message = f"Failed patching project {name}"
         response = self.api_call(
             "PATCH", path, error_message, body=dict_to_json(project), headers=headers
@@ -2210,11 +2260,11 @@ class HTTPRunDB(RunDBInterface):
 
     def create_project(
         self,
-        project: Union[dict, mlrun.projects.MlrunProject, mlrun.api.schemas.Project],
+        project: Union[dict, mlrun.projects.MlrunProject, mlrun.common.schemas.Project],
     ) -> mlrun.projects.MlrunProject:
         """Create a new project. A project with the same name must not exist prior to creation."""
 
-        if isinstance(project, mlrun.api.schemas.Project):
+        if isinstance(project, mlrun.common.schemas.Project):
             project = project.dict()
         elif isinstance(project, mlrun.projects.MlrunProject):
             project = project.to_dict()
@@ -2237,7 +2287,7 @@ class HTTPRunDB(RunDBInterface):
             project = self.get_project(project_name)
             if (
                 project.status.state
-                not in mlrun.api.schemas.ProjectState.terminal_states()
+                not in mlrun.common.schemas.ProjectState.terminal_states()
             ):
                 raise Exception(
                     f"Project not in terminal state. State: {project.status.state}"
@@ -2254,11 +2304,11 @@ class HTTPRunDB(RunDBInterface):
 
     def _wait_for_background_task_to_reach_terminal_state(
         self, name: str
-    ) -> schemas.BackgroundTask:
+    ) -> mlrun.common.schemas.BackgroundTask:
         def _verify_background_task_in_terminal_state():
             background_task = self.get_background_task(name)
             state = background_task.status.state
-            if state not in mlrun.api.schemas.BackgroundTaskState.terminal_states():
+            if state not in mlrun.common.schemas.BackgroundTaskState.terminal_states():
                 raise Exception(
                     f"Background task not in terminal state. name={name}, state={state}"
                 )
@@ -2275,10 +2325,10 @@ class HTTPRunDB(RunDBInterface):
     def _wait_for_project_to_be_deleted(self, project_name: str):
         def _verify_project_deleted():
             projects = self.list_projects(
-                format_=mlrun.api.schemas.ProjectsFormat.name_only
+                format_=mlrun.common.schemas.ProjectsFormat.name_only
             )
             if project_name in projects:
-                raise Exception("Project still exists")
+                raise Exception(f"Project {project_name} still exists")
 
         return mlrun.utils.helpers.retry_until_successful(
             self._wait_for_project_deletion_interval,
@@ -2292,8 +2342,8 @@ class HTTPRunDB(RunDBInterface):
         self,
         project: str,
         provider: Union[
-            str, schemas.SecretProviderName
-        ] = schemas.SecretProviderName.kubernetes,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.kubernetes,
         secrets: dict = None,
     ):
         """Create project-context secrets using either ``vault`` or ``kubernetes`` provider.
@@ -2311,19 +2361,21 @@ class HTTPRunDB(RunDBInterface):
 
         :param project: The project context for which to generate the infra and store secrets.
         :param provider: The name of the secrets-provider to work with. Accepts a
-            :py:class:`~mlrun.api.schemas.secret.SecretProviderName` enum.
+            :py:class:`~mlrun.common.schemas.secret.SecretProviderName` enum.
         :param secrets: A set of secret values to store.
             Example::
 
                 secrets = {'password': 'myPassw0rd', 'aws_key': '111222333'}
                 db.create_project_secrets(
                     "project1",
-                    provider=mlrun.api.schemas.SecretProviderName.kubernetes,
+                    provider=mlrun.common.schemas.SecretProviderName.kubernetes,
                     secrets=secrets
                 )
         """
         path = f"projects/{project}/secrets"
-        secrets_input = schemas.SecretsData(secrets=secrets, provider=provider)
+        secrets_input = mlrun.common.schemas.SecretsData(
+            secrets=secrets, provider=provider
+        )
         body = secrets_input.dict()
         error_message = f"Failed creating secret provider {project}/{provider}"
         self.api_call(
@@ -2338,10 +2390,10 @@ class HTTPRunDB(RunDBInterface):
         project: str,
         token: str = None,
         provider: Union[
-            str, schemas.SecretProviderName
-        ] = schemas.SecretProviderName.kubernetes,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.kubernetes,
         secrets: List[str] = None,
-    ) -> schemas.SecretsData:
+    ) -> mlrun.common.schemas.SecretsData:
         """Retrieve project-context secrets from Vault.
 
         Note:
@@ -2356,14 +2408,17 @@ class HTTPRunDB(RunDBInterface):
             to this specific project. ``kubernetes`` provider only supports an empty list.
         """
 
-        if provider == schemas.SecretProviderName.vault.value and not token:
+        if (
+            provider == mlrun.common.schemas.SecretProviderName.vault.value
+            and not token
+        ):
             raise MLRunInvalidArgumentError(
                 "A vault token must be provided when accessing vault secrets"
             )
 
         path = f"projects/{project}/secrets"
         params = {"provider": provider, "secret": secrets}
-        headers = {schemas.HeaderNames.secret_store_token: token}
+        headers = {mlrun.common.schemas.HeaderNames.secret_store_token: token}
         error_message = f"Failed retrieving secrets {project}/{provider}"
         result = self.api_call(
             "GET",
@@ -2372,16 +2427,16 @@ class HTTPRunDB(RunDBInterface):
             params=params,
             headers=headers,
         )
-        return schemas.SecretsData(**result.json())
+        return mlrun.common.schemas.SecretsData(**result.json())
 
     def list_project_secret_keys(
         self,
         project: str,
         provider: Union[
-            str, schemas.SecretProviderName
-        ] = schemas.SecretProviderName.kubernetes,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.kubernetes,
         token: str = None,
-    ) -> schemas.SecretKeysData:
+    ) -> mlrun.common.schemas.SecretKeysData:
         """Retrieve project-context secret keys from Vault or Kubernetes.
 
         Note:
@@ -2390,12 +2445,15 @@ class HTTPRunDB(RunDBInterface):
 
         :param project: The project name.
         :param provider: The name of the secrets-provider to work with. Accepts a
-            :py:class:`~mlrun.api.schemas.secret.SecretProviderName` enum.
+            :py:class:`~mlrun.common.schemas.secret.SecretProviderName` enum.
         :param token: Vault token to use for retrieving secrets. Only in use if ``provider`` is ``vault``.
             Must be a valid Vault token, with permissions to retrieve secrets of the project in question.
         """
 
-        if provider == schemas.SecretProviderName.vault.value and not token:
+        if (
+            provider == mlrun.common.schemas.SecretProviderName.vault.value
+            and not token
+        ):
             raise MLRunInvalidArgumentError(
                 "A vault token must be provided when accessing vault secrets"
             )
@@ -2403,8 +2461,8 @@ class HTTPRunDB(RunDBInterface):
         path = f"projects/{project}/secret-keys"
         params = {"provider": provider}
         headers = (
-            {schemas.HeaderNames.secret_store_token: token}
-            if provider == schemas.SecretProviderName.vault.value
+            {mlrun.common.schemas.HeaderNames.secret_store_token: token}
+            if provider == mlrun.common.schemas.SecretProviderName.vault.value
             else None
         )
         error_message = f"Failed retrieving secret keys {project}/{provider}"
@@ -2415,14 +2473,14 @@ class HTTPRunDB(RunDBInterface):
             params=params,
             headers=headers,
         )
-        return schemas.SecretKeysData(**result.json())
+        return mlrun.common.schemas.SecretKeysData(**result.json())
 
     def delete_project_secrets(
         self,
         project: str,
         provider: Union[
-            str, schemas.SecretProviderName
-        ] = schemas.SecretProviderName.kubernetes,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.kubernetes,
         secrets: List[str] = None,
     ):
         """Delete project-context secrets from Kubernetes.
@@ -2447,8 +2505,8 @@ class HTTPRunDB(RunDBInterface):
         self,
         user: str,
         provider: Union[
-            str, schemas.SecretProviderName
-        ] = schemas.SecretProviderName.vault,
+            str, mlrun.common.schemas.SecretProviderName
+        ] = mlrun.common.schemas.SecretProviderName.vault,
         secrets: dict = None,
     ):
         """Create user-context secret in Vault. Please refer to :py:func:`create_project_secrets` for more details
@@ -2463,7 +2521,7 @@ class HTTPRunDB(RunDBInterface):
         :param secrets: A set of secret values to store within the Vault.
         """
         path = "user-secrets"
-        secrets_creation_request = schemas.UserSecretCreationRequest(
+        secrets_creation_request = mlrun.common.schemas.UserSecretCreationRequest(
             user=user,
             provider=provider,
             secrets=secrets,
@@ -2533,7 +2591,9 @@ class HTTPRunDB(RunDBInterface):
         self,
         project: str,
         endpoint_id: str,
-        model_endpoint: ModelEndpoint,
+        model_endpoint: Union[
+            mlrun.model_monitoring.model_endpoint.ModelEndpoint, dict
+        ],
     ):
         """
         Creates a DB record with the given model_endpoint record.
@@ -2543,11 +2603,16 @@ class HTTPRunDB(RunDBInterface):
         :param model_endpoint: An object representing the model endpoint.
         """
 
+        if isinstance(
+            model_endpoint, mlrun.model_monitoring.model_endpoint.ModelEndpoint
+        ):
+            model_endpoint = model_endpoint.to_dict()
+
         path = f"projects/{project}/model-endpoints/{endpoint_id}"
         self.api_call(
             method="POST",
             path=path,
-            body=model_endpoint.json(),
+            body=dict_to_json(model_endpoint),
         )
 
     def delete_model_endpoint(
@@ -2556,7 +2621,7 @@ class HTTPRunDB(RunDBInterface):
         endpoint_id: str,
     ):
         """
-        Deletes the KV record of a given model endpoint, project and endpoint_id are used for lookup
+        Deletes the DB record of a given model endpoint, project and endpoint_id are used for lookup
 
         :param project: The name of the project
         :param endpoint_id: The id of the endpoint
@@ -2579,13 +2644,15 @@ class HTTPRunDB(RunDBInterface):
         metrics: Optional[List[str]] = None,
         top_level: bool = False,
         uids: Optional[List[str]] = None,
-    ) -> schemas.ModelEndpointList:
+    ) -> List[mlrun.model_monitoring.model_endpoint.ModelEndpoint]:
         """
-        Returns a list of ModelEndpointState objects. Each object represents the current state of a model endpoint.
-        This functions supports filtering by the following parameters:
+        Returns a list of `ModelEndpoint` objects. Each `ModelEndpoint` object represents the current state of a
+        model endpoint. This functions supports filtering by the following parameters:
         1) model
         2) function
         3) labels
+        4) top level
+        5) uids
         By default, when no filters are applied, all available endpoints for the given project will be listed.
 
         In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
@@ -2595,8 +2662,8 @@ class HTTPRunDB(RunDBInterface):
         :param project: The name of the project
         :param model: The name of the model to filter by
         :param function: The name of the function to filter by
-        :param labels: A list of labels to filter by. Label filters work by either filtering a specific value of a label
-            (i.e. list("key==value")) or by looking for the existence of a given key (i.e. "key")
+        :param labels: A list of labels to filter by. Label filters work by either filtering a specific value of a
+         label (i.e. list("key=value")) or by looking for the existence of a given key (i.e. "key")
         :param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
         :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339
                                  time, a Unix timestamp in milliseconds, a relative time (`'now'` or
@@ -2607,10 +2674,14 @@ class HTTPRunDB(RunDBInterface):
                                  `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` =
                                  days), or 0 for the earliest time.
         :param top_level: if true will return only routers and endpoint that are NOT children of any router
-        :param uids: if passed will return ModelEndpointList of endpoints with uid in uids
+        :param uids: if passed will return a list `ModelEndpoint` object with uid in uids
         """
 
         path = f"projects/{project}/model-endpoints"
+
+        if labels and isinstance(labels, dict):
+            labels = [f"{key}={value}" for key, value in labels.items()]
+
         response = self.api_call(
             method="GET",
             path=path,
@@ -2625,7 +2696,15 @@ class HTTPRunDB(RunDBInterface):
                 "uid": uids,
             },
         )
-        return schemas.ModelEndpointList(**response.json())
+
+        # Generate a list of a model endpoint dictionaries
+        model_endpoints = response.json()["endpoints"]
+        if model_endpoints:
+            return [
+                mlrun.model_monitoring.model_endpoint.ModelEndpoint.from_dict(obj)
+                for obj in model_endpoints
+            ]
+        return []
 
     def get_model_endpoint(
         self,
@@ -2635,21 +2714,29 @@ class HTTPRunDB(RunDBInterface):
         end: Optional[str] = None,
         metrics: Optional[List[str]] = None,
         feature_analysis: bool = False,
-    ) -> schemas.ModelEndpoint:
-        """
-        Returns a ModelEndpoint object with additional metrics and feature related data.
-
-        :param project: The name of the project
-        :param endpoint_id: The id of the model endpoint
-        :param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
-        :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339
-                      time, a Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`,
-                      where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the earliest time.
-        :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339
-                    time, a Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`,
-                    where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the earliest time.
-        :param feature_analysis: When True, the base feature statistics and current feature statistics will be added to
-            the output of the resulting object
+    ) -> mlrun.model_monitoring.model_endpoint.ModelEndpoint:
+        """
+        Returns a single `ModelEndpoint` object with additional metrics and feature related data.
+
+        :param project:                    The name of the project
+        :param endpoint_id:                The unique id of the model endpoint.
+        :param start:                      The start time of the metrics. Can be represented by a string containing an
+                                           RFC 3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                           `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or
+                                           0 for the earliest time.
+        :param end:                        The end time of the metrics. Can be represented by a string containing an
+                                           RFC 3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                           `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or
+                                           0 for the earliest time.
+        :param metrics:                    A list of metrics to return for the model endpoint. There are pre-defined
+                                           metrics for model endpoints such as predictions_per_second and
+                                           latency_avg_5m but also custom metrics defined by the user. Please note that
+                                           these metrics are stored in the time series DB and the results will be
+                                           appeared under model_endpoint.spec.metrics.
+        :param feature_analysis:           When True, the base feature statistics and current feature statistics will
+                                           be added to the output of the resulting object.
+
+        :return: A `ModelEndpoint` object.
         """
 
         path = f"projects/{project}/model-endpoints/{endpoint_id}"
@@ -2663,7 +2750,10 @@ class HTTPRunDB(RunDBInterface):
                 "feature_analysis": feature_analysis,
             },
         )
-        return schemas.ModelEndpoint(**response.json())
+
+        return mlrun.model_monitoring.model_endpoint.ModelEndpoint.from_dict(
+            response.json()
+        )
 
     def patch_model_endpoint(
         self,
@@ -2677,10 +2767,10 @@ class HTTPRunDB(RunDBInterface):
         :param project: The name of the project.
         :param endpoint_id: The id of the endpoint.
         :param attributes: Dictionary of attributes that will be used for update the model endpoint. The keys
-                           of this dictionary should exist in the target table. The values should be
-                           from type string or from a valid numerical type such as int or float. More details
-                           about the model endpoint available attributes can be found under
-                           :py:class:`~mlrun.api.schemas.ModelEndpoint`.
+                           of this dictionary should exist in the target table. Note that the values should be
+                           from type string or from a valid numerical type such as int or float.
+                            More details about the model endpoint available attributes can be found under
+                           :py:class:`~mlrun.common.schemas.ModelEndpoint`.
 
                            Example::
 
@@ -2708,18 +2798,18 @@ class HTTPRunDB(RunDBInterface):
             params=attributes,
         )
 
-    def create_marketplace_source(
-        self, source: Union[dict, schemas.IndexedMarketplaceSource]
+    def create_hub_source(
+        self, source: Union[dict, mlrun.common.schemas.IndexedHubSource]
     ):
         """
-        Add a new marketplace source.
+        Add a new hub source.
 
-        MLRun maintains an ordered list of marketplace sources (“sources”) Each source has
+        MLRun maintains an ordered list of hub sources (“sources”) Each source has
         its details registered and its order within the list. When creating a new source, the special order ``-1``
         can be used to mark this source as last in the list. However, once the source is in the MLRun list,
         its order will always be ``>0``.
 
-        The global marketplace source always exists in the list, and is always the last source
+        The global hub source always exists in the list, and is always the last source
         (``order = -1``). It cannot be modified nor can it be moved to another order in the list.
 
         The source object may contain credentials which are needed to access the datastore where the source is stored.
@@ -2728,49 +2818,55 @@ class HTTPRunDB(RunDBInterface):
 
         Example::
 
-            import mlrun.api.schemas
+            import mlrun.common.schemas
 
             # Add a private source as the last one (will be #1 in the list)
-            private_source = mlrun.api.schemas.IndexedMarketplaceSource(
+            private_source = mlrun.common.schemas.IndexedHubeSource(
                 order=-1,
-                source=mlrun.api.schemas.MarketplaceSource(
-                    metadata=mlrun.api.schemas.MarketplaceObjectMetadata(name="priv", description="a private source"),
-                    spec=mlrun.api.schemas.MarketplaceSourceSpec(path="/local/path/to/source", channel="development")
+                source=mlrun.common.schemas.HubSource(
+                    metadata=mlrun.common.schemas.HubObjectMetadata(
+                        name="priv", description="a private source"
+                    ),
+                    spec=mlrun.common.schemas.HubSourceSpec(path="/local/path/to/source", channel="development")
                 )
             )
-            db.create_marketplace_source(private_source)
+            db.create_hub_source(private_source)
 
             # Add another source as 1st in the list - will push previous one to be #2
-            another_source = mlrun.api.schemas.IndexedMarketplaceSource(
+            another_source = mlrun.common.schemas.IndexedHubSource(
                 order=1,
-                source=mlrun.api.schemas.MarketplaceSource(
-                    metadata=mlrun.api.schemas.MarketplaceObjectMetadata(name="priv-2", description="another source"),
-                    spec=mlrun.api.schemas.MarketplaceSourceSpec(
+                source=mlrun.common.schemas.HubSource(
+                    metadata=mlrun.common.schemas.HubObjectMetadata(
+                        name="priv-2", description="another source"
+                    ),
+                    spec=mlrun.common.schemas.HubSourceSpec(
                         path="/local/path/to/source/2",
                         channel="development",
                         credentials={...}
                     )
                 )
             )
-            db.create_marketplace_source(another_source)
+            db.create_hub_source(another_source)
 
         :param source: The source and its order, of type
-            :py:class:`~mlrun.api.schemas.marketplace.IndexedMarketplaceSource`, or in dictionary form.
+            :py:class:`~mlrun.common.schemas.hub.IndexedHubSource`, or in dictionary form.
         :returns: The source object as inserted into the database, with credentials stripped.
         """
-        path = "marketplace/sources"
-        if isinstance(source, schemas.IndexedMarketplaceSource):
+        path = "hub/sources"
+        if isinstance(source, mlrun.common.schemas.IndexedHubSource):
             source = source.dict()
         response = self.api_call(method="POST", path=path, json=source)
-        return schemas.IndexedMarketplaceSource(**response.json())
+        return mlrun.common.schemas.IndexedHubSource(**response.json())
 
-    def store_marketplace_source(
-        self, source_name: str, source: Union[dict, schemas.IndexedMarketplaceSource]
+    def store_hub_source(
+        self,
+        source_name: str,
+        source: Union[dict, mlrun.common.schemas.IndexedHubSource],
     ):
         """
-        Create or replace a marketplace source.
+        Create or replace a hub source.
         For an example of the source format and explanation of the source order logic,
-        please see :py:func:`~create_marketplace_source`. This method can be used to modify the source itself or its
+        please see :py:func:`~create_hub_source`. This method can be used to modify the source itself or its
         order in the list of sources.
 
         :param source_name: Name of the source object to modify/create. It must match the ``source.metadata.name``
@@ -2778,118 +2874,140 @@ class HTTPRunDB(RunDBInterface):
         :param source: Source object to store in the database.
         :returns: The source object as stored in the DB.
         """
-        path = f"marketplace/sources/{source_name}"
-        if isinstance(source, schemas.IndexedMarketplaceSource):
+        path = f"hub/sources/{source_name}"
+        if isinstance(source, mlrun.common.schemas.IndexedHubSource):
             source = source.dict()
 
         response = self.api_call(method="PUT", path=path, json=source)
-        return schemas.IndexedMarketplaceSource(**response.json())
+        return mlrun.common.schemas.IndexedHubSource(**response.json())
 
-    def list_marketplace_sources(self):
+    def list_hub_sources(self):
         """
-        List marketplace sources in the MLRun DB.
+        List hub sources in the MLRun DB.
         """
-        path = "marketplace/sources"
+        path = "hub/sources"
         response = self.api_call(method="GET", path=path).json()
         results = []
         for item in response:
-            results.append(schemas.IndexedMarketplaceSource(**item))
+            results.append(mlrun.common.schemas.IndexedHubSource(**item))
         return results
 
-    def get_marketplace_source(self, source_name: str):
+    def get_hub_source(self, source_name: str):
         """
-        Retrieve a marketplace source from the DB.
+        Retrieve a hub source from the DB.
 
-        :param source_name: Name of the marketplace source to retrieve.
+        :param source_name: Name of the hub source to retrieve.
         """
-        path = f"marketplace/sources/{source_name}"
+        path = f"hub/sources/{source_name}"
         response = self.api_call(method="GET", path=path)
-        return schemas.IndexedMarketplaceSource(**response.json())
+        return mlrun.common.schemas.IndexedHubSource(**response.json())
 
-    def delete_marketplace_source(self, source_name: str):
+    def delete_hub_source(self, source_name: str):
         """
-        Delete a marketplace source from the DB.
+        Delete a hub source from the DB.
         The source will be deleted from the list, and any following sources will be promoted - for example, if the
         1st source is deleted, the 2nd source will become #1 in the list.
-        The global marketplace source cannot be deleted.
+        The global hub source cannot be deleted.
 
-        :param source_name: Name of the marketplace source to delete.
+        :param source_name: Name of the hub source to delete.
         """
-        path = f"marketplace/sources/{source_name}"
+        path = f"hub/sources/{source_name}"
         self.api_call(method="DELETE", path=path)
 
-    def get_marketplace_catalog(
+    def get_hub_catalog(
         self,
         source_name: str,
-        channel: str = None,
         version: str = None,
         tag: str = None,
         force_refresh: bool = False,
     ):
         """
-        Retrieve the item catalog for a specified marketplace source.
+        Retrieve the item catalog for a specified hub source.
         The list of items can be filtered according to various filters, using item's metadata to filter.
 
         :param source_name: Name of the source.
-        :param channel: Filter items according to their channel. For example ``development``.
         :param version: Filter items according to their version.
         :param tag: Filter items based on tag.
-        :param force_refresh: Make the server fetch the catalog from the actual marketplace source,
+        :param force_refresh: Make the server fetch the catalog from the actual hub source,
             rather than rely on cached information which may exist from previous get requests. For example,
             if the source was re-built,
             this will make the server get the updated information. Default is ``False``.
-        :returns: :py:class:`~mlrun.api.schemas.marketplace.MarketplaceCatalog` object, which is essentially a list
-            of :py:class:`~mlrun.api.schemas.marketplace.MarketplaceItem` entries.
+        :returns: :py:class:`~mlrun.common.schemas.hub.HubCatalog` object, which is essentially a list
+            of :py:class:`~mlrun.common.schemas.hub.HubItem` entries.
         """
-        path = (f"marketplace/sources/{source_name}/items",)
+        path = (f"hub/sources/{source_name}/items",)
         params = {
-            "channel": channel,
             "version": version,
             "tag": tag,
             "force-refresh": force_refresh,
         }
         response = self.api_call(method="GET", path=path, params=params)
-        return schemas.MarketplaceCatalog(**response.json())
+        return mlrun.common.schemas.HubCatalog(**response.json())
 
-    def get_marketplace_item(
+    def get_hub_item(
         self,
         source_name: str,
         item_name: str,
-        channel: str = "development",
         version: str = None,
         tag: str = "latest",
         force_refresh: bool = False,
     ):
         """
-        Retrieve a specific marketplace item.
+        Retrieve a specific hub item.
 
         :param source_name: Name of source.
         :param item_name: Name of the item to retrieve, as it appears in the catalog.
-        :param channel: Get the item from the specified channel. Default is ``development``.
         :param version: Get a specific version of the item. Default is ``None``.
         :param tag: Get a specific version of the item identified by tag. Default is ``latest``.
-        :param force_refresh: Make the server fetch the information from the actual marketplace
+        :param force_refresh: Make the server fetch the information from the actual hub
             source, rather than
             rely on cached information. Default is ``False``.
-        :returns: :py:class:`~mlrun.api.schemas.marketplace.MarketplaceItem`.
+        :returns: :py:class:`~mlrun.common.schemas.hub.HubItem`.
         """
-        path = (f"marketplace/sources/{source_name}/items/{item_name}",)
+        path = (f"hub/sources/{source_name}/items/{item_name}",)
         params = {
-            "channel": channel,
             "version": version,
             "tag": tag,
             "force-refresh": force_refresh,
         }
         response = self.api_call(method="GET", path=path, params=params)
-        return schemas.MarketplaceItem(**response.json())
+        return mlrun.common.schemas.HubItem(**response.json())
+
+    def get_hub_asset(
+        self,
+        source_name: str,
+        item_name: str,
+        asset_name: str,
+        version: str = None,
+        tag: str = "latest",
+    ):
+        """
+        Get hub asset from item.
+
+        :param source_name: Name of source.
+        :param item_name:   Name of the item which holds the asset.
+        :param asset_name:  Name of the asset to retrieve.
+        :param version: Get a specific version of the item. Default is ``None``.
+        :param tag: Get a specific version of the item identified by tag. Default is ``latest``.
+
+        :return: http response with the asset in the content attribute
+        """
+        path = (f"hub/sources/{source_name}/items/{item_name}/assets/{asset_name}",)
+        params = {
+            "version": version,
+            "tag": tag,
+        }
+        response = self.api_call(method="GET", path=path, params=params)
+        return response
 
     def verify_authorization(
-        self, authorization_verification_input: schemas.AuthorizationVerificationInput
+        self,
+        authorization_verification_input: mlrun.common.schemas.AuthorizationVerificationInput,
     ):
         """Verifies authorization for the provided action on the provided resource.
 
         :param authorization_verification_input: Instance of
-            :py:class:`~mlrun.api.schemas.AuthorizationVerificationInput` that includes all the needed parameters for
+            :py:class:`~mlrun.common.schemas.AuthorizationVerificationInput` that includes all the needed parameters for
             the auth verification
         """
         error_message = "Authorization check failed"
@@ -2900,10 +3018,10 @@ class HTTPRunDB(RunDBInterface):
             body=dict_to_json(authorization_verification_input.dict()),
         )
 
-    def trigger_migrations(self) -> Optional[schemas.BackgroundTask]:
+    def trigger_migrations(self) -> Optional[mlrun.common.schemas.BackgroundTask]:
         """Trigger migrations (will do nothing if no migrations are needed) and wait for them to finish if actually
         triggered
-        :returns: :py:class:`~mlrun.api.schemas.BackgroundTask`.
+        :returns: :py:class:`~mlrun.common.schemas.BackgroundTask`.
         """
         response = self.api_call(
             "POST",
@@ -2911,12 +3029,62 @@ class HTTPRunDB(RunDBInterface):
             "Failed triggering migrations",
         )
         if response.status_code == http.HTTPStatus.ACCEPTED:
-            background_task = schemas.BackgroundTask(**response.json())
+            background_task = mlrun.common.schemas.BackgroundTask(**response.json())
             return self._wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name
             )
         return None
 
+    def set_run_notifications(
+        self,
+        project: str,
+        run_uid: str,
+        notifications: typing.List[mlrun.model.Notification] = None,
+    ):
+        """
+        Set notifications on a run. This will override any existing notifications on the run.
+        :param project: Project containing the run.
+        :param run_uid: UID of the run.
+        :param notifications: List of notifications to set on the run. Default is an empty list.
+        """
+        notifications = notifications or []
+
+        self.api_call(
+            "PUT",
+            f"projects/{project}/runs/{run_uid}/notifications",
+            f"Failed to set notifications on run. uid={run_uid}, project={project}",
+            json={
+                "notifications": [
+                    notification.to_dict() for notification in notifications
+                ],
+            },
+        )
+
+    def set_schedule_notifications(
+        self,
+        project: str,
+        schedule_name: str,
+        notifications: typing.List[mlrun.model.Notification] = None,
+    ):
+        """
+        Set notifications on a schedule. This will override any existing notifications on the schedule.
+        :param project: Project containing the schedule.
+        :param schedule_name: Name of the schedule.
+        :param notifications: List of notifications to set on the schedule. Default is an empty list.
+        """
+        notifications = notifications or []
+
+        self.api_call(
+            "PUT",
+            f"projects/{project}/schedules/{schedule_name}/notifications",
+            f"Failed to set notifications on schedule. schedule={schedule_name}, project={project}",
+            json={
+                "notifications": [
+                    notification.to_dict() for notification in notifications
+                ],
+            },
+        )
+
 
 def _as_json(obj):
     fn = getattr(obj, "to_json", None)