mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/api/db/sqldb/db.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,8 +32,10 @@ import mlrun
 import mlrun.api.db.session
 import mlrun.api.utils.helpers
 import mlrun.api.utils.projects.remotes.follower
+import mlrun.api.utils.singletons.k8s
+import mlrun.common.schemas
 import mlrun.errors
-from mlrun.api import schemas
+import mlrun.model
 from mlrun.api.db.base import DBInterface
 from mlrun.api.db.sqldb.helpers import (
     generate_query_predicate_for_name,
@@ -52,14 +54,15 @@ from mlrun.api.db.sqldb.models import (
     FeatureSet,
     FeatureVector,
     Function,
+    HubSource,
     Log,
-    MarketplaceSource,
     Project,
     Run,
     Schedule,
     User,
     _labeled,
     _tagged,
+    _with_notifications,
 )
 from mlrun.config import config
 from mlrun.errors import err_to_str
@@ -323,13 +326,14 @@ class SQLDB(DBInterface):
         start_time_to=None,
         last_update_time_from=None,
         last_update_time_to=None,
-        partition_by: schemas.RunPartitionByField = None,
+        partition_by: mlrun.common.schemas.RunPartitionByField = None,
         rows_per_partition: int = 1,
-        partition_sort_by: schemas.SortField = None,
-        partition_order: schemas.OrderType = schemas.OrderType.desc,
+        partition_sort_by: mlrun.common.schemas.SortField = None,
+        partition_order: mlrun.common.schemas.OrderType = mlrun.common.schemas.OrderType.desc,
         max_partitions: int = 0,
         requested_logs: bool = None,
         return_as_run_structs: bool = True,
+        with_notifications: bool = False,
     ):
         project = project or config.default_project
         query = self._find_runs(session, uid, project, labels)
@@ -359,7 +363,9 @@ class SQLDB(DBInterface):
             query = query.filter(Run.requested_logs == requested_logs)
         if partition_by:
             self._assert_partition_by_parameters(
-                schemas.RunPartitionByField, partition_by, partition_sort_by
+                mlrun.common.schemas.RunPartitionByField,
+                partition_by,
+                partition_sort_by,
             )
             query = self._create_partitioned_query(
                 session,
@@ -374,9 +380,28 @@ class SQLDB(DBInterface):
         if not return_as_run_structs:
             return query.all()
 
+        # Purposefully not using outer join to avoid returning runs without notifications
+        if with_notifications:
+            query = query.join(Run.Notification)
+
         runs = RunList()
         for run in query:
-            runs.append(run.struct)
+            run_struct = run.struct
+            if with_notifications:
+                run_struct.setdefault("spec", {}).setdefault("notifications", [])
+                run_struct.setdefault("status", {}).setdefault("notifications", {})
+                for notification in run.notifications:
+                    (
+                        notification_spec,
+                        notification_status,
+                    ) = self._transform_notification_record_to_spec_and_status(
+                        notification
+                    )
+                    run_struct["spec"]["notifications"].append(notification_spec)
+                    run_struct["status"]["notifications"][
+                        notification.name
+                    ] = notification_status
+            runs.append(run_struct)
 
         return runs
 
@@ -438,7 +463,7 @@ class SQLDB(DBInterface):
         session: Session,
         project: str,
         tag: str,
-        identifiers: typing.List[mlrun.api.schemas.ArtifactIdentifier],
+        identifiers: typing.List[mlrun.common.schemas.ArtifactIdentifier],
     ):
         # query all artifacts which match the identifiers
         artifacts = []
@@ -461,7 +486,7 @@ class SQLDB(DBInterface):
         session: Session,
         project: str,
         tag: str,
-        identifiers: typing.List[mlrun.api.schemas.ArtifactIdentifier],
+        identifiers: typing.List[mlrun.common.schemas.ArtifactIdentifier],
     ):
         # query all artifacts which match the identifiers
         artifacts = []
@@ -478,7 +503,7 @@ class SQLDB(DBInterface):
         session: Session,
         project: str,
         tag: str,
-        identifiers: typing.List[mlrun.api.schemas.ArtifactIdentifier],
+        identifiers: typing.List[mlrun.common.schemas.ArtifactIdentifier],
     ):
         # query all artifacts which match the identifiers
         artifacts = []
@@ -494,7 +519,7 @@ class SQLDB(DBInterface):
         self,
         session: Session,
         project_name: str,
-        identifier: mlrun.api.schemas.ArtifactIdentifier,
+        identifier: mlrun.common.schemas.ArtifactIdentifier,
     ):
         return self.list_artifacts(
             session,
@@ -698,7 +723,7 @@ class SQLDB(DBInterface):
         since=None,
         until=None,
         kind=None,
-        category: schemas.ArtifactCategories = None,
+        category: mlrun.common.schemas.ArtifactCategories = None,
         iter: int = None,
         best_iteration: bool = False,
         as_records: bool = False,
@@ -963,7 +988,31 @@ class SQLDB(DBInterface):
         self.tag_objects_v2(session, [fn], project, tag)
         return hash_key
 
-    def get_function(self, session, name, project="", tag="", hash_key=""):
+    def get_function(self, session, name, project="", tag="", hash_key="") -> dict:
+        """
+        In version 1.4.0 we added a normalization to the function name before storing.
+        To be backwards compatible and allow users to query old non-normalized functions,
+        we're providing a fallback to get_function:
+        normalize the requested name and try to retrieve it from the database.
+        If no answer is received, we will check to see if the original name contained underscores,
+        if so, the retrieval will be repeated and the result (if it exists) returned.
+        """
+        normalized_function_name = mlrun.utils.normalize_name(name)
+        try:
+            return self._get_function(
+                session, normalized_function_name, project, tag, hash_key
+            )
+        except mlrun.errors.MLRunNotFoundError as exc:
+            if "_" in name:
+                logger.warning(
+                    "Failed to get underscore-named function, trying without normalization",
+                    function_name=name,
+                )
+                return self._get_function(session, name, project, tag, hash_key)
+            else:
+                raise exc
+
+    def _get_function(self, session, name, project="", tag="", hash_key=""):
         project = project or config.default_project
         query = self._query(session, Function, name=name, project=project)
         computed_tag = tag or "latest"
@@ -1100,7 +1149,7 @@ class SQLDB(DBInterface):
         return [row[0] for row in query]
 
     def list_artifact_tags(
-        self, session, project, category: schemas.ArtifactCategories = None
+        self, session, project, category: mlrun.common.schemas.ArtifactCategories = None
     ) -> typing.List[typing.Tuple[str, str, str]]:
         """
         :return: a list of Tuple of (project, artifact.key, tag)
@@ -1131,9 +1180,9 @@ class SQLDB(DBInterface):
         session: Session,
         project: str,
         name: str,
-        kind: schemas.ScheduleKinds,
+        kind: mlrun.common.schemas.ScheduleKinds,
         scheduled_object: Any,
-        cron_trigger: schemas.ScheduleCronTrigger,
+        cron_trigger: mlrun.common.schemas.ScheduleCronTrigger,
         concurrency_limit: int,
         labels: Dict = None,
         next_run_time: datetime = None,
@@ -1176,7 +1225,7 @@ class SQLDB(DBInterface):
         project: str,
         name: str,
         scheduled_object: Any = None,
-        cron_trigger: schemas.ScheduleCronTrigger = None,
+        cron_trigger: mlrun.common.schemas.ScheduleCronTrigger = None,
         labels: Dict = None,
         last_run_uri: str = None,
         concurrency_limit: int = None,
@@ -1222,8 +1271,8 @@ class SQLDB(DBInterface):
         project: str = None,
         name: str = None,
         labels: str = None,
-        kind: schemas.ScheduleKinds = None,
-    ) -> List[schemas.ScheduleRecord]:
+        kind: mlrun.common.schemas.ScheduleKinds = None,
+    ) -> List[mlrun.common.schemas.ScheduleRecord]:
         logger.debug("Getting schedules from db", project=project, name=name, kind=kind)
         query = self._query(session, Schedule, project=project, kind=kind)
         if name is not None:
@@ -1239,7 +1288,7 @@ class SQLDB(DBInterface):
 
     def get_schedule(
         self, session: Session, project: str, name: str
-    ) -> schemas.ScheduleRecord:
+    ) -> mlrun.common.schemas.ScheduleRecord:
         logger.debug("Getting schedule from db", project=project, name=name)
         schedule_record = self._get_schedule_record(session, project, name)
         schedule = self._transform_schedule_record_to_scheme(schedule_record)
@@ -1247,7 +1296,7 @@ class SQLDB(DBInterface):
 
     def _get_schedule_record(
         self, session: Session, project: str, name: str
-    ) -> schemas.ScheduleRecord:
+    ) -> mlrun.common.schemas.ScheduleRecord:
         query = self._query(session, Schedule, project=project, name=name)
         schedule_record = query.one_or_none()
         if not schedule_record:
@@ -1340,8 +1389,8 @@ class SQLDB(DBInterface):
             tags.append(tag)
         self._upsert(session, tags)
 
-    def create_project(self, session: Session, project: schemas.Project):
-        logger.debug("Creating project in DB", project=project)
+    def create_project(self, session: Session, project: mlrun.common.schemas.Project):
+        logger.debug("Creating project in DB", project_name=project.metadata.name)
         created = datetime.utcnow()
         project.metadata.created = created
         # TODO: handle taking out the functions/workflows/artifacts out of the project and save them separately
@@ -1359,8 +1408,17 @@ class SQLDB(DBInterface):
         self._upsert(session, [project_record])
 
     @retry_on_conflict
-    def store_project(self, session: Session, name: str, project: schemas.Project):
-        logger.debug("Storing project in DB", name=name)
+    def store_project(
+        self, session: Session, name: str, project: mlrun.common.schemas.Project
+    ):
+        logger.debug(
+            "Storing project in DB",
+            name=name,
+            project_metadata=project.metadata,
+            project_owner=project.spec.owner,
+            project_desired_state=project.spec.desired_state,
+            project_status=project.status,
+        )
         project_record = self._get_project_record(
             session, name, raise_on_not_found=False
         )
@@ -1374,7 +1432,7 @@ class SQLDB(DBInterface):
         session: Session,
         name: str,
         project: dict,
-        patch_mode: schemas.PatchMode = schemas.PatchMode.replace,
+        patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
     ):
         logger.debug("Patching project in DB", name=name, patch_mode=patch_mode)
         project_record = self._get_project_record(session, name)
@@ -1384,7 +1442,7 @@ class SQLDB(DBInterface):
 
     def get_project(
         self, session: Session, name: str = None, project_id: int = None
-    ) -> schemas.Project:
+    ) -> mlrun.common.schemas.Project:
         project_record = self._get_project_record(session, name, project_id)
 
         return self._transform_project_record_to_schema(session, project_record)
@@ -1393,7 +1451,7 @@ class SQLDB(DBInterface):
         self,
         session: Session,
         name: str,
-        deletion_strategy: schemas.DeletionStrategy = schemas.DeletionStrategy.default(),
+        deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
     ):
         logger.debug(
             "Deleting project from DB", name=name, deletion_strategy=deletion_strategy
@@ -1404,16 +1462,16 @@ class SQLDB(DBInterface):
         self,
         session: Session,
         owner: str = None,
-        format_: mlrun.api.schemas.ProjectsFormat = mlrun.api.schemas.ProjectsFormat.full,
+        format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
         labels: List[str] = None,
-        state: mlrun.api.schemas.ProjectState = None,
+        state: mlrun.common.schemas.ProjectState = None,
         names: typing.Optional[typing.List[str]] = None,
-    ) -> schemas.ProjectsOutput:
+    ) -> mlrun.common.schemas.ProjectsOutput:
         query = self._query(session, Project, owner=owner, state=state)
 
         # if format is name_only, we don't need to query the full project object, we can just query the name
         # and return it as a list of strings
-        if format_ == mlrun.api.schemas.ProjectsFormat.name_only:
+        if format_ == mlrun.common.schemas.ProjectsFormat.name_only:
             query = self._query(session, Project.name, owner=owner, state=state)
 
         # attach filters to the query
@@ -1427,12 +1485,12 @@ class SQLDB(DBInterface):
         # format the projects according to the requested format
         projects = []
         for project_record in project_records:
-            if format_ == mlrun.api.schemas.ProjectsFormat.name_only:
+            if format_ == mlrun.common.schemas.ProjectsFormat.name_only:
                 projects.append(project_record.name)
 
-            elif format_ == mlrun.api.schemas.ProjectsFormat.minimal:
+            elif format_ == mlrun.common.schemas.ProjectsFormat.minimal:
                 projects.append(
-                    self._minimize_project_schema(
+                    mlrun.api.utils.helpers.minimize_project_schema(
                         self._transform_project_record_to_schema(
                             session, project_record
                         )
@@ -1441,8 +1499,8 @@ class SQLDB(DBInterface):
 
             # leader format is only for follower mode which will format the projects returned from here
             elif format_ in [
-                mlrun.api.schemas.ProjectsFormat.full,
-                mlrun.api.schemas.ProjectsFormat.leader,
+                mlrun.common.schemas.ProjectsFormat.full,
+                mlrun.common.schemas.ProjectsFormat.leader,
             ]:
                 projects.append(
                     self._transform_project_record_to_schema(session, project_record)
@@ -1451,16 +1509,7 @@ class SQLDB(DBInterface):
                 raise NotImplementedError(
                     f"Provided format is not supported. format={format_}"
                 )
-        return schemas.ProjectsOutput(projects=projects)
-
-    def _minimize_project_schema(
-        self,
-        project: mlrun.api.schemas.Project,
-    ) -> mlrun.api.schemas.Project:
-        project.spec.functions = None
-        project.spec.workflows = None
-        project.spec.artifacts = None
-        return project
+        return mlrun.common.schemas.ProjectsOutput(projects=projects)
 
     async def get_project_resources_counters(
         self,
@@ -1569,7 +1618,10 @@ class SQLDB(DBInterface):
         # We're using the "latest" which gives us only one version of each artifact key, which is what we want to
         # count (artifact count, not artifact versions count)
         file_artifacts = self._find_artifacts(
-            session, None, "latest", category=mlrun.api.schemas.ArtifactCategories.other
+            session,
+            None,
+            "latest",
+            category=mlrun.common.schemas.ArtifactCategories.other,
         )
         project_to_files_count = collections.defaultdict(int)
         for file_artifact in file_artifacts:
@@ -1613,7 +1665,7 @@ class SQLDB(DBInterface):
 
     async def generate_projects_summaries(
         self, session: Session, projects: List[str]
-    ) -> List[mlrun.api.schemas.ProjectSummary]:
+    ) -> List[mlrun.common.schemas.ProjectSummary]:
         (
             project_to_function_count,
             project_to_schedule_count,
@@ -1625,7 +1677,7 @@ class SQLDB(DBInterface):
         project_summaries = []
         for project in projects:
             project_summaries.append(
-                mlrun.api.schemas.ProjectSummary(
+                mlrun.common.schemas.ProjectSummary(
                     name=project,
                     functions_count=project_to_function_count.get(project, 0),
                     schedules_count=project_to_schedule_count.get(project, 0),
@@ -1643,7 +1695,10 @@ class SQLDB(DBInterface):
         return project_summaries
 
     def _update_project_record_from_project(
-        self, session: Session, project_record: Project, project: schemas.Project
+        self,
+        session: Session,
+        project_record: Project,
+        project: mlrun.common.schemas.Project,
     ):
         project.metadata.created = project_record.created
         project_dict = project.dict()
@@ -1663,7 +1718,7 @@ class SQLDB(DBInterface):
         name: str,
         project_record: Project,
         project: dict,
-        patch_mode: schemas.PatchMode,
+        patch_mode: mlrun.common.schemas.PatchMode,
     ):
         project.setdefault("metadata", {})["created"] = project_record.created
         strategy = patch_mode.to_mergedeep_strategy()
@@ -1671,7 +1726,7 @@ class SQLDB(DBInterface):
         mergedeep.merge(project_record_full_object, project, strategy=strategy)
 
         # If a bad kind value was passed, it will fail here (return 422 to caller)
-        project = schemas.Project(**project_record_full_object)
+        project = mlrun.common.schemas.Project(**project_record_full_object)
         self.store_project(
             session,
             name,
@@ -1695,7 +1750,7 @@ class SQLDB(DBInterface):
         name: str = None,
         project_id: int = None,
         raise_on_not_found: bool = True,
-    ) -> Project:
+    ) -> typing.Optional[Project]:
         if not any([project_id, name]):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "One of 'name' or 'project_id' must be provided"
@@ -1721,6 +1776,12 @@ class SQLDB(DBInterface):
         self._verify_empty_list_of_project_related_resources(name, logs, "logs")
         runs = self._find_runs(session, None, name, []).all()
         self._verify_empty_list_of_project_related_resources(name, runs, "runs")
+        notifications = []
+        for cls in _with_notifications:
+            notifications.extend(self._get_db_notifications(session, cls, project=name))
+        self._verify_empty_list_of_project_related_resources(
+            name, notifications, "notifications"
+        )
         schedules = self.list_schedules(session, project=name)
         self._verify_empty_list_of_project_related_resources(
             name, schedules, "schedules"
@@ -1741,6 +1802,7 @@ class SQLDB(DBInterface):
     def delete_project_related_resources(self, session: Session, name: str):
         self.del_artifacts(session, project=name)
         self._delete_logs(session, name)
+        self.delete_run_notifications(session, project=name)
         self.del_runs(session, project=name)
         self.delete_schedules(session, name)
         self._delete_functions(session, name)
@@ -1817,7 +1879,7 @@ class SQLDB(DBInterface):
         name: str,
         tag: str = None,
         uid: str = None,
-    ) -> schemas.FeatureSet:
+    ) -> mlrun.common.schemas.FeatureSet:
         feature_set = self._get_feature_set(session, project, name, tag, uid)
         if not feature_set:
             feature_set_uri = generate_object_uri(project, name, tag)
@@ -1865,10 +1927,10 @@ class SQLDB(DBInterface):
         return results
 
     @staticmethod
-    def _generate_feature_set_digest(feature_set: schemas.FeatureSet):
-        return schemas.FeatureSetDigestOutput(
+    def _generate_feature_set_digest(feature_set: mlrun.common.schemas.FeatureSet):
+        return mlrun.common.schemas.FeatureSetDigestOutput(
             metadata=feature_set.metadata,
-            spec=schemas.FeatureSetDigestSpec(
+            spec=mlrun.common.schemas.FeatureSetDigestSpec(
                 entities=feature_set.spec.entities,
                 features=feature_set.spec.features,
             ),
@@ -1910,7 +1972,7 @@ class SQLDB(DBInterface):
         tag: str = None,
         entities: List[str] = None,
         labels: List[str] = None,
-    ) -> schemas.FeaturesOutput:
+    ) -> mlrun.common.schemas.FeaturesOutput:
         # We don't filter by feature-set name here, as the name parameter refers to features
         feature_set_id_tags = self._get_records_to_tags_map(
             session, FeatureSet, project, tag, name=None
@@ -1925,7 +1987,7 @@ class SQLDB(DBInterface):
 
         features_results = []
         for row in query:
-            feature_record = schemas.FeatureRecord.from_orm(row.Feature)
+            feature_record = mlrun.common.schemas.FeatureRecord.from_orm(row.Feature)
             feature_name = feature_record.name
 
             feature_sets = self._generate_records_with_tags_assigned(
@@ -1952,14 +2014,14 @@ class SQLDB(DBInterface):
                     )
 
                 features_results.append(
-                    schemas.FeatureListOutput(
+                    mlrun.common.schemas.FeatureListOutput(
                         feature=feature,
                         feature_set_digest=self._generate_feature_set_digest(
                             feature_set
                         ),
                     )
                 )
-        return schemas.FeaturesOutput(features=features_results)
+        return mlrun.common.schemas.FeaturesOutput(features=features_results)
 
     def list_entities(
         self,
@@ -1968,7 +2030,7 @@ class SQLDB(DBInterface):
         name: str = None,
         tag: str = None,
         labels: List[str] = None,
-    ) -> schemas.EntitiesOutput:
+    ) -> mlrun.common.schemas.EntitiesOutput:
         feature_set_id_tags = self._get_records_to_tags_map(
             session, FeatureSet, project, tag, name=None
         )
@@ -1979,7 +2041,7 @@ class SQLDB(DBInterface):
 
         entities_results = []
         for row in query:
-            entity_record = schemas.FeatureRecord.from_orm(row.Entity)
+            entity_record = mlrun.common.schemas.FeatureRecord.from_orm(row.Entity)
             entity_name = entity_record.name
 
             feature_sets = self._generate_records_with_tags_assigned(
@@ -2006,14 +2068,14 @@ class SQLDB(DBInterface):
                     )
 
                 entities_results.append(
-                    schemas.EntityListOutput(
+                    mlrun.common.schemas.EntityListOutput(
                         entity=entity,
                         feature_set_digest=self._generate_feature_set_digest(
                             feature_set
                         ),
                     )
                 )
-        return schemas.EntitiesOutput(entities=entities_results)
+        return mlrun.common.schemas.EntitiesOutput(entities=entities_results)
 
     @staticmethod
     def _assert_partition_by_parameters(partition_by_enum_cls, partition_by, sort):
@@ -2036,11 +2098,12 @@ class SQLDB(DBInterface):
         query,
         cls,
         partition_by: typing.Union[
-            schemas.FeatureStorePartitionByField, schemas.RunPartitionByField
+            mlrun.common.schemas.FeatureStorePartitionByField,
+            mlrun.common.schemas.RunPartitionByField,
         ],
         rows_per_partition: int,
-        partition_sort_by: schemas.SortField,
-        partition_order: schemas.OrderType,
+        partition_sort_by: mlrun.common.schemas.SortField,
+        partition_order: mlrun.common.schemas.OrderType,
         max_partitions: int = 0,
     ):
 
@@ -2106,11 +2169,11 @@ class SQLDB(DBInterface):
         entities: List[str] = None,
         features: List[str] = None,
         labels: List[str] = None,
-        partition_by: schemas.FeatureStorePartitionByField = None,
+        partition_by: mlrun.common.schemas.FeatureStorePartitionByField = None,
         rows_per_partition: int = 1,
-        partition_sort_by: schemas.SortField = None,
-        partition_order: schemas.OrderType = schemas.OrderType.desc,
-    ) -> schemas.FeatureSetsOutput:
+        partition_sort_by: mlrun.common.schemas.SortField = None,
+        partition_order: mlrun.common.schemas.OrderType = mlrun.common.schemas.OrderType.desc,
+    ) -> mlrun.common.schemas.FeatureSetsOutput:
         obj_id_tags = self._get_records_to_tags_map(
             session, FeatureSet, project, tag, name
         )
@@ -2133,7 +2196,9 @@ class SQLDB(DBInterface):
 
         if partition_by:
             self._assert_partition_by_parameters(
-                schemas.FeatureStorePartitionByField, partition_by, partition_sort_by
+                mlrun.common.schemas.FeatureStorePartitionByField,
+                partition_by,
+                partition_sort_by,
             )
             query = self._create_partitioned_query(
                 session,
@@ -2155,7 +2220,7 @@ class SQLDB(DBInterface):
                     tag,
                 )
             )
-        return schemas.FeatureSetsOutput(feature_sets=feature_sets)
+        return mlrun.common.schemas.FeatureSetsOutput(feature_sets=feature_sets)
 
     def list_feature_sets_tags(
         self,
@@ -2228,7 +2293,6 @@ class SQLDB(DBInterface):
         feature_set_spec = new_feature_set_dict.get("spec")
         features = feature_set_spec.pop("features", [])
         entities = feature_set_spec.pop("entities", [])
-
         self._update_feature_set_features(feature_set, features)
         self._update_feature_set_entities(feature_set, entities)
 
@@ -2286,7 +2350,7 @@ class SQLDB(DBInterface):
         session,
         project,
         name,
-        feature_set: schemas.FeatureSet,
+        feature_set: mlrun.common.schemas.FeatureSet,
         tag=None,
         uid=None,
         versioned=True,
@@ -2385,7 +2449,7 @@ class SQLDB(DBInterface):
         self,
         session,
         project,
-        feature_set: schemas.FeatureSet,
+        feature_set: mlrun.common.schemas.FeatureSet,
         versioned=True,
     ) -> str:
         (uid, tag, feature_set_dict,) = self._validate_and_enrich_record_for_creation(
@@ -2393,7 +2457,6 @@ class SQLDB(DBInterface):
         )
 
         db_feature_set = FeatureSet(project=project)
-
         self._update_db_record_from_object_dict(db_feature_set, feature_set_dict, uid)
         self._update_feature_set_spec(db_feature_set, feature_set_dict)
 
@@ -2410,7 +2473,7 @@ class SQLDB(DBInterface):
         feature_set_patch: dict,
         tag=None,
         uid=None,
-        patch_mode: schemas.PatchMode = schemas.PatchMode.replace,
+        patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
     ) -> str:
         feature_set_record = self._get_feature_set(session, project, name, tag, uid)
         if not feature_set_record:
@@ -2427,7 +2490,7 @@ class SQLDB(DBInterface):
         versioned = feature_set_record.metadata.uid is not None
 
         # If a bad kind value was passed, it will fail here (return 422 to caller)
-        feature_set = schemas.FeatureSet(**feature_set_struct)
+        feature_set = mlrun.common.schemas.FeatureSet(**feature_set_struct)
         return self.store_feature_set(
             session,
             project,
@@ -2478,7 +2541,7 @@ class SQLDB(DBInterface):
         self,
         session,
         project,
-        feature_vector: schemas.FeatureVector,
+        feature_vector: mlrun.common.schemas.FeatureVector,
         versioned=True,
     ) -> str:
         (
@@ -2529,7 +2592,7 @@ class SQLDB(DBInterface):
 
     def get_feature_vector(
         self, session, project: str, name: str, tag: str = None, uid: str = None
-    ) -> schemas.FeatureVector:
+    ) -> mlrun.common.schemas.FeatureVector:
         feature_vector = self._get_feature_vector(session, project, name, tag, uid)
         if not feature_vector:
             feature_vector_uri = generate_object_uri(project, name, tag)
@@ -2547,11 +2610,11 @@ class SQLDB(DBInterface):
         tag: str = None,
         state: str = None,
         labels: List[str] = None,
-        partition_by: schemas.FeatureStorePartitionByField = None,
+        partition_by: mlrun.common.schemas.FeatureStorePartitionByField = None,
         rows_per_partition: int = 1,
-        partition_sort_by: schemas.SortField = None,
-        partition_order: schemas.OrderType = schemas.OrderType.desc,
-    ) -> schemas.FeatureVectorsOutput:
+        partition_sort_by: mlrun.common.schemas.SortField = None,
+        partition_order: mlrun.common.schemas.OrderType = mlrun.common.schemas.OrderType.desc,
+    ) -> mlrun.common.schemas.FeatureVectorsOutput:
         obj_id_tags = self._get_records_to_tags_map(
             session, FeatureVector, project, tag, name
         )
@@ -2570,7 +2633,9 @@ class SQLDB(DBInterface):
 
         if partition_by:
             self._assert_partition_by_parameters(
-                schemas.FeatureStorePartitionByField, partition_by, partition_sort_by
+                mlrun.common.schemas.FeatureStorePartitionByField,
+                partition_by,
+                partition_sort_by,
             )
             query = self._create_partitioned_query(
                 session,
@@ -2592,7 +2657,9 @@ class SQLDB(DBInterface):
                     tag,
                 )
             )
-        return schemas.FeatureVectorsOutput(feature_vectors=feature_vectors)
+        return mlrun.common.schemas.FeatureVectorsOutput(
+            feature_vectors=feature_vectors
+        )
 
     def list_feature_vectors_tags(
         self,
@@ -2613,7 +2680,7 @@ class SQLDB(DBInterface):
         session,
         project,
         name,
-        feature_vector: schemas.FeatureVector,
+        feature_vector: mlrun.common.schemas.FeatureVector,
         tag=None,
         uid=None,
         versioned=True,
@@ -2676,7 +2743,7 @@ class SQLDB(DBInterface):
         feature_vector_update: dict,
         tag=None,
         uid=None,
-        patch_mode: schemas.PatchMode = schemas.PatchMode.replace,
+        patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
     ) -> str:
         feature_vector_record = self._get_feature_vector(
             session, project, name, tag, uid
@@ -2694,7 +2761,7 @@ class SQLDB(DBInterface):
 
         versioned = feature_vector_record.metadata.uid is not None
 
-        feature_vector = schemas.FeatureVector(**feature_vector_struct)
+        feature_vector = mlrun.common.schemas.FeatureVector(**feature_vector_struct)
         return self.store_feature_vector(
             session,
             project,
@@ -2745,17 +2812,6 @@ class SQLDB(DBInterface):
         kw = {k: v for k, v in kw.items() if v is not None}
         return session.query(cls).filter_by(**kw)
 
-    def _function_latest_uid(self, session, project, name):
-        # FIXME
-        query = (
-            self._query(session, Function.uid)
-            .filter(Function.project == project, Function.name == name)
-            .order_by(Function.updated.desc())
-        ).limit(1)
-        out = query.one_or_none()
-        if out:
-            return out[0]
-
     def _find_or_create_users(self, session, user_names):
         users = list(self._query(session, User).filter(User.name.in_(user_names)))
         new = set(user_names) - {user.name for user in users}
@@ -2864,6 +2920,13 @@ class SQLDB(DBInterface):
             query = query.filter(Run.uid.in_(uid))
         return self._add_labels_filter(session, query, Run, labels)
 
+    def _get_db_notifications(
+        self, session, cls, name: str = None, parent_id: str = None, project: str = None
+    ):
+        return self._query(
+            session, cls.Notification, name=name, parent_id=parent_id, project=project
+        ).all()
+
     def _latest_uid_filter(self, session, query):
         # Create a sub query of latest uid (by updated) per (project,key)
         subq = (
@@ -2939,7 +3002,7 @@ class SQLDB(DBInterface):
         until=None,
         name=None,
         kind=None,
-        category: schemas.ArtifactCategories = None,
+        category: mlrun.common.schemas.ArtifactCategories = None,
         iter=None,
         use_tag_as_uid: bool = None,
     ):
@@ -2990,7 +3053,7 @@ class SQLDB(DBInterface):
             return query.all()
 
     def _filter_artifacts_by_category(
-        self, artifacts, category: schemas.ArtifactCategories
+        self, artifacts, category: mlrun.common.schemas.ArtifactCategories
     ):
         kinds, exclude = category.to_kinds_filter()
         return self._filter_artifacts_by_kinds(artifacts, kinds, exclude)
@@ -3137,8 +3200,8 @@ class SQLDB(DBInterface):
     def _transform_schedule_record_to_scheme(
         self,
         schedule_record: Schedule,
-    ) -> schemas.ScheduleRecord:
-        schedule = schemas.ScheduleRecord.from_orm(schedule_record)
+    ) -> mlrun.common.schemas.ScheduleRecord:
+        schedule = mlrun.common.schemas.ScheduleRecord.from_orm(schedule_record)
         schedule.creation_time = self._add_utc_timezone(schedule.creation_time)
         schedule.next_run_time = self._add_utc_timezone(schedule.next_run_time)
         return schedule
@@ -3158,9 +3221,9 @@ class SQLDB(DBInterface):
     def _transform_feature_set_model_to_schema(
         feature_set_record: FeatureSet,
         tag=None,
-    ) -> schemas.FeatureSet:
+    ) -> mlrun.common.schemas.FeatureSet:
         feature_set_full_dict = feature_set_record.full_object
-        feature_set_resp = schemas.FeatureSet(**feature_set_full_dict)
+        feature_set_resp = mlrun.common.schemas.FeatureSet(**feature_set_full_dict)
 
         feature_set_resp.metadata.tag = tag
         return feature_set_resp
@@ -3169,9 +3232,11 @@ class SQLDB(DBInterface):
     def _transform_feature_vector_model_to_schema(
         feature_vector_record: FeatureVector,
         tag=None,
-    ) -> schemas.FeatureVector:
+    ) -> mlrun.common.schemas.FeatureVector:
         feature_vector_full_dict = feature_vector_record.full_object
-        feature_vector_resp = schemas.FeatureVector(**feature_vector_full_dict)
+        feature_vector_resp = mlrun.common.schemas.FeatureVector(
+            **feature_vector_full_dict
+        )
 
         feature_vector_resp.metadata.tag = tag
         feature_vector_resp.metadata.created = feature_vector_record.created
@@ -3179,26 +3244,55 @@ class SQLDB(DBInterface):
 
     def _transform_project_record_to_schema(
         self, session: Session, project_record: Project
-    ) -> schemas.Project:
+    ) -> mlrun.common.schemas.Project:
         # in projects that was created before 0.6.0 the full object wasn't created properly - fix that, and return
         if not project_record.full_object:
-            project = schemas.Project(
-                metadata=schemas.ProjectMetadata(
+            project = mlrun.common.schemas.Project(
+                metadata=mlrun.common.schemas.ProjectMetadata(
                     name=project_record.name,
                     created=project_record.created,
                 ),
-                spec=schemas.ProjectSpec(
+                spec=mlrun.common.schemas.ProjectSpec(
                     description=project_record.description,
                     source=project_record.source,
                 ),
-                status=schemas.ObjectStatus(
+                status=mlrun.common.schemas.ObjectStatus(
                     state=project_record.state,
                 ),
             )
             self.store_project(session, project_record.name, project)
             return project
         # TODO: handle transforming the functions/workflows/artifacts references to real objects
-        return schemas.Project(**project_record.full_object)
+        return mlrun.common.schemas.Project(**project_record.full_object)
+
+    def _transform_notification_record_to_spec_and_status(
+        self,
+        notification_record,
+    ) -> typing.Tuple[dict, dict]:
+        notification_spec = self._transform_notification_record_to_schema(
+            notification_record
+        ).to_dict()
+        notification_status = {
+            "status": notification_spec.pop("status", None),
+            "sent_time": notification_spec.pop("sent_time", None),
+        }
+        return notification_spec, notification_status
+
+    @staticmethod
+    def _transform_notification_record_to_schema(
+        notification_record,
+    ) -> mlrun.model.Notification:
+        return mlrun.model.Notification(
+            kind=notification_record.kind,
+            name=notification_record.name,
+            message=notification_record.message,
+            severity=notification_record.severity,
+            when=notification_record.when.split(","),
+            condition=notification_record.condition,
+            params=notification_record.params,
+            status=notification_record.status,
+            sent_time=notification_record.sent_time,
+        )
 
     def _move_and_reorder_table_items(
         self, session, moved_object, move_to=None, move_from=None
@@ -3226,11 +3320,9 @@ class SQLDB(DBInterface):
             else:
                 start, end = move_to, move_from - 1
 
-        query = session.query(MarketplaceSource).filter(
-            MarketplaceSource.index >= start
-        )
+        query = session.query(HubSource).filter(HubSource.index >= start)
         if end:
-            query = query.filter(MarketplaceSource.index <= end)
+            query = query.filter(HubSource.index <= end)
 
         for source_record in query:
             source_record.index = source_record.index + modifier
@@ -3245,54 +3337,54 @@ class SQLDB(DBInterface):
         session.commit()
 
     @staticmethod
-    def _transform_marketplace_source_record_to_schema(
-        marketplace_source_record: MarketplaceSource,
-    ) -> schemas.IndexedMarketplaceSource:
-        source_full_dict = marketplace_source_record.full_object
-        marketplace_source = schemas.MarketplaceSource(**source_full_dict)
-        return schemas.IndexedMarketplaceSource(
-            index=marketplace_source_record.index, source=marketplace_source
+    def _transform_hub_source_record_to_schema(
+        hub_source_record: HubSource,
+    ) -> mlrun.common.schemas.IndexedHubSource:
+        source_full_dict = hub_source_record.full_object
+        hub_source = mlrun.common.schemas.HubSource(**source_full_dict)
+        return mlrun.common.schemas.IndexedHubSource(
+            index=hub_source_record.index, source=hub_source
         )
 
     @staticmethod
-    def _transform_marketplace_source_schema_to_record(
-        marketplace_source_schema: schemas.IndexedMarketplaceSource,
-        current_object: MarketplaceSource = None,
+    def _transform_hub_source_schema_to_record(
+        hub_source_schema: mlrun.common.schemas.IndexedHubSource,
+        current_object: HubSource = None,
     ):
         now = datetime.now(timezone.utc)
         if current_object:
-            if current_object.name != marketplace_source_schema.source.metadata.name:
+            if current_object.name != hub_source_schema.source.metadata.name:
                 raise mlrun.errors.MLRunInternalServerError(
                     "Attempt to update object while replacing its name"
                 )
             created_timestamp = current_object.created
         else:
-            created_timestamp = marketplace_source_schema.source.metadata.created or now
-        updated_timestamp = marketplace_source_schema.source.metadata.updated or now
+            created_timestamp = hub_source_schema.source.metadata.created or now
+        updated_timestamp = hub_source_schema.source.metadata.updated or now
 
-        marketplace_source_record = MarketplaceSource(
+        hub_source_record = HubSource(
             id=current_object.id if current_object else None,
-            name=marketplace_source_schema.source.metadata.name,
-            index=marketplace_source_schema.index,
+            name=hub_source_schema.source.metadata.name,
+            index=hub_source_schema.index,
             created=created_timestamp,
             updated=updated_timestamp,
         )
-        full_object = marketplace_source_schema.source.dict()
+        full_object = hub_source_schema.source.dict()
         full_object["metadata"]["created"] = str(created_timestamp)
         full_object["metadata"]["updated"] = str(updated_timestamp)
-        # Make sure we don't keep any credentials in the DB. These are handled in the marketplace crud object.
+        # Make sure we don't keep any credentials in the DB. These are handled in the hub crud object.
         full_object["spec"].pop("credentials", None)
 
-        marketplace_source_record.full_object = full_object
-        return marketplace_source_record
+        hub_source_record.full_object = full_object
+        return hub_source_record
 
     @staticmethod
-    def _validate_and_adjust_marketplace_order(session, order):
-        max_order = session.query(func.max(MarketplaceSource.index)).scalar()
+    def _validate_and_adjust_hub_order(session, order):
+        max_order = session.query(func.max(HubSource.index)).scalar()
         if not max_order or max_order < 0:
             max_order = 0
 
-        if order == schemas.marketplace.last_source_index:
+        if order == mlrun.common.schemas.hub.last_source_index:
             order = max_order + 1
 
         if order > max_order + 1:
@@ -3302,62 +3394,54 @@ class SQLDB(DBInterface):
         if order < 1:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Order of inserted source must be greater than 0 or "
-                + f"{schemas.marketplace.last_source_index} (for last). order = {order}"
+                + f"{mlrun.common.schemas.hub.last_source_index} (for last). order = {order}"
             )
         return order
 
-    def create_marketplace_source(
-        self, session, ordered_source: schemas.IndexedMarketplaceSource
+    def create_hub_source(
+        self, session, ordered_source: mlrun.common.schemas.IndexedHubSource
     ):
         logger.debug(
-            "Creating marketplace source in DB",
+            "Creating hub source in DB",
             index=ordered_source.index,
             name=ordered_source.source.metadata.name,
         )
 
-        order = self._validate_and_adjust_marketplace_order(
-            session, ordered_source.index
-        )
+        order = self._validate_and_adjust_hub_order(session, ordered_source.index)
         name = ordered_source.source.metadata.name
-        source_record = self._query(session, MarketplaceSource, name=name).one_or_none()
+        source_record = self._query(session, HubSource, name=name).one_or_none()
         if source_record:
             raise mlrun.errors.MLRunConflictError(
-                f"Marketplace source name already exists. name = {name}"
+                f"Hub source name already exists. name = {name}"
             )
-        source_record = self._transform_marketplace_source_schema_to_record(
-            ordered_source
-        )
+        source_record = self._transform_hub_source_schema_to_record(ordered_source)
 
         self._move_and_reorder_table_items(
             session, source_record, move_to=order, move_from=None
         )
 
     @retry_on_conflict
-    def store_marketplace_source(
+    def store_hub_source(
         self,
         session,
         name,
-        ordered_source: schemas.IndexedMarketplaceSource,
+        ordered_source: mlrun.common.schemas.IndexedHubSource,
     ):
-        logger.debug(
-            "Storing marketplace source in DB", index=ordered_source.index, name=name
-        )
+        logger.debug("Storing hub source in DB", index=ordered_source.index, name=name)
 
         if name != ordered_source.source.metadata.name:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Conflict between resource name and metadata.name in the stored object"
             )
-        order = self._validate_and_adjust_marketplace_order(
-            session, ordered_source.index
-        )
+        order = self._validate_and_adjust_hub_order(session, ordered_source.index)
 
-        source_record = self._query(session, MarketplaceSource, name=name).one_or_none()
+        source_record = self._query(session, HubSource, name=name).one_or_none()
         current_order = source_record.index if source_record else None
-        if current_order == schemas.marketplace.last_source_index:
+        if current_order == mlrun.common.schemas.hub.last_source_index:
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "Attempting to modify the global marketplace source."
+                "Attempting to modify the global hub source."
             )
-        source_record = self._transform_marketplace_source_schema_to_record(
+        source_record = self._transform_hub_source_schema_to_record(
             ordered_source, source_record
         )
 
@@ -3365,47 +3449,46 @@ class SQLDB(DBInterface):
             session, source_record, move_to=order, move_from=current_order
         )
 
-    def list_marketplace_sources(
-        self, session
-    ) -> List[schemas.IndexedMarketplaceSource]:
+    def list_hub_sources(self, session) -> List[mlrun.common.schemas.IndexedHubSource]:
         results = []
-        query = self._query(session, MarketplaceSource).order_by(
-            MarketplaceSource.index.desc()
-        )
+        query = self._query(session, HubSource).order_by(HubSource.index.desc())
         for record in query:
-            ordered_source = self._transform_marketplace_source_record_to_schema(record)
+            ordered_source = self._transform_hub_source_record_to_schema(record)
             # Need this to make the list return such that the default source is last in the response.
-            if ordered_source.index != schemas.last_source_index:
+            if ordered_source.index != mlrun.common.schemas.last_source_index:
                 results.insert(0, ordered_source)
             else:
                 results.append(ordered_source)
         return results
 
-    def delete_marketplace_source(self, session, name):
-        logger.debug("Deleting marketplace source from DB", name=name)
+    def _list_hub_sources_without_transform(self, session) -> List[HubSource]:
+        return self._query(session, HubSource).all()
 
-        source_record = self._query(session, MarketplaceSource, name=name).one_or_none()
+    def delete_hub_source(self, session, name):
+        logger.debug("Deleting hub source from DB", name=name)
+
+        source_record = self._query(session, HubSource, name=name).one_or_none()
         if not source_record:
             return
 
         current_order = source_record.index
-        if current_order == schemas.marketplace.last_source_index:
+        if current_order == mlrun.common.schemas.hub.last_source_index:
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "Attempting to delete the global marketplace source."
+                "Attempting to delete the global hub source."
             )
 
         self._move_and_reorder_table_items(
             session, source_record, move_to=None, move_from=current_order
         )
 
-    def get_marketplace_source(self, session, name) -> schemas.IndexedMarketplaceSource:
-        source_record = self._query(session, MarketplaceSource, name=name).one_or_none()
+    def get_hub_source(self, session, name) -> mlrun.common.schemas.IndexedHubSource:
+        source_record = self._query(session, HubSource, name=name).one_or_none()
         if not source_record:
             raise mlrun.errors.MLRunNotFoundError(
-                f"Marketplace source not found. name = {name}"
+                f"Hub source not found. name = {name}"
             )
 
-        return self._transform_marketplace_source_record_to_schema(source_record)
+        return self._transform_hub_source_record_to_schema(source_record)
 
     def get_current_data_version(
         self, session, raise_on_not_found=True
@@ -3443,7 +3526,7 @@ class SQLDB(DBInterface):
         session,
         name: str,
         project: str,
-        state: str = mlrun.api.schemas.BackgroundTaskState.running,
+        state: str = mlrun.common.schemas.BackgroundTaskState.running,
         timeout: int = None,
     ):
         background_task_record = self._query(
@@ -3457,7 +3540,7 @@ class SQLDB(DBInterface):
             # we don't want to be able to change state after it reached terminal state
             if (
                 background_task_record.state
-                in mlrun.api.schemas.BackgroundTaskState.terminal_states()
+                in mlrun.common.schemas.BackgroundTaskState.terminal_states()
                 and state != background_task_record.state
             ):
                 raise mlrun.errors.MLRunRuntimeError(
@@ -3484,7 +3567,7 @@ class SQLDB(DBInterface):
 
     def get_background_task(
         self, session, name: str, project: str
-    ) -> schemas.BackgroundTask:
+    ) -> mlrun.common.schemas.BackgroundTask:
         background_task_record = self._get_background_task_record(
             session, name, project
         )
@@ -3495,7 +3578,7 @@ class SQLDB(DBInterface):
                 session,
                 name,
                 project,
-                mlrun.api.schemas.background_task.BackgroundTaskState.failed,
+                mlrun.common.schemas.background_task.BackgroundTaskState.failed,
             )
             background_task_record = self._get_background_task_record(
                 session, name, project
@@ -3506,17 +3589,17 @@ class SQLDB(DBInterface):
     @staticmethod
     def _transform_background_task_record_to_schema(
         background_task_record: BackgroundTask,
-    ) -> schemas.BackgroundTask:
-        return schemas.BackgroundTask(
-            metadata=schemas.BackgroundTaskMetadata(
+    ) -> mlrun.common.schemas.BackgroundTask:
+        return mlrun.common.schemas.BackgroundTask(
+            metadata=mlrun.common.schemas.BackgroundTaskMetadata(
                 name=background_task_record.name,
                 project=background_task_record.project,
                 created=background_task_record.created,
                 updated=background_task_record.updated,
                 timeout=background_task_record.timeout,
             ),
-            spec=schemas.BackgroundTaskSpec(),
-            status=schemas.BackgroundTaskStatus(
+            spec=mlrun.common.schemas.BackgroundTaskSpec(),
+            status=mlrun.common.schemas.BackgroundTaskStatus(
                 state=background_task_record.state,
             ),
         )
@@ -3569,9 +3652,169 @@ class SQLDB(DBInterface):
         if (
             timeout
             and background_task_record.state
-            not in mlrun.api.schemas.BackgroundTaskState.terminal_states()
+            not in mlrun.common.schemas.BackgroundTaskState.terminal_states()
             and datetime.utcnow()
             > timedelta(seconds=int(timeout)) + background_task_record.updated
         ):
             return True
         return False
+
+    def store_run_notifications(
+        self,
+        session,
+        notification_objects: typing.List[mlrun.model.Notification],
+        run_uid: str,
+        project: str,
+    ):
+        # iteration is 0, as we don't support multiple notifications per hyper param run, only for the whole run
+        run = self._get_run(session, run_uid, project, 0)
+        if not run:
+            raise mlrun.errors.MLRunNotFoundError(
+                f"Run not found: uid={run_uid}, project={project}"
+            )
+
+        self._store_notifications(session, Run, notification_objects, run.id, project)
+
+    def _store_notifications(
+        self,
+        session,
+        cls,
+        notification_objects: typing.List[mlrun.model.Notification],
+        parent_id: str,
+        project: str,
+    ):
+        db_notifications = {
+            notification.name: notification
+            for notification in self._get_db_notifications(
+                session, cls, parent_id=parent_id
+            )
+        }
+        notifications = []
+        logger.debug(
+            "Storing notifications",
+            notifications_length=len(notification_objects),
+            parent_id=parent_id,
+            project=project,
+        )
+        for notification_model in notification_objects:
+            new_notification = False
+            notification = db_notifications.get(notification_model.name, None)
+            if not notification:
+                new_notification = True
+                notification = cls.Notification(
+                    name=notification_model.name, parent_id=parent_id, project=project
+                )
+
+            notification.kind = notification_model.kind
+            notification.message = notification_model.message
+            notification.severity = notification_model.severity
+            notification.when = ",".join(notification_model.when)
+            notification.condition = notification_model.condition
+            notification.params = notification_model.params
+            notification.status = (
+                notification_model.status
+                or mlrun.common.schemas.NotificationStatus.PENDING
+            )
+            notification.sent_time = notification_model.sent_time
+
+            logger.debug(
+                f"Storing {'new' if new_notification else 'existing'} notification",
+                notification_name=notification.name,
+                notification_status=notification.status,
+                parent_id=parent_id,
+                project=project,
+            )
+            notifications.append(notification)
+
+        self._upsert(session, notifications)
+
+    def list_run_notifications(
+        self,
+        session,
+        run_uid: str,
+        project: str = "",
+    ) -> typing.List[mlrun.model.Notification]:
+
+        # iteration is 0, as we don't support multiple notifications per hyper param run, only for the whole run
+        run = self._get_run(session, run_uid, project, 0)
+        if not run:
+            return []
+
+        return [
+            self._transform_notification_record_to_schema(notification)
+            for notification in self._query(
+                session, Run.Notification, parent_id=run.id
+            ).all()
+        ]
+
+    def delete_run_notifications(
+        self,
+        session,
+        name: str = None,
+        run_uid: str = None,
+        project: str = None,
+        commit: bool = True,
+    ):
+        run_id = None
+        if run_uid:
+
+            # iteration is 0, as we don't support multiple notifications per hyper param run, only for the whole run
+            run = self._get_run(session, run_uid, project, 0)
+            if not run:
+                raise mlrun.errors.MLRunNotFoundError(
+                    f"Run not found: uid={run_uid}, project={project}"
+                )
+            run_id = run.id
+
+        project = project or config.default_project
+        if project == "*":
+            project = None
+
+        query = self._get_db_notifications(session, Run, name, run_id, project)
+        for notification in query:
+            session.delete(notification)
+
+        if commit:
+            session.commit()
+
+    def set_run_notifications(
+        self,
+        session: Session,
+        project: str,
+        notifications: typing.List[mlrun.model.Notification],
+        identifier: mlrun.common.schemas.RunIdentifier,
+        **kwargs,
+    ):
+        """
+        Set notifications for a run. This will replace any existing notifications.
+        :param session: SQLAlchemy session
+        :param project: Project name
+        :param notifications: List of notifications to set
+        :param identifier: Run identifier
+        :param kwargs: Ignored additional arguments (for interfacing purposes)
+        """
+        run = self._get_run(session, identifier.uid, project, None)
+        if not run:
+            raise mlrun.errors.MLRunNotFoundError(
+                f"Run not found: project={project}, uid={identifier.uid}"
+            )
+
+        run.struct.setdefault("spec", {})["notifications"] = [
+            notification.to_dict() for notification in notifications
+        ]
+
+        # update run, delete and store notifications all in one transaction.
+        # using session.add instead of upsert, so we don't commit the run.
+        # the commit will happen at the end (in store_run_notifications, or manually at the end).
+        session.add(run)
+        self.delete_run_notifications(
+            session, run_uid=run.uid, project=project, commit=False
+        )
+        if notifications:
+            self.store_run_notifications(
+                session,
+                notification_objects=notifications,
+                run_uid=run.uid,
+                project=project,
+            )
+        self._commit(session, [run], ignore=True)