mlrun 1.3.1rc5__py3-none-any.whl → 1.4.0rc2__py3-none-any.whl

mlrun/__main__.py

@@ -1052,6 +1052,15 @@ def logs(uid, project, offset, db, watch):
     is_flag=True,
     help="Store the project secrets as k8s secrets",
 )
+@click.option(
+    "--notifications",
+    "--notification",
+    "-nt",
+    multiple=True,
+    help="To have a notification for the run set notification file "
+    "destination define: file=notification.json or a "
+    'dictionary configuration e.g \'{"slack":{"webhook":"<webhook>"}}\'',
+)
 def project(
     context,
     name,
@@ -1077,6 +1086,7 @@ def project(
     timeout,
     ensure_project,
     schedule,
+    notifications,
     overwrite_schedule,
     save_secrets,
     save,
@@ -1152,6 +1162,8 @@ def project(
                     "token": proj.get_param("GIT_TOKEN"),
                 },
             )
+        if notifications:
+            load_notification(notifications, proj)
         try:
             proj.run(
                 name=run,
@@ -1169,11 +1181,9 @@ def project(
                 timeout=timeout,
                 overwrite=overwrite_schedule,
             )
-
-        except Exception as exc:
+        except Exception as err:
             print(traceback.format_exc())
-            message = f"failed to run pipeline, {err_to_str(exc)}"
-            proj.notifiers.push(message, "error")
+            send_workflow_error_notification(run, proj, err)
             exit(1)
 
     elif sync:
@@ -1450,5 +1460,48 @@ def func_url_to_runtime(func_url, ensure_project: bool = False):
     return runtime
 
 
+def load_notification(notifications: str, project: mlrun.projects.MlrunProject):
+    """
+    A dictionary or json file containing notification dictionaries can be used by the user to set notifications.
+    Each notification is stored in a tuple called notifications.
+    The code then goes through each value in the notifications tuple and check
+    if the notification starts with "file=", such as "file=notification.json," in those cases it loads the
+    notification.json file and uses add_notification_to_project to add the notifications from the file to
+    the project. If not, it adds the notification dictionary to the project.
+    :param notifications:  Notifications file or a dictionary to be added to the project
+    :param project: The object to which the notifications will be added
+    :return:
+    """
+    for notification in notifications:
+        if notification.startswith("file="):
+            file_path = notification.split("=")[-1]
+            notification = open(file_path, "r")
+            notification = json.load(notification)
+        else:
+            notification = json.loads(notification)
+        add_notification_to_project(notification, project)
+
+
+def add_notification_to_project(
+    notification: str, project: mlrun.projects.MlrunProject
+):
+    for notification_type, notification_params in notification.items():
+        project.notifiers.add_notification(
+            notification_type=notification_type, params=notification_params
+        )
+
+
+def send_workflow_error_notification(
+    run_id: str, project: mlrun.projects.MlrunProject, error: KeyError
+):
+    message = (
+        f":x: Failed to run scheduled workflow {run_id} in Project {project.name} !\n"
+        f"error: ```{err_to_str(error)}```"
+    )
+    project.notifiers.push(
+        message=message, severity=mlrun.api.schemas.NotificationSeverity.ERROR
+    )
+
+
 if __name__ == "__main__":
     main()

mlrun/api/api/endpoints/marketplace.py

@@ -159,7 +159,6 @@ async def store_source(
 )
 async def get_catalog(
     source_name: str,
-    channel: Optional[str] = Query(None),
     version: Optional[str] = Query(None),
     tag: Optional[str] = Query(None),
     force_refresh: Optional[bool] = Query(False, alias="force-refresh"),
@@ -180,7 +179,6 @@ async def get_catalog(
     return await run_in_threadpool(
         mlrun.api.crud.Marketplace().get_source_catalog,
         ordered_source.source,
-        channel,
         version,
         tag,
         force_refresh,
@@ -194,7 +192,6 @@ async def get_catalog(
 async def get_item(
     source_name: str,
     item_name: str,
-    channel: Optional[str] = Query("development"),
     version: Optional[str] = Query(None),
     tag: Optional[str] = Query("latest"),
     force_refresh: Optional[bool] = Query(False, alias="force-refresh"),
@@ -216,7 +213,6 @@ async def get_item(
         mlrun.api.crud.Marketplace().get_item,
         ordered_source.source,
         item_name,
-        channel,
         version,
         tag,
         force_refresh,
@@ -255,3 +251,60 @@ async def get_object(
     if not ctype:
         ctype = "application/octet-stream"
     return Response(content=object_data, media_type=ctype)
+
+
+@router.get("/marketplace/sources/{source_name}/items/{item_name}/assets/{asset_name}")
+async def get_asset(
+    source_name: str,
+    item_name: str,
+    asset_name: str,
+    tag: Optional[str] = Query("latest"),
+    version: Optional[str] = Query(None),
+    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
+    auth_info: mlrun.api.schemas.AuthInfo = Depends(
+        mlrun.api.api.deps.authenticate_request
+    ),
+):
+    """
+    Retrieve asset from a specific item in specific marketplace source.
+
+    :param source_name: marketplace source name
+    :param item_name:   the name of the item
+    :param asset_name:  the name of the asset to retrieve
+    :param tag:         tag of item - latest or version number
+    :param version:     item version
+    :param db_session:  a session that manages the current dialog with the database
+    :param auth_info:   the auth info of the request
+
+    :return: fastapi response with the asset in content
+    """
+    source = await run_in_threadpool(
+        get_db().get_marketplace_source, db_session, source_name
+    )
+
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
+        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
+        AuthorizationAction.read,
+        auth_info,
+    )
+    # Getting the relevant item which hold the asset information
+    item = await run_in_threadpool(
+        mlrun.api.crud.Marketplace().get_item,
+        source.source,
+        item_name,
+        version,
+        tag,
+    )
+
+    # Getting the asset from the item
+    asset, url = await run_in_threadpool(
+        mlrun.api.crud.Marketplace().get_asset,
+        source.source,
+        item,
+        asset_name,
+    )
+
+    ctype, _ = mimetypes.guess_type(url)
+    if not ctype:
+        ctype = "application/octet-stream"
+    return Response(content=asset, media_type=ctype)

mlrun/api/api/endpoints/runs.py

@@ -178,6 +178,7 @@ async def list_runs(
         mlrun.api.schemas.OrderType.desc, alias="partition-order"
     ),
     max_partitions: int = Query(0, alias="max-partitions", ge=0),
+    with_notifications: bool = Query(False, alias="with-notifications"),
     auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
     db_session: Session = Depends(deps.get_db_session),
 ):
@@ -207,6 +208,7 @@ async def list_runs(
         partition_sort_by,
         partition_order,
         max_partitions,
+        with_notifications=with_notifications,
     )
     filtered_runs = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
         mlrun.api.schemas.AuthorizationResourceTypes.run,

mlrun/api/api/utils.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 #
 import collections
+import json
 import re
 import traceback
 import typing
@@ -29,6 +30,7 @@ from sqlalchemy.orm import Session
 import mlrun.api.crud
 import mlrun.api.utils.auth.verifier
 import mlrun.api.utils.clients.iguazio
+import mlrun.api.utils.singletons.k8s
 import mlrun.errors
 import mlrun.runtimes.pod
 import mlrun.utils.helpers
@@ -185,6 +187,7 @@ def _generate_function_and_task_from_submit_run_body(
             function = enrich_function_from_dict(function, function_dict)
 
     apply_enrichment_and_validation_on_function(function, auth_info)
+    apply_enrichment_and_validation_on_task(task)
 
     return function, task
 
@@ -196,6 +199,105 @@ async def submit_run(db_session: Session, auth_info: mlrun.api.schemas.AuthInfo,
     return response
 
 
+def apply_enrichment_and_validation_on_task(task):
+
+    # Masking notification config params from the task object
+    mask_notification_params_on_task(task)
+
+
+def mask_notification_params_on_task(task):
+    run_uid = get_in(task, "metadata.uid")
+    project = get_in(task, "metadata.project")
+    notifications = task.get("spec", {}).get("notifications", [])
+    if notifications:
+        for notification in notifications:
+            notification_object = mlrun.model.Notification.from_dict(notification)
+            mask_notification_params_with_secret(project, run_uid, notification_object)
+
+
+def mask_notification_params_with_secret(
+    project: str, run_uid: str, notification_object: mlrun.model.Notification
+) -> mlrun.model.Notification:
+    if notification_object.params and "secret" not in notification_object.params:
+        secret_key = mlrun.api.crud.Secrets().generate_client_project_secret_key(
+            mlrun.api.crud.SecretsClientType.notifications,
+            run_uid,
+            notification_object.name,
+        )
+        mlrun.api.crud.Secrets().store_project_secrets(
+            project,
+            mlrun.api.schemas.SecretsData(
+                provider=mlrun.api.schemas.SecretProviderName.kubernetes,
+                secrets={secret_key: json.dumps(notification_object.params)},
+            ),
+            allow_internal_secrets=True,
+        )
+        notification_object.params = {"secret": secret_key}
+
+    return notification_object
+
+
+def unmask_notification_params_secret_on_task(run):
+    if isinstance(run, dict):
+        run = mlrun.model.RunObject.from_dict(run)
+
+    run.spec.notifications = [
+        unmask_notification_params_secret(run.metadata.project, notification)
+        for notification in run.spec.notifications
+    ]
+    return run
+
+
+def unmask_notification_params_secret(
+    project: str, notification_object: mlrun.model.Notification
+) -> mlrun.model.Notification:
+    params = notification_object.params or {}
+    params_secret = params.get("secret", "")
+    if not params_secret:
+        return notification_object
+
+    k8s = mlrun.api.utils.singletons.k8s.get_k8s()
+    if not k8s:
+        raise mlrun.errors.MLRunRuntimeError(
+            "Not running in k8s environment, cannot load notification params secret"
+        )
+
+    notification_object.params = json.loads(
+        mlrun.api.crud.Secrets().get_project_secret(
+            project,
+            mlrun.api.schemas.SecretProviderName.kubernetes,
+            secret_key=params_secret,
+            allow_internal_secrets=True,
+            allow_secrets_from_k8s=True,
+        )
+    )
+
+    return notification_object
+
+
+def delete_notification_params_secret(
+    project: str, notification_object: mlrun.model.Notification
+) -> None:
+    params = notification_object.params or {}
+    params_secret = params.get("secret", "")
+    if not params_secret:
+        return
+
+    k8s = mlrun.api.utils.singletons.k8s.get_k8s()
+    if not k8s:
+        raise mlrun.errors.MLRunRuntimeError(
+            "Not running in k8s environment, cannot delete notification params secret"
+        )
+
+    mlrun.api.crud.Secrets().delete_project_secret(
+        project,
+        mlrun.api.schemas.SecretProviderName.kubernetes,
+        secret_key=params_secret,
+        allow_internal_secrets=True,
+        allow_secrets_from_k8s=True,
+    )
+
+
 def apply_enrichment_and_validation_on_function(
     function,
     auth_info: mlrun.api.schemas.AuthInfo,

mlrun/api/crud/__init__.py

@@ -22,6 +22,7 @@ from .functions import Functions
 from .logs import Logs
 from .marketplace import Marketplace
 from .model_monitoring import ModelEndpoints
+from .notifications import Notifications
 from .pipelines import Pipelines
 from .projects import Projects
 from .runs import Runs

mlrun/api/crud/marketplace.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 #
 import json
+from typing import Any, Dict, List, Optional, Tuple
 
 import mlrun.errors
 import mlrun.utils.singleton
@@ -121,43 +122,79 @@ class Marketplace(metaclass=mlrun.utils.singleton.Singleton):
         return source_secrets
 
     @staticmethod
-    def _transform_catalog_dict_to_schema(source, catalog_dict):
-        catalog_dict = catalog_dict.get("functions")
-        if not catalog_dict:
-            raise mlrun.errors.MLRunInternalServerError(
-                "Invalid catalog file - no 'functions' section found."
+    def _get_asset_full_path(
+        source: MarketplaceSource, item: MarketplaceItem, asset: str
+    ):
+        """
+        Combining the item path with the asset path.
+
+        :param source:  Marketplace source object.
+        :param item:    The relevant item to get the asset from.
+        :param asset:   The asset name
+        :return:    Full path to the asset, relative to the item directory.
+        """
+        asset_path = item.spec.assets.get(asset, None)
+        if not asset_path:
+            raise mlrun.errors.MLRunNotFoundError(
+                f"Asset={asset} not found. "
+                f"item={item.metadata.name}, version={item.metadata.version}, tag={item.metadata.tag}"
             )
+        item_path = item.metadata.get_relative_path()
+        return source.get_full_uri(item_path + asset_path)
 
-        catalog = MarketplaceCatalog(catalog=[])
-        # Loop over channels, then per function extract versions.
-        for channel_name in catalog_dict:
-            channel_dict = catalog_dict[channel_name]
-            for function_name in channel_dict:
-                function_dict = channel_dict[function_name]
-                for version_tag in function_dict:
-                    version_dict = function_dict[version_tag]
-                    function_details_dict = version_dict.copy()
-                    spec_dict = function_details_dict.pop("spec", None)
-                    metadata = MarketplaceItemMetadata(
-                        channel=channel_name, tag=version_tag, **function_details_dict
-                    )
-                    item_uri = source.get_full_uri(metadata.get_relative_path())
-                    spec = MarketplaceItemSpec(item_uri=item_uri, **spec_dict)
-                    item = MarketplaceItem(
-                        metadata=metadata, spec=spec, status=ObjectStatus()
-                    )
-                    catalog.catalog.append(item)
+    @staticmethod
+    def _transform_catalog_dict_to_schema(
+        source: MarketplaceSource, catalog_dict: Dict[str, Any]
+    ) -> MarketplaceCatalog:
+        """
+        Transforms catalog dictionary to MarketplaceCatalog schema
+        :param source:          Marketplace source object.
+        :param catalog_dict:    raw catalog dict, top level keys are item names,
+                                second level keys are version tags ("latest, "1.1.0", ...) and
+                                bottom level keys include spec as a dict and all the rest is considered as metadata.
+        :return: catalog object
+        """
+        catalog = MarketplaceCatalog(catalog=[], channel=source.spec.channel)
+        # Loop over objects, then over object versions.
+        for object_name, object_dict in catalog_dict.items():
+            for version_tag, version_dict in object_dict.items():
+                object_details_dict = version_dict.copy()
+                spec_dict = object_details_dict.pop("spec", {})
+                assets = object_details_dict.pop("assets", {})
+                metadata = MarketplaceItemMetadata(
+                    tag=version_tag, **object_details_dict
+                )
+                item_uri = source.get_full_uri(metadata.get_relative_path())
+                spec = MarketplaceItemSpec(
+                    item_uri=item_uri, assets=assets, **spec_dict
+                )
+                item = MarketplaceItem(
+                    metadata=metadata,
+                    spec=spec,
+                    status=ObjectStatus(),
+                )
+                catalog.catalog.append(item)
 
         return catalog
 
     def get_source_catalog(
         self,
         source: MarketplaceSource,
-        channel=None,
-        version=None,
-        tag=None,
-        force_refresh=False,
+        version: Optional[str] = None,
+        tag: Optional[str] = None,
+        force_refresh: bool = False,
     ) -> MarketplaceCatalog:
+        """
+        Getting the catalog object by source.
+        If version and/or tag are given, the catalog will be filtered accordingly.
+
+        :param source:          Marketplace source object.
+        :param version:         version of items to filter by
+        :param tag:             tag of items to filter by
+        :param force_refresh:   if True, the catalog will be loaded from source always,
+                                otherwise will be pulled from db (if loaded before)
+        :return: catalog object
+        """
         source_name = source.metadata.name
         if not self._catalogs.get(source_name) or force_refresh:
             url = source.get_catalog_uri()
@@ -169,12 +206,12 @@ class Marketplace(metaclass=mlrun.utils.singleton.Singleton):
         else:
             catalog = self._catalogs[source_name]
 
-        result_catalog = MarketplaceCatalog(catalog=[])
+        result_catalog = MarketplaceCatalog(catalog=[], channel=source.spec.channel)
         for item in catalog.catalog:
-            if (
-                (channel is None or item.metadata.channel == channel)
-                and (tag is None or item.metadata.tag == tag)
-                and (version is None or item.metadata.version == version)
+            # Because tag and version are optionals,
+            # we filter the catalog by one of them with priority to tag
+            if (tag is None or item.metadata.tag == tag) and (
+                version is None or item.metadata.version == version
             ):
                 result_catalog.catalog.append(item)
 
@@ -183,25 +220,55 @@ class Marketplace(metaclass=mlrun.utils.singleton.Singleton):
     def get_item(
         self,
         source: MarketplaceSource,
-        item_name,
-        channel,
-        version=None,
-        tag=None,
-        force_refresh=False,
+        item_name: str,
+        version: Optional[str] = None,
+        tag: Optional[str] = None,
+        force_refresh: bool = False,
     ) -> MarketplaceItem:
-        catalog = self.get_source_catalog(source, channel, version, tag, force_refresh)
-        items = [item for item in catalog.catalog if item.metadata.name == item_name]
-        if not items:
+        """
+        Retrieve item from source. The item is filtered by tag and version.
+
+        :param source:          Marketplace source object
+        :param item_name:       name of the item to retrieve
+        :param version:         version of the item
+        :param tag:             tag of the item
+        :param force_refresh:   if True, the catalog will be loaded from source always,
+                                otherwise will be pulled from db (if loaded before)
+
+        :return: marketplace item object
+
+        :raise if the number of collected items from catalog is not exactly one.
+        """
+        catalog = self.get_source_catalog(source, version, tag, force_refresh)
+        items = self._get_catalog_items_filtered_by_name(catalog.catalog, item_name)
+        num_items = len(items)
+
+        if not num_items:
             raise mlrun.errors.MLRunNotFoundError(
-                f"Item not found. source={item_name}, channel={channel}, version={version}"
+                f"Item not found. source={item_name}, version={version}"
             )
-        if len(items) > 1:
+        if num_items > 1:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Query resulted in more than 1 catalog items. "
-                + f"source={item_name}, channel={channel}, version={version}, tag={tag}"
+                + f"source={item_name}, version={version}, tag={tag}"
             )
         return items[0]
 
+    @staticmethod
+    def _get_catalog_items_filtered_by_name(
+        catalog: List[MarketplaceItem],
+        item_name: str,
+    ) -> List[MarketplaceItem]:
+        """
+        Retrieve items from catalog filtered by name
+
+        :param catalog:     list of items
+        :param item_name:   item name to filter by
+
+        :return:   list of item objects from catalog
+        """
+        return [item for item in catalog if item.metadata.name == item_name]
+
     def get_item_object_using_source_credentials(self, source: MarketplaceSource, url):
         credentials = self._get_source_credentials(source.metadata.name)
 
@@ -219,3 +286,25 @@ class Marketplace(metaclass=mlrun.utils.singleton.Singleton):
         else:
             catalog_data = mlrun.run.get_object(url=url, secrets=credentials)
         return catalog_data
+
+    def get_asset(
+        self,
+        source: MarketplaceSource,
+        item: MarketplaceItem,
+        asset_name: str,
+    ) -> Tuple[bytes, str]:
+        """
+        Retrieve asset object from marketplace source.
+
+        :param source:      marketplace source
+        :param item:        marketplace item which contains the assets
+        :param asset_name:  asset name, like source, example, etc.
+
+        :return: tuple of asset as bytes and url of asset
+        """
+        credentials = self._get_source_credentials(source.metadata.name)
+        asset_path = self._get_asset_full_path(source, item, asset_name)
+        return (
+            mlrun.run.get_object(url=asset_path, secrets=credentials),
+            asset_path,
+        )

mlrun/api/crud/notifications.py

@@ -0,0 +1,80 @@
+# Copyright 2018 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import typing
+
+import sqlalchemy.orm
+
+import mlrun.api.api.utils
+import mlrun.api.utils.singletons.db
+import mlrun.utils.singleton
+
+
+class Notifications(
+    metaclass=mlrun.utils.singleton.Singleton,
+):
+    def store_run_notifications(
+        self,
+        session: sqlalchemy.orm.Session,
+        notification_objects: typing.List[mlrun.model.Notification],
+        run_uid: str,
+        project: str = None,
+    ):
+        project = project or mlrun.mlconf.default_project
+        notification_objects_to_store = []
+        for notification_object in notification_objects:
+            notification_objects_to_store.append(
+                mlrun.api.api.utils.mask_notification_params_with_secret(
+                    project, run_uid, notification_object
+                )
+            )
+
+        mlrun.api.utils.singletons.db.get_db().store_run_notifications(
+            session, notification_objects_to_store, run_uid, project
+        )
+
+    def list_run_notifications(
+        self,
+        session: sqlalchemy.orm.Session,
+        run_uid: str,
+        project: str = "",
+    ) -> typing.List[mlrun.model.Notification]:
+        project = project or mlrun.mlconf.default_project
+        return mlrun.api.utils.singletons.db.get_db().list_run_notifications(
+            session, run_uid, project
+        )
+
+    def delete_run_notifications(
+        self,
+        session: sqlalchemy.orm.Session,
+        name: str = None,
+        run_uid: str = None,
+        project: str = None,
+    ):
+        project = project or mlrun.mlconf.default_project
+
+        # Delete notification param project secret
+        notifications = [
+            notification
+            for notification in self.list_run_notifications(session, run_uid, project)
+            if notification.name == name
+        ]
+        if notifications:
+            # unique constraint on name, run_uid, project, so the list will contain one item at most
+            notification = notifications[0]
+            mlrun.api.api.utils.delete_notification_params_secret(project, notification)
+
+        mlrun.api.utils.singletons.db.get_db().delete_run_notifications(
+            session, name, run_uid, project
+        )

mlrun/api/crud/runs.py

@@ -127,6 +127,7 @@ class Runs(
         max_partitions: int = 0,
         requested_logs: bool = None,
         return_as_run_structs: bool = True,
+        with_notifications: bool = False,
     ):
         project = project or mlrun.mlconf.default_project
         return mlrun.api.utils.singletons.db.get_db().list_runs(
@@ -150,6 +151,7 @@ class Runs(
             max_partitions,
             requested_logs,
             return_as_run_structs,
+            with_notifications,
         )
 
     def delete_run(

mlrun/api/crud/secrets.py

@@ -31,6 +31,7 @@ class SecretsClientType(str, enum.Enum):
     model_monitoring = "model-monitoring"
     service_accounts = "service-accounts"
     marketplace = "marketplace"
+    notifications = "notifications"
 
 
 class Secrets(