mlrun 1.5.0rc5__py3-none-any.whl → 1.5.0rc6__py3-none-any.whl

mlrun/api/api/endpoints/datastore_profile.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 #
 
+from http import HTTPStatus
 
 from fastapi import APIRouter, Depends
 from fastapi.concurrency import run_in_threadpool
@@ -24,12 +25,13 @@ import mlrun.api.crud
 import mlrun.api.utils.auth.verifier
 import mlrun.api.utils.singletons.db
 import mlrun.common.schemas
+from mlrun.api.api.utils import log_and_raise
 
 router = APIRouter()
 
 
 @router.put(
-    path="/projects/{project_name}/datastore_profiles",
+    path="/projects/{project_name}/datastore-profiles",
 )
 async def store_datastore_profile(
     project_name: str,
@@ -45,13 +47,20 @@ async def store_datastore_profile(
         project_name,
         auth_info.session,
     )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
         mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
-        mlrun.common.schemas.AuthorizationAction.create,
+        project_name,
+        info.name,
+        mlrun.common.schemas.AuthorizationAction.store,
         auth_info,
     )
     # overwrite the project
-    info.project = project_name
+    if info.project != project_name:
+        log_and_raise(
+            HTTPStatus.BAD_REQUEST.value,
+            reason="The project name provided in the URI does not match the one specified in the DatastoreProfile",
+        )
+
     await run_in_threadpool(
         mlrun.api.utils.singletons.db.get_db().store_datastore_profile,
         db_session,
@@ -67,7 +76,7 @@ async def store_datastore_profile(
 
 
 @router.get(
-    path="/projects/{project_name}/datastore_profiles",
+    path="/projects/{project_name}/datastore-profiles",
 )
 async def list_datastore_profiles(
     project_name: str,
@@ -82,20 +91,29 @@ async def list_datastore_profiles(
         project_name,
         auth_info.session,
     )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
+        project_name,
         mlrun.common.schemas.AuthorizationAction.read,
         auth_info,
     )
-    return await run_in_threadpool(
+    profiles = await run_in_threadpool(
         mlrun.api.utils.singletons.db.get_db().list_datastore_profiles,
         db_session,
         project_name,
     )
+    if len(profiles) == 0:
+        return profiles
+    filtered_data = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
+        mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
+        profiles,
+        lambda profile: (project_name, profile.name),
+        auth_info,
+    )
+    return filtered_data
 
 
 @router.get(
-    path="/projects/{project_name}/datastore_profiles/{profile}",
+    path="/projects/{project_name}/datastore-profiles/{profile}",
 )
 async def get_datastore_profile(
     project_name: str,
@@ -111,8 +129,10 @@ async def get_datastore_profile(
         project_name,
         auth_info.session,
     )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
         mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
+        project_name,
+        profile,
         mlrun.common.schemas.AuthorizationAction.read,
         auth_info,
     )
@@ -125,7 +145,7 @@ async def get_datastore_profile(
 
 
 @router.delete(
-    path="/projects/{project_name}/datastore_profiles/{profile}",
+    path="/projects/{project_name}/datastore-profiles/{profile}",
 )
 async def delete_datastore_profile(
     project_name: str,
@@ -141,9 +161,11 @@ async def delete_datastore_profile(
         project_name,
         auth_info.session,
     )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
+    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
         mlrun.common.schemas.AuthorizationResourceTypes.datastore_profile,
-        mlrun.common.schemas.AuthorizationAction.read,
+        project_name,
+        profile,
+        mlrun.common.schemas.AuthorizationAction.delete,
         auth_info,
     )
     return await run_in_threadpool(

mlrun/api/api/endpoints/frontend_spec.py

@@ -73,7 +73,7 @@ def get_frontend_spec(
         function_deployment_target_image_template=function_deployment_target_image_template,
         function_deployment_target_image_name_prefix_template=function_target_image_name_prefix_template,
         function_deployment_target_image_registries_to_enforce_prefix=registries_to_enforce_prefix,
-        function_deployment_mlrun_command=_resolve_function_deployment_mlrun_command(),
+        function_deployment_mlrun_requirement=mlrun.api.utils.builder.resolve_mlrun_install_command_version(),
         auto_mount_type=config.storage.auto_mount_type,
         auto_mount_params=config.get_storage_auto_mount_params(),
         default_artifact_path=config.artifact_path,
@@ -87,15 +87,6 @@ def get_frontend_spec(
     )
 
 
-def _resolve_function_deployment_mlrun_command():
-    # TODO: When UI adds a requirements section, mlrun should be specified there instead of the commands section i.e.
-    #  frontend spec will contain only the mlrun_version_specifier instead of the full command
-    mlrun_version_specifier = (
-        mlrun.api.utils.builder.resolve_mlrun_install_command_version()
-    )
-    return f'python -m pip install "{mlrun_version_specifier}"'
-
-
 def _resolve_jobs_dashboard_url(session: str) -> typing.Optional[str]:
     iguazio_client = mlrun.api.utils.clients.iguazio.Client()
     grafana_service_url = iguazio_client.try_get_grafana_service_url(session)

mlrun/api/api/endpoints/functions.py

@@ -267,7 +267,7 @@ async def build_function(
     except ValueError:
         log_and_raise(HTTPStatus.BAD_REQUEST.value, reason="bad JSON body")
 
-    logger.info(f"build_function:\n{data}")
+    logger.info("Building function", data=data)
     function = data.get("function")
     project = function.get("metadata", {}).get("project", mlrun.mlconf.default_project)
     function_name = function.get("metadata", {}).get("name")

mlrun/api/api/endpoints/hub.py

@@ -81,13 +81,9 @@ async def list_sources(
         auth_info,
     )
 
-    hub_sources = await run_in_threadpool(
-        mlrun.api.utils.singletons.db.get_db().list_hub_sources, db_session
-    )
-
     return await run_in_threadpool(
-        mlrun.api.crud.Hub().filter_hub_sources,
-        hub_sources,
+        mlrun.api.crud.Hub().list_hub_sources,
+        db_session,
         item_name,
         tag,
         version,

mlrun/api/crud/client_spec.py

@@ -104,6 +104,9 @@ class ClientSpec(
             model_endpoint_monitoring_store_type=self._get_config_value_if_not_default(
                 "model_endpoint_monitoring.store_type"
             ),
+            model_endpoint_monitoring_endpoint_store_connection=self._get_config_value_if_not_default(
+                "model_endpoint_monitoring.endpoint_store_connection"
+            ),
             packagers=self._get_config_value_if_not_default("packagers"),
         )
 

mlrun/api/crud/datastore_profiles.py

@@ -37,7 +37,7 @@ class DatastoreProfiles(
     def _store_secret(self, project, profile_name, profile_secret_json):
         if not self._in_k8s():
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "MLRun is not configured with k8s, hub source credentials cannot be stored securely"
+                "MLRun is not configured with k8s, datastore profile credentials cannot be stored securely"
             )
 
         adjusted_secret = {
@@ -56,7 +56,7 @@ class DatastoreProfiles(
     def _delete_secret(self, project, profile_name):
         if not self._in_k8s():
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "MLRun is not configured with k8s, hub source credentials cannot be stored securely"
+                "MLRun is not configured with k8s, datastore profile credentials cannot be deleted"
             )
 
         adjusted_secret = dsp.generate_secret_key(profile_name, project)

mlrun/api/crud/hub.py

@@ -15,6 +15,9 @@
 import json
 from typing import Any, Dict, List, Optional, Tuple
 
+import sqlalchemy.orm
+
+import mlrun.api.utils.singletons.db
 import mlrun.api.utils.singletons.k8s
 import mlrun.common.schemas
 import mlrun.common.schemas.hub
@@ -35,20 +38,6 @@ class Hub(metaclass=mlrun.utils.singleton.Singleton):
         self._internal_project_name = config.hub.k8s_secrets_project_name
         self._catalogs = {}
 
-    @staticmethod
-    def _in_k8s():
-        k8s_helper = mlrun.api.utils.singletons.k8s.get_k8s_helper()
-        return (
-            k8s_helper is not None and k8s_helper.is_running_inside_kubernetes_cluster()
-        )
-
-    @staticmethod
-    def _generate_credentials_secret_key(source, key=""):
-        full_key = source + secret_name_separator + key
-        return Secrets().generate_client_project_secret_key(
-            SecretsClientType.hub, full_key
-        )
-
     def add_source(self, source: mlrun.common.schemas.hub.HubSource):
         source_name = source.metadata.name
         credentials = source.spec.credentials
@@ -74,118 +63,6 @@ class Hub(metaclass=mlrun.utils.singleton.Singleton):
             allow_internal_secrets=True,
         )
 
-    def _store_source_credentials(self, source_name, credentials: dict):
-        if not self._in_k8s():
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "MLRun is not configured with k8s, hub source credentials cannot be stored securely"
-            )
-
-        adjusted_credentials = {
-            self._generate_credentials_secret_key(source_name, key): value
-            for key, value in credentials.items()
-        }
-        Secrets().store_project_secrets(
-            self._internal_project_name,
-            mlrun.common.schemas.SecretsData(
-                provider=mlrun.common.schemas.SecretProviderName.kubernetes,
-                secrets=adjusted_credentials,
-            ),
-            allow_internal_secrets=True,
-        )
-
-    def _get_source_credentials(self, source_name):
-        if not self._in_k8s():
-            return {}
-
-        secret_prefix = self._generate_credentials_secret_key(source_name)
-        secrets = (
-            Secrets()
-            .list_project_secrets(
-                self._internal_project_name,
-                mlrun.common.schemas.SecretProviderName.kubernetes,
-                allow_secrets_from_k8s=True,
-                allow_internal_secrets=True,
-            )
-            .secrets
-        )
-
-        source_secrets = {}
-        for key, value in secrets.items():
-            if key.startswith(secret_prefix):
-                source_secrets[key[len(secret_prefix) :]] = value
-
-        return source_secrets
-
-    @staticmethod
-    def _get_asset_full_path(
-        source: mlrun.common.schemas.hub.HubSource,
-        item: mlrun.common.schemas.hub.HubItem,
-        asset: str,
-    ):
-        """
-        Combining the item path with the asset path.
-
-        :param source:  Hub source object.
-        :param item:    The relevant item to get the asset from.
-        :param asset:   The asset name
-        :return:    Full path to the asset, relative to the item directory.
-        """
-        asset_path = item.spec.assets.get(asset, None)
-        if not asset_path:
-            raise mlrun.errors.MLRunNotFoundError(
-                f"Asset={asset} not found. "
-                f"item={item.metadata.name}, version={item.metadata.version}, tag={item.metadata.tag}"
-            )
-        item_path = item.metadata.get_relative_path()
-        return source.get_full_uri(item_path + asset_path)
-
-    @staticmethod
-    def _transform_catalog_dict_to_schema(
-        source: mlrun.common.schemas.hub.HubSource, catalog_dict: Dict[str, Any]
-    ) -> mlrun.common.schemas.hub.HubCatalog:
-        """
-        Transforms catalog dictionary to HubCatalog schema
-        :param source:          Hub source object.
-        :param catalog_dict:    raw catalog dict, top level keys are item names,
-                                second level keys are version tags ("latest, "1.1.0", ...) and
-                                bottom level keys include spec as a dict and all the rest is considered as metadata.
-        :return: catalog object
-        """
-        catalog = mlrun.common.schemas.hub.HubCatalog(
-            catalog=[], channel=source.spec.channel
-        )
-        # Loop over objects, then over object versions.
-        for object_name, object_dict in catalog_dict.items():
-            for version_tag, version_dict in object_dict.items():
-                object_details_dict = version_dict.copy()
-                spec_dict = object_details_dict.pop("spec", {})
-                assets = object_details_dict.pop("assets", {})
-                # We want to align all item names to be normalized.
-                # This is necessary since the item names are originally collected from the yaml files
-                # which may can contain underscores.
-                object_details_dict.update(
-                    {
-                        "name": mlrun.utils.helpers.normalize_name(
-                            object_name, verbose=False
-                        )
-                    }
-                )
-                metadata = mlrun.common.schemas.hub.HubItemMetadata(
-                    tag=version_tag, **object_details_dict
-                )
-                item_uri = source.get_full_uri(metadata.get_relative_path())
-                spec = mlrun.common.schemas.hub.HubItemSpec(
-                    item_uri=item_uri, assets=assets, **spec_dict
-                )
-                item = mlrun.common.schemas.hub.HubItem(
-                    metadata=metadata,
-                    spec=spec,
-                    status=mlrun.common.schemas.ObjectStatus(),
-                )
-                catalog.catalog.append(item)
-
-        return catalog
-
     def get_source_catalog(
         self,
         source: mlrun.common.schemas.hub.HubSource,
@@ -265,22 +142,6 @@ class Hub(metaclass=mlrun.utils.singleton.Singleton):
             )
         return items[0]
 
-    @staticmethod
-    def _get_catalog_items_filtered_by_name(
-        catalog: List[mlrun.common.schemas.hub.HubItem],
-        item_name: str,
-    ) -> List[mlrun.common.schemas.hub.HubItem]:
-        """
-        Retrieve items from catalog filtered by name
-
-        :param catalog:     list of items
-        :param item_name:   item name to filter by
-
-        :return:   list of item objects from catalog
-        """
-        normalized_name = mlrun.utils.helpers.normalize_name(item_name, verbose=False)
-        return [item for item in catalog if item.metadata.name == normalized_name]
-
     def get_item_object_using_source_credentials(
         self, source: mlrun.common.schemas.hub.HubSource, url
     ):
@@ -323,6 +184,19 @@ class Hub(metaclass=mlrun.utils.singleton.Singleton):
             asset_path,
         )
 
+    def list_hub_sources(
+        self,
+        db_session: sqlalchemy.orm.Session,
+        item_name: Optional[str] = None,
+        tag: Optional[str] = None,
+        version: Optional[str] = None,
+    ) -> List[mlrun.common.schemas.IndexedHubSource]:
+
+        hub_sources = mlrun.api.utils.singletons.db.get_db().list_hub_sources(
+            db_session
+        )
+        return self.filter_hub_sources(hub_sources, item_name, tag, version)
+
     def filter_hub_sources(
         self,
         sources: List[mlrun.common.schemas.IndexedHubSource],
@@ -360,3 +234,145 @@ class Hub(metaclass=mlrun.utils.singleton.Singleton):
                     filtered_sources.append(source)
                     break
         return filtered_sources
+
+    @staticmethod
+    def _in_k8s():
+        k8s_helper = mlrun.api.utils.singletons.k8s.get_k8s_helper()
+        return (
+            k8s_helper is not None and k8s_helper.is_running_inside_kubernetes_cluster()
+        )
+
+    @staticmethod
+    def _generate_credentials_secret_key(source, key=""):
+        full_key = source + secret_name_separator + key
+        return Secrets().generate_client_project_secret_key(
+            SecretsClientType.hub, full_key
+        )
+
+    def _store_source_credentials(self, source_name, credentials: dict):
+        if not self._in_k8s():
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "MLRun is not configured with k8s, hub source credentials cannot be stored securely"
+            )
+
+        adjusted_credentials = {
+            self._generate_credentials_secret_key(source_name, key): value
+            for key, value in credentials.items()
+        }
+        Secrets().store_project_secrets(
+            self._internal_project_name,
+            mlrun.common.schemas.SecretsData(
+                provider=mlrun.common.schemas.SecretProviderName.kubernetes,
+                secrets=adjusted_credentials,
+            ),
+            allow_internal_secrets=True,
+        )
+
+    def _get_source_credentials(self, source_name):
+        if not self._in_k8s():
+            return {}
+
+        secret_prefix = self._generate_credentials_secret_key(source_name)
+        secrets = (
+            Secrets()
+            .list_project_secrets(
+                self._internal_project_name,
+                mlrun.common.schemas.SecretProviderName.kubernetes,
+                allow_secrets_from_k8s=True,
+                allow_internal_secrets=True,
+            )
+            .secrets
+        )
+
+        source_secrets = {}
+        for key, value in secrets.items():
+            if key.startswith(secret_prefix):
+                source_secrets[key[len(secret_prefix) :]] = value
+
+        return source_secrets
+
+    @staticmethod
+    def _get_asset_full_path(
+        source: mlrun.common.schemas.hub.HubSource,
+        item: mlrun.common.schemas.hub.HubItem,
+        asset: str,
+    ):
+        """
+        Combining the item path with the asset path.
+
+        :param source:  Hub source object.
+        :param item:    The relevant item to get the asset from.
+        :param asset:   The asset name
+        :return:    Full path to the asset, relative to the item directory.
+        """
+        asset_path = item.spec.assets.get(asset, None)
+        if not asset_path:
+            raise mlrun.errors.MLRunNotFoundError(
+                f"Asset={asset} not found. "
+                f"item={item.metadata.name}, version={item.metadata.version}, tag={item.metadata.tag}"
+            )
+        item_path = item.metadata.get_relative_path()
+        return source.get_full_uri(item_path + asset_path)
+
+    @staticmethod
+    def _transform_catalog_dict_to_schema(
+        source: mlrun.common.schemas.hub.HubSource, catalog_dict: Dict[str, Any]
+    ) -> mlrun.common.schemas.hub.HubCatalog:
+        """
+        Transforms catalog dictionary to HubCatalog schema
+        :param source:          Hub source object.
+        :param catalog_dict:    raw catalog dict, top level keys are item names,
+                                second level keys are version tags ("latest, "1.1.0", ...) and
+                                bottom level keys include spec as a dict and all the rest is considered as metadata.
+        :return: catalog object
+        """
+        catalog = mlrun.common.schemas.hub.HubCatalog(
+            catalog=[], channel=source.spec.channel
+        )
+        # Loop over objects, then over object versions.
+        for object_name, object_dict in catalog_dict.items():
+            for version_tag, version_dict in object_dict.items():
+                object_details_dict = version_dict.copy()
+                spec_dict = object_details_dict.pop("spec", {})
+                assets = object_details_dict.pop("assets", {})
+                # We want to align all item names to be normalized.
+                # This is necessary since the item names are originally collected from the yaml files
+                # which may can contain underscores.
+                object_details_dict.update(
+                    {
+                        "name": mlrun.utils.helpers.normalize_name(
+                            object_name, verbose=False
+                        )
+                    }
+                )
+                metadata = mlrun.common.schemas.hub.HubItemMetadata(
+                    tag=version_tag, **object_details_dict
+                )
+                item_uri = source.get_full_uri(metadata.get_relative_path())
+                spec = mlrun.common.schemas.hub.HubItemSpec(
+                    item_uri=item_uri, assets=assets, **spec_dict
+                )
+                item = mlrun.common.schemas.hub.HubItem(
+                    metadata=metadata,
+                    spec=spec,
+                    status=mlrun.common.schemas.ObjectStatus(),
+                )
+                catalog.catalog.append(item)
+
+        return catalog
+
+    @staticmethod
+    def _get_catalog_items_filtered_by_name(
+        catalog: List[mlrun.common.schemas.hub.HubItem],
+        item_name: str,
+    ) -> List[mlrun.common.schemas.hub.HubItem]:
+        """
+        Retrieve items from catalog filtered by name
+
+        :param catalog:     list of items
+        :param item_name:   item name to filter by
+
+        :return:   list of item objects from catalog
+        """
+        normalized_name = mlrun.utils.helpers.normalize_name(item_name, verbose=False)
+        return [item for item in catalog if item.metadata.name == normalized_name]

mlrun/api/crud/workflows.py

@@ -26,6 +26,7 @@ from mlrun.api.api.utils import (
 )
 from mlrun.config import config
 from mlrun.model import Credentials, RunMetadata, RunObject, RunSpec
+from mlrun.utils import fill_project_path_template
 
 
 class WorkflowRunners(
@@ -161,9 +162,12 @@ class WorkflowRunners(
                 ),
                 handler="mlrun.projects.load_and_run",
                 scrape_metrics=config.scrape_metrics,
-                output_path=(
-                    workflow_request.artifact_path or config.artifact_path
-                ).replace("{{run.uid}}", meta_uid),
+                output_path=fill_project_path_template(
+                    (workflow_request.artifact_path or config.artifact_path).replace(
+                        "{{run.uid}}", meta_uid
+                    ),
+                    project.metadata.name,
+                ),
             ),
             metadata=RunMetadata(
                 uid=meta_uid, name=workflow_spec.name, project=project.metadata.name