mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/api/api/endpoints/marketplace.py

@@ -1,257 +0,0 @@
-# Copyright 2018 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import mimetypes
-from http import HTTPStatus
-from typing import List, Optional
-
-from fastapi import APIRouter, Depends, Query, Response
-from fastapi.concurrency import run_in_threadpool
-from sqlalchemy.orm import Session
-
-import mlrun
-import mlrun.api.api.deps
-import mlrun.api.crud
-import mlrun.api.utils.auth.verifier
-from mlrun.api.schemas import AuthorizationAction
-from mlrun.api.schemas.marketplace import (
-    IndexedMarketplaceSource,
-    MarketplaceCatalog,
-    MarketplaceItem,
-)
-from mlrun.api.utils.singletons.db import get_db
-
-router = APIRouter()
-
-
-@router.post(
-    path="/marketplace/sources",
-    status_code=HTTPStatus.CREATED.value,
-    response_model=IndexedMarketplaceSource,
-)
-async def create_source(
-    source: IndexedMarketplaceSource,
-    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-):
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
-        AuthorizationAction.create,
-        auth_info,
-    )
-
-    await run_in_threadpool(get_db().create_marketplace_source, db_session, source)
-    # Handle credentials if they exist
-    await run_in_threadpool(mlrun.api.crud.Marketplace().add_source, source.source)
-    return await run_in_threadpool(
-        get_db().get_marketplace_source, db_session, source.source.metadata.name
-    )
-
-
-@router.get(
-    path="/marketplace/sources",
-    response_model=List[IndexedMarketplaceSource],
-)
-async def list_sources(
-    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-):
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
-        AuthorizationAction.read,
-        auth_info,
-    )
-
-    return await run_in_threadpool(get_db().list_marketplace_sources, db_session)
-
-
-@router.delete(
-    path="/marketplace/sources/{source_name}",
-    status_code=HTTPStatus.NO_CONTENT.value,
-)
-async def delete_source(
-    source_name: str,
-    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-):
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
-        AuthorizationAction.delete,
-        auth_info,
-    )
-
-    await run_in_threadpool(get_db().delete_marketplace_source, db_session, source_name)
-    await run_in_threadpool(mlrun.api.crud.Marketplace().remove_source, source_name)
-
-
-@router.get(
-    path="/marketplace/sources/{source_name}",
-    response_model=IndexedMarketplaceSource,
-)
-async def get_source(
-    source_name: str,
-    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-):
-    marketplace_source = await run_in_threadpool(
-        get_db().get_marketplace_source, db_session, source_name
-    )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
-        AuthorizationAction.read,
-        auth_info,
-    )
-
-    return marketplace_source
-
-
-@router.put(
-    path="/marketplace/sources/{source_name}", response_model=IndexedMarketplaceSource
-)
-async def store_source(
-    source_name: str,
-    source: IndexedMarketplaceSource,
-    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-):
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
-        AuthorizationAction.store,
-        auth_info,
-    )
-
-    await run_in_threadpool(
-        get_db().store_marketplace_source, db_session, source_name, source
-    )
-    # Handle credentials if they exist
-    await run_in_threadpool(mlrun.api.crud.Marketplace().add_source, source.source)
-
-    return await run_in_threadpool(
-        get_db().get_marketplace_source, db_session, source_name
-    )
-
-
-@router.get(
-    path="/marketplace/sources/{source_name}/items",
-    response_model=MarketplaceCatalog,
-)
-async def get_catalog(
-    source_name: str,
-    channel: Optional[str] = Query(None),
-    version: Optional[str] = Query(None),
-    tag: Optional[str] = Query(None),
-    force_refresh: Optional[bool] = Query(False, alias="force-refresh"),
-    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-):
-    ordered_source = await run_in_threadpool(
-        get_db().get_marketplace_source, db_session, source_name
-    )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
-        AuthorizationAction.read,
-        auth_info,
-    )
-
-    return await run_in_threadpool(
-        mlrun.api.crud.Marketplace().get_source_catalog,
-        ordered_source.source,
-        channel,
-        version,
-        tag,
-        force_refresh,
-    )
-
-
-@router.get(
-    "/marketplace/sources/{source_name}/items/{item_name}",
-    response_model=MarketplaceItem,
-)
-async def get_item(
-    source_name: str,
-    item_name: str,
-    channel: Optional[str] = Query("development"),
-    version: Optional[str] = Query(None),
-    tag: Optional[str] = Query("latest"),
-    force_refresh: Optional[bool] = Query(False, alias="force-refresh"),
-    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-):
-    ordered_source = await run_in_threadpool(
-        get_db().get_marketplace_source, db_session, source_name
-    )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
-        AuthorizationAction.read,
-        auth_info,
-    )
-
-    return await run_in_threadpool(
-        mlrun.api.crud.Marketplace().get_item,
-        ordered_source.source,
-        item_name,
-        channel,
-        version,
-        tag,
-        force_refresh,
-    )
-
-
-@router.get(
-    "/marketplace/sources/{source_name}/item-object",
-)
-async def get_object(
-    source_name: str,
-    url: str,
-    db_session: Session = Depends(mlrun.api.api.deps.get_db_session),
-    auth_info: mlrun.api.schemas.AuthInfo = Depends(
-        mlrun.api.api.deps.authenticate_request
-    ),
-):
-    ordered_source = await run_in_threadpool(
-        get_db().get_marketplace_source, db_session, source_name
-    )
-    object_data = await run_in_threadpool(
-        mlrun.api.crud.Marketplace().get_item_object_using_source_credentials,
-        ordered_source.source,
-        url,
-    )
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_global_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.marketplace_source,
-        AuthorizationAction.read,
-        auth_info,
-    )
-
-    if url.endswith("/"):
-        return object_data
-
-    ctype, _ = mimetypes.guess_type(url)
-    if not ctype:
-        ctype = "application/octet-stream"
-    return Response(content=object_data, media_type=ctype)

mlrun/api/crud/marketplace.py

@@ -1,221 +0,0 @@
-# Copyright 2018 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import json
-
-import mlrun.errors
-import mlrun.utils.singleton
-from mlrun.api.schemas.marketplace import (
-    MarketplaceCatalog,
-    MarketplaceItem,
-    MarketplaceItemMetadata,
-    MarketplaceItemSpec,
-    MarketplaceSource,
-    ObjectStatus,
-)
-from mlrun.api.utils.singletons.k8s import get_k8s
-from mlrun.config import config
-from mlrun.datastore import store_manager
-
-from ..schemas import SecretProviderName
-from .secrets import Secrets, SecretsClientType
-
-# Using a complex separator, as it's less likely someone will use it in a real secret name
-secret_name_separator = "-__-"
-
-
-class Marketplace(metaclass=mlrun.utils.singleton.Singleton):
-    def __init__(self):
-        self._internal_project_name = config.marketplace.k8s_secrets_project_name
-        self._catalogs = {}
-
-    @staticmethod
-    def _in_k8s():
-        k8s_helper = get_k8s()
-        return (
-            k8s_helper is not None and k8s_helper.is_running_inside_kubernetes_cluster()
-        )
-
-    @staticmethod
-    def _generate_credentials_secret_key(source, key=""):
-        full_key = source + secret_name_separator + key
-        return Secrets().generate_client_project_secret_key(
-            SecretsClientType.marketplace, full_key
-        )
-
-    def add_source(self, source: MarketplaceSource):
-        source_name = source.metadata.name
-        credentials = source.spec.credentials
-        if credentials:
-            self._store_source_credentials(source_name, credentials)
-
-    def remove_source(self, source_name):
-        self._catalogs.pop(source_name, None)
-        if not self._in_k8s():
-            return
-
-        source_credentials = self._get_source_credentials(source_name)
-        if not source_credentials:
-            return
-        secrets_to_delete = [
-            self._generate_credentials_secret_key(source_name, key)
-            for key in source_credentials
-        ]
-        Secrets().delete_project_secrets(
-            self._internal_project_name,
-            SecretProviderName.kubernetes,
-            secrets_to_delete,
-            allow_internal_secrets=True,
-        )
-
-    def _store_source_credentials(self, source_name, credentials: dict):
-        if not self._in_k8s():
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "MLRun is not configured with k8s, marketplace source credentials cannot be stored securely"
-            )
-
-        adjusted_credentials = {
-            self._generate_credentials_secret_key(source_name, key): value
-            for key, value in credentials.items()
-        }
-        Secrets().store_project_secrets(
-            self._internal_project_name,
-            mlrun.api.schemas.SecretsData(
-                provider=SecretProviderName.kubernetes, secrets=adjusted_credentials
-            ),
-            allow_internal_secrets=True,
-        )
-
-    def _get_source_credentials(self, source_name):
-        if not self._in_k8s():
-            return {}
-
-        secret_prefix = self._generate_credentials_secret_key(source_name)
-        secrets = (
-            Secrets()
-            .list_project_secrets(
-                self._internal_project_name,
-                SecretProviderName.kubernetes,
-                allow_secrets_from_k8s=True,
-                allow_internal_secrets=True,
-            )
-            .secrets
-        )
-
-        source_secrets = {}
-        for key, value in secrets.items():
-            if key.startswith(secret_prefix):
-                source_secrets[key[len(secret_prefix) :]] = value
-
-        return source_secrets
-
-    @staticmethod
-    def _transform_catalog_dict_to_schema(source, catalog_dict):
-        catalog_dict = catalog_dict.get("functions")
-        if not catalog_dict:
-            raise mlrun.errors.MLRunInternalServerError(
-                "Invalid catalog file - no 'functions' section found."
-            )
-
-        catalog = MarketplaceCatalog(catalog=[])
-        # Loop over channels, then per function extract versions.
-        for channel_name in catalog_dict:
-            channel_dict = catalog_dict[channel_name]
-            for function_name in channel_dict:
-                function_dict = channel_dict[function_name]
-                for version_tag in function_dict:
-                    version_dict = function_dict[version_tag]
-                    function_details_dict = version_dict.copy()
-                    spec_dict = function_details_dict.pop("spec", None)
-                    metadata = MarketplaceItemMetadata(
-                        channel=channel_name, tag=version_tag, **function_details_dict
-                    )
-                    item_uri = source.get_full_uri(metadata.get_relative_path())
-                    spec = MarketplaceItemSpec(item_uri=item_uri, **spec_dict)
-                    item = MarketplaceItem(
-                        metadata=metadata, spec=spec, status=ObjectStatus()
-                    )
-                    catalog.catalog.append(item)
-
-        return catalog
-
-    def get_source_catalog(
-        self,
-        source: MarketplaceSource,
-        channel=None,
-        version=None,
-        tag=None,
-        force_refresh=False,
-    ) -> MarketplaceCatalog:
-        source_name = source.metadata.name
-        if not self._catalogs.get(source_name) or force_refresh:
-            url = source.get_catalog_uri()
-            credentials = self._get_source_credentials(source_name)
-            catalog_data = mlrun.run.get_object(url=url, secrets=credentials)
-            catalog_dict = json.loads(catalog_data)
-            catalog = self._transform_catalog_dict_to_schema(source, catalog_dict)
-            self._catalogs[source_name] = catalog
-        else:
-            catalog = self._catalogs[source_name]
-
-        result_catalog = MarketplaceCatalog(catalog=[])
-        for item in catalog.catalog:
-            if (
-                (channel is None or item.metadata.channel == channel)
-                and (tag is None or item.metadata.tag == tag)
-                and (version is None or item.metadata.version == version)
-            ):
-                result_catalog.catalog.append(item)
-
-        return result_catalog
-
-    def get_item(
-        self,
-        source: MarketplaceSource,
-        item_name,
-        channel,
-        version=None,
-        tag=None,
-        force_refresh=False,
-    ) -> MarketplaceItem:
-        catalog = self.get_source_catalog(source, channel, version, tag, force_refresh)
-        items = [item for item in catalog.catalog if item.metadata.name == item_name]
-        if not items:
-            raise mlrun.errors.MLRunNotFoundError(
-                f"Item not found. source={item_name}, channel={channel}, version={version}"
-            )
-        if len(items) > 1:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Query resulted in more than 1 catalog items. "
-                + f"source={item_name}, channel={channel}, version={version}, tag={tag}"
-            )
-        return items[0]
-
-    def get_item_object_using_source_credentials(self, source: MarketplaceSource, url):
-        credentials = self._get_source_credentials(source.metadata.name)
-
-        if not url.startswith(source.spec.path):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "URL to retrieve must be located in the source filesystem tree"
-            )
-
-        if url.endswith("/"):
-            obj = store_manager.object(url=url, secrets=credentials)
-            listdir = obj.listdir()
-            return {
-                "listdir": listdir,
-            }
-        else:
-            catalog_data = mlrun.run.get_object(url=url, secrets=credentials)
-        return catalog_data