mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/api/crud/feature_store.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,10 +16,10 @@ import typing
 
 import sqlalchemy.orm
 
-import mlrun.api.schemas
 import mlrun.api.utils.projects.remotes.follower
 import mlrun.api.utils.singletons.db
 import mlrun.api.utils.singletons.project_member
+import mlrun.common.schemas
 import mlrun.config
 import mlrun.errors
 import mlrun.utils.singleton
@@ -32,9 +32,12 @@ class FeatureStore(
         self,
         db_session: sqlalchemy.orm.Session,
         project: str,
-        feature_set: mlrun.api.schemas.FeatureSet,
+        feature_set: mlrun.common.schemas.FeatureSet,
         versioned: bool = True,
     ) -> str:
+        if not feature_set.spec.engine:
+            feature_set.spec.engine = "storey"
+
         return self._create_object(
             db_session,
             project,
@@ -47,11 +50,14 @@ class FeatureStore(
         db_session: sqlalchemy.orm.Session,
         project: str,
         name: str,
-        feature_set: mlrun.api.schemas.FeatureSet,
+        feature_set: mlrun.common.schemas.FeatureSet,
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
         versioned: bool = True,
     ) -> str:
+        if not feature_set.spec.engine:
+            feature_set.spec.engine = "storey"
+
         return self._store_object(
             db_session,
             project,
@@ -70,11 +76,11 @@ class FeatureStore(
         feature_set_patch: dict,
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
-        patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
+        patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
     ) -> str:
         return self._patch_object(
             db_session,
-            mlrun.api.schemas.FeatureSet,
+            mlrun.common.schemas.FeatureSet,
             project,
             name,
             feature_set_patch,
@@ -90,9 +96,9 @@ class FeatureStore(
         name: str,
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
-    ) -> mlrun.api.schemas.FeatureSet:
+    ) -> mlrun.common.schemas.FeatureSet:
         return self._get_object(
-            db_session, mlrun.api.schemas.FeatureSet, project, name, tag, uid
+            db_session, mlrun.common.schemas.FeatureSet, project, name, tag, uid
         )
 
     def list_feature_sets_tags(
@@ -104,7 +110,7 @@ class FeatureStore(
         :return: a list of Tuple of (project, feature_set.name, tag)
         """
         return self._list_object_type_tags(
-            db_session, mlrun.api.schemas.FeatureSet, project
+            db_session, mlrun.common.schemas.FeatureSet, project
         )
 
     def list_feature_sets(
@@ -117,11 +123,11 @@ class FeatureStore(
         entities: typing.List[str] = None,
         features: typing.List[str] = None,
         labels: typing.List[str] = None,
-        partition_by: mlrun.api.schemas.FeatureStorePartitionByField = None,
+        partition_by: mlrun.common.schemas.FeatureStorePartitionByField = None,
         rows_per_partition: int = 1,
-        partition_sort_by: mlrun.api.schemas.SortField = None,
-        partition_order: mlrun.api.schemas.OrderType = mlrun.api.schemas.OrderType.desc,
-    ) -> mlrun.api.schemas.FeatureSetsOutput:
+        partition_sort_by: mlrun.common.schemas.SortField = None,
+        partition_order: mlrun.common.schemas.OrderType = mlrun.common.schemas.OrderType.desc,
+    ) -> mlrun.common.schemas.FeatureSetsOutput:
         project = project or mlrun.mlconf.default_project
         return mlrun.api.utils.singletons.db.get_db().list_feature_sets(
             db_session,
@@ -148,7 +154,7 @@ class FeatureStore(
     ):
         self._delete_object(
             db_session,
-            mlrun.api.schemas.FeatureSet,
+            mlrun.common.schemas.FeatureSet,
             project,
             name,
             tag,
@@ -163,7 +169,7 @@ class FeatureStore(
         tag: typing.Optional[str] = None,
         entities: typing.List[str] = None,
         labels: typing.List[str] = None,
-    ) -> mlrun.api.schemas.FeaturesOutput:
+    ) -> mlrun.common.schemas.FeaturesOutput:
         project = project or mlrun.mlconf.default_project
         return mlrun.api.utils.singletons.db.get_db().list_features(
             db_session,
@@ -181,7 +187,7 @@ class FeatureStore(
         name: str,
         tag: typing.Optional[str] = None,
         labels: typing.List[str] = None,
-    ) -> mlrun.api.schemas.EntitiesOutput:
+    ) -> mlrun.common.schemas.EntitiesOutput:
         project = project or mlrun.mlconf.default_project
         return mlrun.api.utils.singletons.db.get_db().list_entities(
             db_session,
@@ -195,7 +201,7 @@ class FeatureStore(
         self,
         db_session: sqlalchemy.orm.Session,
         project: str,
-        feature_vector: mlrun.api.schemas.FeatureVector,
+        feature_vector: mlrun.common.schemas.FeatureVector,
         versioned: bool = True,
     ) -> str:
         return self._create_object(db_session, project, feature_vector, versioned)
@@ -205,7 +211,7 @@ class FeatureStore(
         db_session: sqlalchemy.orm.Session,
         project: str,
         name: str,
-        feature_vector: mlrun.api.schemas.FeatureVector,
+        feature_vector: mlrun.common.schemas.FeatureVector,
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
         versioned: bool = True,
@@ -228,11 +234,11 @@ class FeatureStore(
         feature_vector_patch: dict,
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
-        patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
+        patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
     ) -> str:
         return self._patch_object(
             db_session,
-            mlrun.api.schemas.FeatureVector,
+            mlrun.common.schemas.FeatureVector,
             project,
             name,
             feature_vector_patch,
@@ -248,10 +254,10 @@ class FeatureStore(
         name: str,
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
-    ) -> mlrun.api.schemas.FeatureVector:
+    ) -> mlrun.common.schemas.FeatureVector:
         return self._get_object(
             db_session,
-            mlrun.api.schemas.FeatureVector,
+            mlrun.common.schemas.FeatureVector,
             project,
             name,
             tag,
@@ -267,7 +273,7 @@ class FeatureStore(
         :return: a list of Tuple of (project, feature_vector.name, tag)
         """
         return self._list_object_type_tags(
-            db_session, mlrun.api.schemas.FeatureVector, project
+            db_session, mlrun.common.schemas.FeatureVector, project
         )
 
     def list_feature_vectors(
@@ -278,11 +284,11 @@ class FeatureStore(
         tag: typing.Optional[str] = None,
         state: str = None,
         labels: typing.List[str] = None,
-        partition_by: mlrun.api.schemas.FeatureStorePartitionByField = None,
+        partition_by: mlrun.common.schemas.FeatureStorePartitionByField = None,
         rows_per_partition: int = 1,
-        partition_sort_by: mlrun.api.schemas.SortField = None,
-        partition_order: mlrun.api.schemas.OrderType = mlrun.api.schemas.OrderType.desc,
-    ) -> mlrun.api.schemas.FeatureVectorsOutput:
+        partition_sort_by: mlrun.common.schemas.SortField = None,
+        partition_order: mlrun.common.schemas.OrderType = mlrun.common.schemas.OrderType.desc,
+    ) -> mlrun.common.schemas.FeatureVectorsOutput:
         project = project or mlrun.mlconf.default_project
         return mlrun.api.utils.singletons.db.get_db().list_feature_vectors(
             db_session,
@@ -307,7 +313,7 @@ class FeatureStore(
     ):
         self._delete_object(
             db_session,
-            mlrun.api.schemas.FeatureVector,
+            mlrun.common.schemas.FeatureVector,
             project,
             name,
             tag,
@@ -319,17 +325,17 @@ class FeatureStore(
         db_session: sqlalchemy.orm.Session,
         project: str,
         object_: typing.Union[
-            mlrun.api.schemas.FeatureSet, mlrun.api.schemas.FeatureVector
+            mlrun.common.schemas.FeatureSet, mlrun.common.schemas.FeatureVector
         ],
         versioned: bool = True,
     ) -> str:
         project = project or mlrun.mlconf.default_project
         self._validate_and_enrich_identity_for_object_creation(project, object_)
-        if isinstance(object_, mlrun.api.schemas.FeatureSet):
+        if isinstance(object_, mlrun.common.schemas.FeatureSet):
             return mlrun.api.utils.singletons.db.get_db().create_feature_set(
                 db_session, project, object_, versioned
             )
-        elif isinstance(object_, mlrun.api.schemas.FeatureVector):
+        elif isinstance(object_, mlrun.common.schemas.FeatureVector):
             return mlrun.api.utils.singletons.db.get_db().create_feature_vector(
                 db_session, project, object_, versioned
             )
@@ -344,7 +350,7 @@ class FeatureStore(
         project: str,
         name: str,
         object_: typing.Union[
-            mlrun.api.schemas.FeatureSet, mlrun.api.schemas.FeatureVector
+            mlrun.common.schemas.FeatureSet, mlrun.common.schemas.FeatureVector
         ],
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
@@ -354,7 +360,7 @@ class FeatureStore(
         self._validate_and_enrich_identity_for_object_store(
             object_, project, name, tag, uid
         )
-        if isinstance(object_, mlrun.api.schemas.FeatureSet):
+        if isinstance(object_, mlrun.common.schemas.FeatureSet):
             return mlrun.api.utils.singletons.db.get_db().store_feature_set(
                 db_session,
                 project,
@@ -364,7 +370,7 @@ class FeatureStore(
                 uid,
                 versioned,
             )
-        elif isinstance(object_, mlrun.api.schemas.FeatureVector):
+        elif isinstance(object_, mlrun.common.schemas.FeatureVector):
             return mlrun.api.utils.singletons.db.get_db().store_feature_vector(
                 db_session,
                 project,
@@ -388,7 +394,7 @@ class FeatureStore(
         object_patch: dict,
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
-        patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
+        patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
     ) -> str:
         project = project or mlrun.mlconf.default_project
         self._validate_identity_for_object_patch(
@@ -399,7 +405,7 @@ class FeatureStore(
             tag,
             uid,
         )
-        if object_schema.__name__ == mlrun.api.schemas.FeatureSet.__name__:
+        if object_schema.__name__ == mlrun.common.schemas.FeatureSet.__name__:
             return mlrun.api.utils.singletons.db.get_db().patch_feature_set(
                 db_session,
                 project,
@@ -409,7 +415,7 @@ class FeatureStore(
                 uid,
                 patch_mode,
             )
-        elif object_schema.__name__ == mlrun.api.schemas.FeatureVector.__name__:
+        elif object_schema.__name__ == mlrun.common.schemas.FeatureVector.__name__:
             return mlrun.api.utils.singletons.db.get_db().patch_feature_vector(
                 db_session,
                 project,
@@ -432,13 +438,15 @@ class FeatureStore(
         name: str,
         tag: typing.Optional[str] = None,
         uid: typing.Optional[str] = None,
-    ) -> typing.Union[mlrun.api.schemas.FeatureSet, mlrun.api.schemas.FeatureVector]:
+    ) -> typing.Union[
+        mlrun.common.schemas.FeatureSet, mlrun.common.schemas.FeatureVector
+    ]:
         project = project or mlrun.mlconf.default_project
-        if object_schema.__name__ == mlrun.api.schemas.FeatureSet.__name__:
+        if object_schema.__name__ == mlrun.common.schemas.FeatureSet.__name__:
             return mlrun.api.utils.singletons.db.get_db().get_feature_set(
                 db_session, project, name, tag, uid
             )
-        elif object_schema.__name__ == mlrun.api.schemas.FeatureVector.__name__:
+        elif object_schema.__name__ == mlrun.common.schemas.FeatureVector.__name__:
             return mlrun.api.utils.singletons.db.get_db().get_feature_vector(
                 db_session, project, name, tag, uid
             )
@@ -454,11 +462,11 @@ class FeatureStore(
         project: str,
     ) -> typing.List[typing.Tuple[str, str, str]]:
         project = project or mlrun.mlconf.default_project
-        if object_schema.__name__ == mlrun.api.schemas.FeatureSet.__name__:
+        if object_schema.__name__ == mlrun.common.schemas.FeatureSet.__name__:
             return mlrun.api.utils.singletons.db.get_db().list_feature_sets_tags(
                 db_session, project
             )
-        elif object_schema.__name__ == mlrun.api.schemas.FeatureVector.__name__:
+        elif object_schema.__name__ == mlrun.common.schemas.FeatureVector.__name__:
             return mlrun.api.utils.singletons.db.get_db().list_feature_vectors_tags(
                 db_session, project
             )
@@ -477,11 +485,11 @@ class FeatureStore(
         uid: typing.Optional[str] = None,
     ):
         project = project or mlrun.mlconf.default_project
-        if object_schema.__name__ == mlrun.api.schemas.FeatureSet.__name__:
+        if object_schema.__name__ == mlrun.common.schemas.FeatureSet.__name__:
             mlrun.api.utils.singletons.db.get_db().delete_feature_set(
                 db_session, project, name, tag, uid
             )
-        elif object_schema.__name__ == mlrun.api.schemas.FeatureVector.__name__:
+        elif object_schema.__name__ == mlrun.common.schemas.FeatureVector.__name__:
             mlrun.api.utils.singletons.db.get_db().delete_feature_vector(
                 db_session, project, name, tag, uid
             )
@@ -519,7 +527,7 @@ class FeatureStore(
     @staticmethod
     def _validate_and_enrich_identity_for_object_store(
         object_: typing.Union[
-            mlrun.api.schemas.FeatureSet, mlrun.api.schemas.FeatureVector
+            mlrun.common.schemas.FeatureSet, mlrun.common.schemas.FeatureVector
         ],
         project: str,
         name: str,
@@ -550,7 +558,7 @@ class FeatureStore(
     def _validate_and_enrich_identity_for_object_creation(
         project: str,
         object_: typing.Union[
-            mlrun.api.schemas.FeatureSet, mlrun.api.schemas.FeatureVector
+            mlrun.common.schemas.FeatureSet, mlrun.common.schemas.FeatureVector
         ],
     ):
         object_type = object_.__class__.__name__

mlrun/api/crud/functions.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,10 +17,10 @@ import typing
 import sqlalchemy.orm
 
 import mlrun.api.api.utils
-import mlrun.api.schemas
 import mlrun.api.utils.projects.remotes.follower
 import mlrun.api.utils.singletons.db
 import mlrun.api.utils.singletons.project_member
+import mlrun.common.schemas
 import mlrun.config
 import mlrun.errors
 import mlrun.utils.singleton
@@ -37,7 +37,7 @@ class Functions(
         project: str = mlrun.mlconf.default_project,
         tag: str = "",
         versioned: bool = False,
-        auth_info: mlrun.api.schemas.AuthInfo = None,
+        auth_info: mlrun.common.schemas.AuthInfo = None,
     ) -> str:
         project = project or mlrun.mlconf.default_project
         if auth_info:

mlrun/api/crud/hub.py

@@ -0,0 +1,312 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+from typing import Any, Dict, List, Optional, Tuple
+
+import mlrun.api.utils.singletons.k8s
+import mlrun.common.schemas
+import mlrun.common.schemas.hub
+import mlrun.errors
+import mlrun.utils.singleton
+from mlrun.config import config
+from mlrun.datastore import store_manager
+
+from .secrets import Secrets, SecretsClientType
+
+# Using a complex separator, as it's less likely someone will use it in a real secret name
+secret_name_separator = "-__-"
+
+
+class Hub(metaclass=mlrun.utils.singleton.Singleton):
+    def __init__(self):
+        self._internal_project_name = config.hub.k8s_secrets_project_name
+        self._catalogs = {}
+
+    @staticmethod
+    def _in_k8s():
+        k8s_helper = mlrun.api.utils.singletons.k8s.get_k8s_helper()
+        return (
+            k8s_helper is not None and k8s_helper.is_running_inside_kubernetes_cluster()
+        )
+
+    @staticmethod
+    def _generate_credentials_secret_key(source, key=""):
+        full_key = source + secret_name_separator + key
+        return Secrets().generate_client_project_secret_key(
+            SecretsClientType.hub, full_key
+        )
+
+    def add_source(self, source: mlrun.common.schemas.hub.HubSource):
+        source_name = source.metadata.name
+        credentials = source.spec.credentials
+        if credentials:
+            self._store_source_credentials(source_name, credentials)
+
+    def remove_source(self, source_name):
+        self._catalogs.pop(source_name, None)
+        if not self._in_k8s():
+            return
+
+        source_credentials = self._get_source_credentials(source_name)
+        if not source_credentials:
+            return
+        secrets_to_delete = [
+            self._generate_credentials_secret_key(source_name, key)
+            for key in source_credentials
+        ]
+        Secrets().delete_project_secrets(
+            self._internal_project_name,
+            mlrun.common.schemas.SecretProviderName.kubernetes,
+            secrets_to_delete,
+            allow_internal_secrets=True,
+        )
+
+    def _store_source_credentials(self, source_name, credentials: dict):
+        if not self._in_k8s():
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "MLRun is not configured with k8s, hub source credentials cannot be stored securely"
+            )
+
+        adjusted_credentials = {
+            self._generate_credentials_secret_key(source_name, key): value
+            for key, value in credentials.items()
+        }
+        Secrets().store_project_secrets(
+            self._internal_project_name,
+            mlrun.common.schemas.SecretsData(
+                provider=mlrun.common.schemas.SecretProviderName.kubernetes,
+                secrets=adjusted_credentials,
+            ),
+            allow_internal_secrets=True,
+        )
+
+    def _get_source_credentials(self, source_name):
+        if not self._in_k8s():
+            return {}
+
+        secret_prefix = self._generate_credentials_secret_key(source_name)
+        secrets = (
+            Secrets()
+            .list_project_secrets(
+                self._internal_project_name,
+                mlrun.common.schemas.SecretProviderName.kubernetes,
+                allow_secrets_from_k8s=True,
+                allow_internal_secrets=True,
+            )
+            .secrets
+        )
+
+        source_secrets = {}
+        for key, value in secrets.items():
+            if key.startswith(secret_prefix):
+                source_secrets[key[len(secret_prefix) :]] = value
+
+        return source_secrets
+
+    @staticmethod
+    def _get_asset_full_path(
+        source: mlrun.common.schemas.hub.HubSource,
+        item: mlrun.common.schemas.hub.HubItem,
+        asset: str,
+    ):
+        """
+        Combining the item path with the asset path.
+
+        :param source:  Hub source object.
+        :param item:    The relevant item to get the asset from.
+        :param asset:   The asset name
+        :return:    Full path to the asset, relative to the item directory.
+        """
+        asset_path = item.spec.assets.get(asset, None)
+        if not asset_path:
+            raise mlrun.errors.MLRunNotFoundError(
+                f"Asset={asset} not found. "
+                f"item={item.metadata.name}, version={item.metadata.version}, tag={item.metadata.tag}"
+            )
+        item_path = item.metadata.get_relative_path()
+        return source.get_full_uri(item_path + asset_path)
+
+    @staticmethod
+    def _transform_catalog_dict_to_schema(
+        source: mlrun.common.schemas.hub.HubSource, catalog_dict: Dict[str, Any]
+    ) -> mlrun.common.schemas.hub.HubCatalog:
+        """
+        Transforms catalog dictionary to HubCatalog schema
+        :param source:          Hub source object.
+        :param catalog_dict:    raw catalog dict, top level keys are item names,
+                                second level keys are version tags ("latest, "1.1.0", ...) and
+                                bottom level keys include spec as a dict and all the rest is considered as metadata.
+        :return: catalog object
+        """
+        catalog = mlrun.common.schemas.hub.HubCatalog(
+            catalog=[], channel=source.spec.channel
+        )
+        # Loop over objects, then over object versions.
+        for object_name, object_dict in catalog_dict.items():
+            for version_tag, version_dict in object_dict.items():
+                object_details_dict = version_dict.copy()
+                spec_dict = object_details_dict.pop("spec", {})
+                assets = object_details_dict.pop("assets", {})
+                metadata = mlrun.common.schemas.hub.HubItemMetadata(
+                    tag=version_tag, **object_details_dict
+                )
+                item_uri = source.get_full_uri(metadata.get_relative_path())
+                spec = mlrun.common.schemas.hub.HubItemSpec(
+                    item_uri=item_uri, assets=assets, **spec_dict
+                )
+                item = mlrun.common.schemas.hub.HubItem(
+                    metadata=metadata,
+                    spec=spec,
+                    status=mlrun.common.schemas.ObjectStatus(),
+                )
+                catalog.catalog.append(item)
+
+        return catalog
+
+    def get_source_catalog(
+        self,
+        source: mlrun.common.schemas.hub.HubSource,
+        version: Optional[str] = None,
+        tag: Optional[str] = None,
+        force_refresh: bool = False,
+    ) -> mlrun.common.schemas.hub.HubCatalog:
+        """
+        Getting the catalog object by source.
+        If version and/or tag are given, the catalog will be filtered accordingly.
+
+        :param source:          Hub source object.
+        :param version:         version of items to filter by
+        :param tag:             tag of items to filter by
+        :param force_refresh:   if True, the catalog will be loaded from source always,
+                                otherwise will be pulled from db (if loaded before)
+        :return: catalog object
+        """
+        source_name = source.metadata.name
+        if not self._catalogs.get(source_name) or force_refresh:
+            url = source.get_catalog_uri()
+            credentials = self._get_source_credentials(source_name)
+            catalog_data = mlrun.run.get_object(url=url, secrets=credentials)
+            catalog_dict = json.loads(catalog_data)
+            catalog = self._transform_catalog_dict_to_schema(source, catalog_dict)
+            self._catalogs[source_name] = catalog
+        else:
+            catalog = self._catalogs[source_name]
+
+        result_catalog = mlrun.common.schemas.hub.HubCatalog(
+            catalog=[], channel=source.spec.channel
+        )
+        for item in catalog.catalog:
+            # Because tag and version are optionals,
+            # we filter the catalog by one of them with priority to tag
+            if (tag is None or item.metadata.tag == tag) and (
+                version is None or item.metadata.version == version
+            ):
+                result_catalog.catalog.append(item)
+
+        return result_catalog
+
+    def get_item(
+        self,
+        source: mlrun.common.schemas.hub.HubSource,
+        item_name: str,
+        version: Optional[str] = None,
+        tag: Optional[str] = None,
+        force_refresh: bool = False,
+    ) -> mlrun.common.schemas.hub.HubItem:
+        """
+        Retrieve item from source. The item is filtered by tag and version.
+
+        :param source:          Hub source object
+        :param item_name:       name of the item to retrieve
+        :param version:         version of the item
+        :param tag:             tag of the item
+        :param force_refresh:   if True, the catalog will be loaded from source always,
+                                otherwise will be pulled from db (if loaded before)
+
+        :return: hub item object
+
+        :raise if the number of collected items from catalog is not exactly one.
+        """
+        catalog = self.get_source_catalog(source, version, tag, force_refresh)
+        items = self._get_catalog_items_filtered_by_name(catalog.catalog, item_name)
+        num_items = len(items)
+
+        if not num_items:
+            raise mlrun.errors.MLRunNotFoundError(
+                f"Item not found. source={item_name}, version={version}"
+            )
+        if num_items > 1:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Query resulted in more than 1 catalog items. "
+                + f"source={item_name}, version={version}, tag={tag}"
+            )
+        return items[0]
+
+    @staticmethod
+    def _get_catalog_items_filtered_by_name(
+        catalog: List[mlrun.common.schemas.hub.HubItem],
+        item_name: str,
+    ) -> List[mlrun.common.schemas.hub.HubItem]:
+        """
+        Retrieve items from catalog filtered by name
+
+        :param catalog:     list of items
+        :param item_name:   item name to filter by
+
+        :return:   list of item objects from catalog
+        """
+        return [item for item in catalog if item.metadata.name == item_name]
+
+    def get_item_object_using_source_credentials(
+        self, source: mlrun.common.schemas.hub.HubSource, url
+    ):
+        credentials = self._get_source_credentials(source.metadata.name)
+
+        if not url.startswith(source.spec.path):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "URL to retrieve must be located in the source filesystem tree"
+            )
+
+        if url.endswith("/"):
+            obj = store_manager.object(url=url, secrets=credentials)
+            listdir = obj.listdir()
+            return {
+                "listdir": listdir,
+            }
+        else:
+            catalog_data = mlrun.run.get_object(url=url, secrets=credentials)
+        return catalog_data
+
+    def get_asset(
+        self,
+        source: mlrun.common.schemas.hub.HubSource,
+        item: mlrun.common.schemas.hub.HubItem,
+        asset_name: str,
+    ) -> Tuple[bytes, str]:
+        """
+        Retrieve asset object from hub source.
+
+        :param source:      hub source
+        :param item:        hub item which contains the assets
+        :param asset_name:  asset name, like source, example, etc.
+
+        :return: tuple of asset as bytes and url of asset
+        """
+        credentials = self._get_source_credentials(source.metadata.name)
+        asset_path = self._get_asset_full_path(source, item, asset_name)
+        return (
+            mlrun.run.get_object(url=asset_path, secrets=credentials),
+            asset_path,
+        )