digitalhub 0.7.0b2__py3-none-any.whl → 0.8.0__py3-none-any.whl

digitalhub/stores/builder.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import os
+import typing
+
+from pydantic import ValidationError
+
+from digitalhub.stores._base.store import StoreParameters
+from digitalhub.stores.local.store import LocalStore, LocalStoreConfig
+from digitalhub.stores.remote.store import RemoteStore, RemoteStoreConfig
+from digitalhub.stores.s3.store import S3Store, S3StoreConfig
+from digitalhub.stores.sql.store import SqlStore, SQLStoreConfig
+from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.uri_utils import map_uri_scheme
+
+if typing.TYPE_CHECKING:
+    from digitalhub.stores._base.store import Store
+
+
+REGISTRY_STORES = {
+    "local": LocalStore,
+    "s3": S3Store,
+    "remote": RemoteStore,
+    "sql": SqlStore,
+}
+
+
+class StoreBuilder:
+    """
+    Store builder class.
+    """
+
+    def __init__(self) -> None:
+        self._instances: dict[str, Store] = {}
+        self._default: Store | None = None
+        self._def_scheme = "s3"
+
+    def build(self, store_cfg: StoreParameters) -> None:
+        """
+        Build a store instance and register it.
+        It overrides any existing instance.
+
+        Parameters
+        ----------
+        store_cfg : StoreParameters
+            Store configuration.
+
+        Returns
+        -------
+        None
+        """
+        scheme = map_uri_scheme(store_cfg.type)
+        self._instances[scheme] = self.build_store(store_cfg)
+
+    def get(self, uri: str) -> Store:
+        """
+        Get a store instance by URI.
+
+        Parameters
+        ----------
+        uri : str
+            URI to parse.
+
+        Returns
+        -------
+        Store
+            The store instance.
+        """
+        scheme = map_uri_scheme(uri)
+        if scheme not in self._instances:
+            store_cfg = get_env_store_config(scheme)
+            self._instances[scheme] = self.build_store(store_cfg)
+        return self._instances[scheme]
+
+    def default(self) -> Store:
+        """
+        Get the default store instance.
+
+        Returns
+        -------
+        Store
+            The default store instance.
+
+        Raises
+        ------
+        StoreError
+            If no default store is set.
+        """
+        if self._default is None:
+            store_cfg = get_env_store_config(self._def_scheme)
+            self._default = self.build_store(store_cfg)
+        return self._default
+
+    def build_store(self, cfg: StoreParameters) -> Store:
+        """
+        Build a store instance.
+
+        Parameters
+        ----------
+        cfg : StoreParameters
+            Store configuration.
+
+        Returns
+        -------
+        Store
+            The store instance.
+
+        Raises
+        ------
+        NotImplementedError
+            If the store type is not implemented.
+        """
+        try:
+            obj = REGISTRY_STORES[cfg.type](cfg.name, cfg.type, cfg.config)
+            if cfg.is_default and self._default is not None:
+                raise StoreError("Only one default store!")
+            return obj
+        except KeyError as e:
+            raise NotImplementedError from e
+
+    @staticmethod
+    def _check_config(config: StoreParameters | dict) -> StoreParameters:
+        """
+        Check the store configuration validity.
+
+        Parameters
+        ----------
+        config : StoreParameters | dict
+            The store configuration.
+
+        Returns
+        -------
+        StoreParameters
+            The store configuration.
+
+        Raises
+        ------
+        TypeError
+            If the config parameter is not a StoreParameters instance or a well-formed dictionary.
+        """
+        if not isinstance(config, StoreParameters):
+            try:
+                return StoreParameters(**config)
+            except TypeError as e:
+                raise StoreError("Invalid store configuration type.") from e
+            except ValidationError as e:
+                raise StoreError("Malformed store configuration parameters.") from e
+        return config
+
+
+def get_env_store_config(scheme: str) -> StoreParameters:
+    """
+    Get a store configuration from the environment.
+
+    Parameters
+    ----------
+    scheme : str
+        URI scheme.
+
+    Returns
+    -------
+    StoreParameters
+        The store configuration based on the scheme.
+
+    Raises
+    ------
+    ValueError
+        If the scheme is not supported.
+    """
+    if scheme == "s3":
+        return StoreParameters(
+            name="s3",
+            type="s3",
+            config=S3StoreConfig(
+                endpoint_url=os.getenv("S3_ENDPOINT_URL"),  # type: ignore
+                aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),  # type: ignore
+                aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),  # type: ignore
+                bucket_name=os.getenv("S3_BUCKET_NAME"),  # type: ignore
+            ),
+        )
+    if scheme == "sql":
+        return StoreParameters(
+            name="sql",
+            type="sql",
+            config=SQLStoreConfig(
+                host=os.getenv("POSTGRES_HOST"),  # type: ignore
+                port=os.getenv("POSTGRES_PORT"),  # type: ignore
+                user=os.getenv("POSTGRES_USER"),  # type: ignore
+                password=os.getenv("POSTGRES_PASSWORD"),  # type: ignore
+                database=os.getenv("POSTGRES_DATABASE"),  # type: ignore
+                pg_schema=os.getenv("POSTGRES_SCHEMA"),  # type: ignore
+            ),
+        )
+    if scheme == "remote":
+        return StoreParameters(
+            name="remote",
+            type="remote",
+            config=RemoteStoreConfig(),
+        )
+    if scheme == "local":
+        return StoreParameters(
+            name="local",
+            type="local",
+            config=LocalStoreConfig(
+                path="tempsdk",
+            ),
+        )
+    raise ValueError(f"Unsupported scheme {scheme}")
+
+
+store_builder = StoreBuilder()

digitalhub/stores/local/store.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+from digitalhub.stores._base.store import Store, StoreConfig
+from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.file_utils import get_file_info_from_local
+
+
+class LocalStoreConfig(StoreConfig):
+    """
+    Local store configuration class.
+    """
+
+    path: str
+    """Local path."""
+
+
+class LocalStore(Store):
+    """
+    Local store class. It implements the Store interface and provides methods to fetch and persist
+    artifacts on local filesystem based storage.
+    """
+
+    def __init__(self, name: str, store_type: str, config: LocalStoreConfig) -> None:
+        super().__init__(name, store_type)
+        self.config = config
+
+    ##############################
+    # IO methods
+    ##############################
+
+    def download(
+        self,
+        root: str,
+        dst: Path,
+        src: list[str],
+        overwrite: bool = False,
+    ) -> str:
+        """
+        Download artifacts from storage.
+
+        Parameters
+        ----------
+        root : str
+            The root path of the artifact.
+        dst : str
+            The destination of the artifact on local filesystem.
+        src : list[str]
+            List of sources.
+        overwrite : bool
+            Specify if overwrite existing file(s).
+
+        Returns
+        -------
+        str
+            Destination path of the downloaded artifact.
+        """
+        raise StoreError("Local store does not support download.")
+
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
+        """
+        Upload an artifact to storage.
+
+        Raises
+        ------
+        StoreError
+            This method is not implemented.
+        """
+        raise StoreError("Local store does not support upload.")
+
+    def get_file_info(self, paths: list[str]) -> list[dict]:
+        """
+        Method to get file metadata.
+
+        Parameters
+        ----------
+        paths : list
+            List of source paths.
+
+        Returns
+        -------
+        list[dict]
+            Returns files metadata.
+        """
+        return [get_file_info_from_local(p) for p in paths]
+
+    ##############################
+    # Private I/O methods
+    ##############################
+
+    def _get_src_dst_files(self, src: Path, dst: Path) -> list[str]:
+        """
+        Copy files from source to destination.
+
+        Parameters
+        ----------
+        src : Path
+            The source path.
+        dst : Path
+            The destination path.
+
+        Returns
+        -------
+        list[str]
+            Returns the list of destination and source paths of the
+            copied files.
+        """
+        return [self._get_src_dst_file(i, dst) for i in src.rglob("*") if i.is_file()]
+
+    def _get_src_dst_file(self, src: Path, dst: Path) -> str:
+        """
+        Copy file from source to destination.
+
+        Parameters
+        ----------
+        src : Path
+            The source path.
+        dst : Path
+            The destination path.
+
+        Returns
+        -------
+        str
+        """
+        dst_pth = self._copy_file(src, dst, True)
+        return str(dst_pth), str(src)
+
+    def _copy_dir(self, src: Path, dst: Path, overwrite: bool) -> list[str]:
+        """
+        Download file from source to destination.
+
+        Parameters
+        ----------
+        src : Path
+            The source path.
+        dst : Path
+            The destination path.
+
+        Returns
+        -------
+        list[str]
+        """
+        dst = self._rebuild_path(dst, src)
+        shutil.copytree(src, dst, dirs_exist_ok=overwrite)
+        return [str(i) for i in dst.rglob("*") if i.is_file()]
+
+    def _copy_file(self, src: Path, dst: Path, overwrite: bool) -> str:
+        """
+        Copy file from source to destination.
+
+        Parameters
+        ----------
+        src : Path
+            The source path.
+        dst : Path
+            The destination path.
+
+        Returns
+        -------
+        str
+        """
+        dst = self._rebuild_path(dst, src)
+        self._check_overwrite(dst, overwrite)
+        return str(shutil.copy2(src, dst))
+
+    def _rebuild_path(self, dst: Path, src: Path) -> Path:
+        """
+        Rebuild path.
+
+        Parameters
+        ----------
+        dst : Path
+            The destination path.
+        src : Path
+            The source path.
+
+        Returns
+        -------
+        Path
+            The rebuilt path.
+        """
+        if dst.is_dir():
+            if src.is_absolute():
+                raise StoreError("Source must be a relative path if the destination is a directory.")
+            dst = dst / src
+        self._build_path(dst)
+        return dst
+
+    ##############################
+    # Static methods
+    ##############################
+
+    @staticmethod
+    def is_partition_or_dir(path: str) -> bool:
+        """
+        Check if path is a directory or a partition.
+
+        Parameters
+        ----------
+        path : str
+            The path to check.
+
+        Returns
+        -------
+        bool
+        """
+        return Path(path).is_dir()
+
+    @staticmethod
+    def build_object_path(root: str, paths: str | list[str]) -> list[str]:
+        """
+        Method to build object path.
+
+        Parameters
+        ----------
+        root : str
+            The root of the object path.
+        paths : str | list[str]
+            The path to build.
+
+        Returns
+        -------
+        list[str]
+            Returns the path of the object.
+        """
+        if isinstance(paths, str):
+            paths = [paths]
+        return [str(Path(root) / path) for path in paths]

digitalhub/stores/remote/store.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import requests
+
+from digitalhub.stores._base.store import Store, StoreConfig
+from digitalhub.utils.exceptions import StoreError
+
+
+class RemoteStoreConfig(StoreConfig):
+    """
+    Remote store configuration class.
+    """
+
+
+class RemoteStore(Store):
+    """
+    HTTP store class. It implements the Store interface and provides methods to fetch
+    artifacts from remote HTTP based storage.
+    """
+
+    def __init__(self, name: str, store_type: str, config: RemoteStoreConfig) -> None:
+        super().__init__(name, store_type)
+        self.config = config
+
+    ##############################
+    # IO methods
+    ##############################
+
+    def download(
+        self,
+        root: str,
+        dst: Path,
+        src: list[str],
+        overwrite: bool = False,
+    ) -> str:
+        """
+        Download artifacts from storage.
+
+        Parameters
+        ----------
+        root : str
+            The root path of the artifact.
+        dst : str
+            The destination of the artifact on local filesystem.
+        src : list[str]
+            List of sources.
+        overwrite : bool
+            Specify if overwrite existing file(s).
+
+        Returns
+        -------
+        str
+            Destination path of the downloaded artifact.
+        """
+        # Handle destination
+        if dst is None:
+            dst = self._build_temp()
+        else:
+            self._check_local_dst(str(dst))
+
+        if dst.suffix == "":
+            dst = dst / "data.file"
+
+        self._check_overwrite(dst, overwrite)
+        self._build_path(dst)
+
+        return self._download_file(root, dst, overwrite)
+
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
+        """
+        Upload an artifact to storage.
+
+        Raises
+        ------
+        StoreError
+            This method is not implemented.
+        """
+        raise StoreError("Remote HTTP store does not support upload.")
+
+    def get_file_info(self, paths: list[str]) -> list[dict]:
+        """
+        Get file information from HTTP(s) storage.
+
+        Raises
+        ------
+        NotImplementedError
+            This method is not implemented.
+        """
+        raise NotImplementedError("Remote store does not support get_file_info.")
+
+    ##############################
+    # Private helper methods
+    ##############################
+
+    @staticmethod
+    def _check_head(src) -> None:
+        """
+        Check if the source exists.
+
+        Parameters
+        ----------
+        src : str
+            The source location.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        HTTPError
+            If an error occurs while checking the source.
+        """
+        r = requests.head(src, timeout=60)
+        r.raise_for_status()
+
+    def _download_file(self, url: str, dst: Path, overwrite: bool) -> str:
+        """
+        Method to download a file from a given url.
+
+        Parameters
+        ----------
+        url : str
+            The url of the file to download.
+        dst : Path
+            The destination of the file.
+        overwrite : bool
+            Whether to overwrite existing files.
+
+        Returns
+        -------
+        str
+            The path of the downloaded file.
+        """
+        self._check_head(url)
+        with requests.get(url, stream=True, timeout=60) as r:
+            r.raise_for_status()
+            with open(dst, "wb") as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        return str(dst)