digitalhub 0.8.0b0__py3-none-any.whl → 0.8.0b2__py3-none-any.whl

digitalhub/stores/objects/base.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+from abc import ABCMeta, abstractmethod
+from pathlib import Path
+from tempfile import mkdtemp
+from typing import Literal
+
+from pydantic import BaseModel
+
+from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.uri_utils import map_uri_scheme
+
+
+class Store(metaclass=ABCMeta):
+    """
+    Store abstract class.
+    """
+
+    def __init__(self, name: str, store_type: str) -> None:
+        """
+        Constructor.
+
+        Parameters
+        ----------
+        name : str
+            Store name.
+        store_type : str
+            Store type. Used to choose the right store implementation.
+
+        Returns
+        -------
+        None
+        """
+        self.name = name
+        self.type = store_type
+
+    ##############################
+    # IO methods
+    ##############################
+
+    @abstractmethod
+    def download(
+        self,
+        root: str,
+        dst: Path,
+        src: list[str],
+        overwrite: bool = False,
+    ) -> str:
+        """
+        Method to download artifact from storage.
+        """
+
+    @abstractmethod
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
+        """
+        Method to upload artifact to storage.
+        """
+
+    @abstractmethod
+    def get_file_info(self, paths: list[str]) -> list[dict]:
+        """
+        Method to get file metadata.
+        """
+
+    ##############################
+    # Helpers methods
+    ##############################
+
+    def _check_local_src(self, src: str) -> None:
+        """
+        Check if the source path is local.
+
+        Parameters
+        ----------
+        src : str
+            The source path.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        StoreError
+            If the source is not a local path.
+        """
+        if map_uri_scheme(src) != "local":
+            raise StoreError(f"Source '{src}' is not a local path.")
+
+    def _check_local_dst(self, dst: str) -> None:
+        """
+        Check if the destination path is local.
+
+        Parameters
+        ----------
+        dst : str
+            The destination path.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        StoreError
+            If the destination is not a local path.
+        """
+        if map_uri_scheme(dst) != "local":
+            raise StoreError(f"Destination '{dst}' is not a local path.")
+
+    def _check_overwrite(self, dst: Path, overwrite: bool) -> None:
+        """
+        Check if destination path exists for overwrite.
+
+        Parameters
+        ----------
+        dst : Path
+            Destination path as filename.
+        overwrite : bool
+            Specify if overwrite an existing file.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        StoreError
+            If destination path exists and overwrite is False.
+        """
+        if dst.exists() and not overwrite:
+            raise StoreError(f"Destination {str(dst)} already exists.")
+
+    @staticmethod
+    def _build_path(path: str | Path) -> None:
+        """
+        Get path from store path and path.
+
+        Parameters
+        ----------
+        path : str
+            The path to build.
+
+        Returns
+        -------
+        None
+        """
+        if not isinstance(path, Path):
+            path = Path(path)
+        if path.suffix != "":
+            path = path.parent
+        path.mkdir(parents=True, exist_ok=True)
+
+    @staticmethod
+    def _build_temp() -> Path:
+        """
+        Build a temporary path.
+
+        Returns
+        -------
+        Path
+            Temporary path.
+        """
+        tmpdir = mkdtemp()
+        return Path(tmpdir)
+
+
+class StoreConfig(BaseModel):
+    """
+    Store configuration base class.
+    """
+
+
+class StoreParameters(BaseModel):
+    """
+    Store configuration class.
+    """
+
+    name: str
+    """Store id."""
+
+    type: Literal["local", "s3", "remote", "sql"]
+    """Store type to instantiate."""
+
+    config: StoreConfig = None
+    """Configuration for the store."""
+
+    is_default: bool = False
+    """Flag to determine if the store is the default one."""

digitalhub/stores/objects/local.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+from digitalhub.stores.objects.base import Store, StoreConfig
+from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.file_utils import get_file_info_from_local
+
+
+class LocalStoreConfig(StoreConfig):
+    """
+    Local store configuration class.
+    """
+
+    path: str
+    """Local path."""
+
+
+class LocalStore(Store):
+    """
+    Local store class. It implements the Store interface and provides methods to fetch and persist
+    artifacts on local filesystem based storage.
+    """
+
+    def __init__(self, name: str, store_type: str, config: LocalStoreConfig) -> None:
+        super().__init__(name, store_type)
+        self.config = config
+
+    ##############################
+    # IO methods
+    ##############################
+
+    def download(
+        self,
+        root: str,
+        dst: Path,
+        src: list[str],
+        overwrite: bool = False,
+    ) -> str:
+        """
+        Download artifacts from storage.
+
+        Parameters
+        ----------
+        root : str
+            The root path of the artifact.
+        dst : str
+            The destination of the artifact on local filesystem.
+        src : list[str]
+            List of sources.
+        overwrite : bool
+            Specify if overwrite existing file(s).
+
+        Returns
+        -------
+        str
+            Destination path of the downloaded artifact.
+        """
+        raise StoreError("Local store does not support download.")
+
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
+        """
+        Upload an artifact to storage.
+
+        Raises
+        ------
+        StoreError
+            This method is not implemented.
+        """
+        raise StoreError("Local store does not support upload.")
+
+    def get_file_info(self, paths: list[str]) -> list[dict]:
+        """
+        Method to get file metadata.
+
+        Parameters
+        ----------
+        paths : list
+            List of source paths.
+
+        Returns
+        -------
+        list[dict]
+            Returns files metadata.
+        """
+        return [get_file_info_from_local(p) for p in paths]
+
+    ##############################
+    # Private I/O methods
+    ##############################
+
+    def _get_src_dst_files(self, src: Path, dst: Path) -> list[str]:
+        """
+        Copy files from source to destination.
+
+        Parameters
+        ----------
+        src : Path
+            The source path.
+        dst : Path
+            The destination path.
+
+        Returns
+        -------
+        list[str]
+            Returns the list of destination and source paths of the
+            copied files.
+        """
+        return [self._get_src_dst_file(i, dst) for i in src.rglob("*") if i.is_file()]
+
+    def _get_src_dst_file(self, src: Path, dst: Path) -> str:
+        """
+        Copy file from source to destination.
+
+        Parameters
+        ----------
+        src : Path
+            The source path.
+        dst : Path
+            The destination path.
+
+        Returns
+        -------
+        str
+        """
+        dst_pth = self._copy_file(src, dst, True)
+        return str(dst_pth), str(src)
+
+    def _copy_dir(self, src: Path, dst: Path, overwrite: bool) -> list[str]:
+        """
+        Download file from source to destination.
+
+        Parameters
+        ----------
+        src : Path
+            The source path.
+        dst : Path
+            The destination path.
+
+        Returns
+        -------
+        list[str]
+        """
+        dst = self._rebuild_path(dst, src)
+        shutil.copytree(src, dst, dirs_exist_ok=overwrite)
+        return [str(i) for i in dst.rglob("*") if i.is_file()]
+
+    def _copy_file(self, src: Path, dst: Path, overwrite: bool) -> str:
+        """
+        Copy file from source to destination.
+
+        Parameters
+        ----------
+        src : Path
+            The source path.
+        dst : Path
+            The destination path.
+
+        Returns
+        -------
+        str
+        """
+        dst = self._rebuild_path(dst, src)
+        self._check_overwrite(dst, overwrite)
+        return str(shutil.copy2(src, dst))
+
+    def _rebuild_path(self, dst: Path, src: Path) -> Path:
+        """
+        Rebuild path.
+
+        Parameters
+        ----------
+        dst : Path
+            The destination path.
+        src : Path
+            The source path.
+
+        Returns
+        -------
+        Path
+            The rebuilt path.
+        """
+        if dst.is_dir():
+            if src.is_absolute():
+                raise StoreError("Source must be a relative path if the destination is a directory.")
+            dst = dst / src
+        self._build_path(dst)
+        return dst
+
+    ##############################
+    # Static methods
+    ##############################
+
+    @staticmethod
+    def is_partition_or_dir(path: str) -> bool:
+        """
+        Check if path is a directory or a partition.
+
+        Parameters
+        ----------
+        path : str
+            The path to check.
+
+        Returns
+        -------
+        bool
+        """
+        return Path(path).is_dir()
+
+    @staticmethod
+    def build_object_path(root: str, paths: str | list[str]) -> list[str]:
+        """
+        Method to build object path.
+
+        Parameters
+        ----------
+        root : str
+            The root of the object path.
+        paths : str | list[str]
+            The path to build.
+
+        Returns
+        -------
+        list[str]
+            Returns the path of the object.
+        """
+        if isinstance(paths, str):
+            paths = [paths]
+        return [str(Path(root) / path) for path in paths]

digitalhub/stores/objects/remote.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import requests
+
+from digitalhub.stores.objects.base import Store, StoreConfig
+from digitalhub.utils.exceptions import StoreError
+
+
+class RemoteStoreConfig(StoreConfig):
+    """
+    Remote store configuration class.
+    """
+
+
+class RemoteStore(Store):
+    """
+    HTTP store class. It implements the Store interface and provides methods to fetch
+    artifacts from remote HTTP based storage.
+    """
+
+    def __init__(self, name: str, store_type: str, config: RemoteStoreConfig) -> None:
+        super().__init__(name, store_type)
+        self.config = config
+
+    ##############################
+    # IO methods
+    ##############################
+
+    def download(
+        self,
+        root: str,
+        dst: Path,
+        src: list[str],
+        overwrite: bool = False,
+    ) -> str:
+        """
+        Download artifacts from storage.
+
+        Parameters
+        ----------
+        root : str
+            The root path of the artifact.
+        dst : str
+            The destination of the artifact on local filesystem.
+        src : list[str]
+            List of sources.
+        overwrite : bool
+            Specify if overwrite existing file(s).
+
+        Returns
+        -------
+        str
+            Destination path of the downloaded artifact.
+        """
+        # Handle destination
+        if dst is None:
+            dst = self._build_temp()
+        else:
+            self._check_local_dst(str(dst))
+
+        if dst.suffix == "":
+            dst = dst / "data.file"
+
+        self._check_overwrite(dst, overwrite)
+        self._build_path(dst)
+
+        return self._download_file(root, dst, overwrite)
+
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
+        """
+        Upload an artifact to storage.
+
+        Raises
+        ------
+        StoreError
+            This method is not implemented.
+        """
+        raise StoreError("Remote HTTP store does not support upload.")
+
+    def get_file_info(self, paths: list[str]) -> list[dict]:
+        """
+        Get file information from HTTP(s) storage.
+
+        Raises
+        ------
+        NotImplementedError
+            This method is not implemented.
+        """
+        raise NotImplementedError("Remote store does not support get_file_info.")
+
+    ##############################
+    # Private helper methods
+    ##############################
+
+    @staticmethod
+    def _check_head(src) -> None:
+        """
+        Check if the source exists.
+
+        Parameters
+        ----------
+        src : str
+            The source location.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        HTTPError
+            If an error occurs while checking the source.
+        """
+        r = requests.head(src, timeout=60)
+        r.raise_for_status()
+
+    def _download_file(self, url: str, dst: Path, overwrite: bool) -> str:
+        """
+        Method to download a file from a given url.
+
+        Parameters
+        ----------
+        url : str
+            The url of the file to download.
+        dst : Path
+            The destination of the file.
+        overwrite : bool
+            Whether to overwrite existing files.
+
+        Returns
+        -------
+        str
+            The path of the downloaded file.
+        """
+        self._check_head(url)
+        with requests.get(url, stream=True, timeout=60) as r:
+            r.raise_for_status()
+            with open(dst, "wb") as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        return str(dst)