pybioos 0.0.3__py3-none-any.whl

bioos/resource/files.py

@@ -0,0 +1,229 @@
+import datetime
+from typing import Iterable, List, Union
+
+import pandas as pd
+import tos
+from cachetools import TTLCache, cached
+from pandas import DataFrame
+from tos.credential import FederationCredentials, FederationToken
+from tos.models2 import ListedObject
+
+from bioos.config import Config
+from bioos.errors import ParameterError
+from bioos.internal.tos import TOSHandler
+from bioos.models.models import DisplayListedObject
+from bioos.utils.common_tools import SingletonType, dict_str, s3_endpoint_mapping
+
+
+class FileResource(metaclass=SingletonType):
+    TOS_RETRY_TIMES = 5
+    DEFAULT_PRE_SIGNED_TIME = 15 * 60
+
+    def __init__(self, workspace_id=str, bucket=str):
+        self.workspace_id = workspace_id
+        self.bucket = bucket
+        res = Config.service().get_tos_access({
+            'WorkspaceID': self.workspace_id,
+        })
+        self.endpoint = s3_endpoint_mapping(res["Endpoint"])
+        self.region = res["Region"]
+        self.tos_handler = TOSHandler(
+            tos.TosClientV2(ak=None,
+                            sk=None,
+                            region=None,
+                            endpoint=self.endpoint,
+                            auth=tos.FederationAuth(
+                                FederationCredentials(
+                                    self._refresh_federation_credentials),
+                                self.region),
+                            max_retry_count=self.TOS_RETRY_TIMES), self.bucket)
+
+    def __repr__(self) -> str:
+        info_dict = dict_str({
+            "name": self.bucket,
+            "endpoint": self.endpoint,
+            "region": self.region,
+            "size": f"{self.size} bytes",
+            "counts": self.counts,
+        })
+        return f"BucketInfo:\n{info_dict}"
+
+    def _refresh_federation_credentials(self) -> FederationToken:
+        res = Config.service().get_tos_access({
+            'WorkspaceID': self.workspace_id,
+        })
+
+        tos_ak = res["AccessKey"]
+        tos_sk = res["SecretKey"]
+        sts_token = res["SessionToken"]
+
+        return FederationToken(
+            tos_ak, tos_sk, sts_token,
+            (datetime.datetime.fromisoformat(res["ExpiredTime"]) -
+             datetime.timedelta(minutes=5)).timestamp())
+
+    @property
+    @cached(cache=TTLCache(maxsize=10, ttl=1))
+    def size(self) -> int:
+        """Returns the size of all files.
+
+        :return: size of all files calculated with bytes
+        :rtype: int
+        """
+        size = 0
+        for o in self._list_with_cache():
+            size += o.size
+        return size
+
+    @property
+    @cached(cache=TTLCache(maxsize=10, ttl=1))
+    def counts(self) -> int:
+        """Returns the number of all files .
+
+        :return: number of all files
+        :rtype: int
+        """
+        return len(self._list_with_cache())
+
+    @cached(cache=TTLCache(maxsize=10, ttl=1))
+    def _list_with_cache(self) -> List[ListedObject]:
+        return self.tos_handler.list_objects("", 0)
+
+    def _build_s3_url(self, file_path) -> str:
+        return f"s3://{self.bucket}/{file_path}"
+
+    def _build_https_url(self, file_path) -> str:
+        return self.tos_handler.presign_download_url(
+            file_path, self.DEFAULT_PRE_SIGNED_TIME)
+
+    def s3_urls(self, sources: Union[str, Iterable[str]]) -> List[str]:
+        """Returns the S3 URLs for all the specified files .
+
+        *Example*:
+        ::
+
+            ws = bioos.workspace("foo")
+            ws.files.s3_urls(sources = ['bar/baz.txt']) #output: ["s3://xxxxxx-foo/bar/baz.txt"]
+
+        :param sources: The name of the file or a batch of files
+        :type sources: Union[str, Iterable[str]]
+        :return: S3 URLS of the file or a batch of file
+        :rtype: List[str]
+        """
+        if isinstance(sources, str):
+            sources = [sources]
+
+        if isinstance(sources, Iterable):
+            return [self._build_s3_url(elem) for elem in sources]
+        raise ParameterError("sources")
+
+    def https_urls(self, sources: Union[str, Iterable[str]]) -> List[str]:
+        """Returns the HTTPS URLs for all the specified files .
+
+        *Example*:
+        ::
+
+            ws = bioos.workspace("foo")
+            ws.files.https_urls(sources = ['bar/baz.txt']) #output: ["https://xxxxxx-foo.tos-s3-xxxxxx.volces.com/bar/baz.txt"]
+
+        :param sources: The name of the file or a batch of files
+        :type sources: Union[str, Iterable[str]]
+        :return: Pre-signed Downloading HTTPS URLS of the file or a batch of file
+        :rtype: List[str]
+        """
+        if isinstance(sources, str):
+            sources = [sources]
+
+        if isinstance(sources, Iterable):
+            return [self._build_https_url(elem) for elem in sources]
+        raise ParameterError("sources")
+
+    def list(self) -> DataFrame:
+        """Lists all files' information .
+
+        *Example*:
+        ::
+
+            ws = bioos.workspace("foo")
+            ws.files.list()
+
+        :return: files' information
+        :rtype: DataFrame
+        """
+        files = self.tos_handler.list_objects("", 0)
+        return pd.DataFrame.from_records([
+            DisplayListedObject(f, self._build_s3_url(f.key),
+                                self._build_https_url(f.key)).__dict__
+            for f in files
+        ])
+
+    # TODO support s3 url input in the future
+    def download(self, sources: Union[str, Iterable[str]], target: str,
+                 flatten: bool) -> bool:
+        """Downloads all the specified file from internal tos bucket bound to workspace to
+        local path.
+
+        *Example*:
+        ::
+
+            ws = bioos.workspace("foo")
+            ws.files.download(sources = ['foo/bar.txt'], target="baz/", flatten=False)
+
+        :param sources: The name of the file or a batch of files from internal tos bucket to download
+        :type sources: Union[str, Iterable[str]]
+        :param target: Local path
+        :type target: str
+        :param flatten: Whether to flatten the files locally
+        :type flatten: bool
+        :return: Downloading result
+        :rtype: bool
+        """
+
+        if isinstance(sources, str):
+            sources = [sources]
+
+        return len(self.tos_handler.download_objects(sources, target,
+                                                     flatten)) == 0
+
+    def upload(self, sources: Union[str, Iterable[str]], target: str,
+               flatten: bool) -> bool:
+        """Uploads a local file or a batch of local files to internal tos bucket bound to workspace.
+
+        *Example*:
+        ::
+
+            ws = bioos.workspace("foo")
+            ws.files.upload(sources = ['foo/bar.txt'], target="baz/", flatten=False)
+        
+        :param sources: The name of the local file or a batch of local files to upload
+        :type sources: Union[str, Iterable[str]]
+        :param target: The target internal path of the file or a batch of files
+        :type target: str
+        :param flatten: Whether to flatten the files at internal
+        :type flatten: bool
+        :return: Uploading result
+        :rtype: bool
+        """
+        if isinstance(sources, str):
+            sources = [sources]
+
+        return len(self.tos_handler.upload_objects(sources, target,
+                                                   flatten)) == 0
+
+    def delete(self, sources: Union[str, Iterable[str]]) -> bool:
+        """Deletes the given file from the tos bucket bound to workspace .
+
+        *Example*:
+        ::
+
+            ws = bioos.workspace("foo")
+            ws.files.delete(sources = ['foo/bar.txt', "baz.csv"])
+
+        :param sources: The name of the file to delete
+        :type sources: Union[str, Iterable[str]]
+        :return: Deleting result
+        :rtype: bool
+        """
+        if isinstance(sources, str):
+            sources = [sources]
+        return len(self.tos_handler.delete_objects(sources)) == 0

bioos/resource/utility.py

@@ -0,0 +1,45 @@
+from typing import Iterable, Union
+
+from bioos.utils.common_tools import SingletonType
+
+
+class UtilityResource(metaclass=SingletonType):
+    """
+    **[Deprecated] This resource will be implemented in the future**
+    Common tool collection
+    """
+
+    @staticmethod
+    def upload_wdl_image(source_image: str, target_image: str) -> bool:
+        """Uploads an executable WDL image .
+
+        *Example*:
+        ::
+
+            bioos.utility.upload_wdl_image(source_image="aaa/bbb:1.0.0", target_image:"AAA/BBB:miracle-1.0.0")
+
+        :param source_image: A local executable WDL image name
+        :type source_image: str
+        :param target_image: Target image name
+        :type target_image: str
+        :return: Result of uploading
+        :rtype: bool
+        """
+        pass
+
+    @staticmethod
+    def json_transfer(
+            json_: Union[str, Iterable[str]]) -> Union[str, Iterable[str]]:
+        """Convert the local JSON to meet the requirements of MiracleCloud
+
+        *Example*:
+        ::
+
+            bioos.utility.json_transfer(json='{\"aaa\":\"bbb\"}')
+
+        :param json_: JSON or JSON list to be converted
+        :type json_: Union[str, Iterable[str]]
+        :return: JSON or JSON list after converting
+        :rtype: Union[str, Iterable[str]]
+        """
+        pass