NGPIris 5.3.0__tar.gz → 5.4.0__tar.gz

{ngpiris-5.3.0 → ngpiris-5.4.0}/NGPIris/cli/__init__.py

@@ -4,11 +4,7 @@ from click.core import Context
 from json import dump
 from pathlib import Path
 from boto3 import set_stream_logger
-from botocore.paginate import PageIterator, Paginator
 from typing import Any, Generator
-from os import get_terminal_size
-from math import floor
-from tabulate import tabulate
 from bitmath import Byte, TiB
 
 from NGPIris.hcp import HCPHandler
@@ -26,36 +22,9 @@ def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool, fil
     different from `list_objects` in `hcp.py` in order to make the output 
     printable in a terminal
     """
-    paginator : Paginator = hcph.s3_client.get_paginator("list_objects_v2")
-    pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name, Prefix = path)
-    (nb_of_cols, _) = get_terminal_size()
-    max_width = floor(nb_of_cols / 5)
-    if (not name_only):
-        yield tabulate(
-            [],
-            headers = ["Key", "LastModified", "ETag", "Size", "StorageClass"],
-            tablefmt = "plain",
-            stralign = "center"
-        ) + "\n" + "-"*nb_of_cols + "\n"
-    if files_only:
-        filter_string = "Contents[?!ends_with(Key, '/')][]" # filter objects that does not end with "/"
-    else:
-        filter_string = "Contents[*][]"
-    for object in pages.search(filter_string): 
-        if name_only:
-            yield str(object["Key"]) + "\n"
-        else:
-            yield tabulate(
-                [
-                    [str(object["Key"]), 
-                        str(object["LastModified"]), 
-                        str(object["ETag"]), 
-                        str(object["Size"]), 
-                        str(object["StorageClass"])]
-                ],
-                maxcolwidths = max_width,
-                tablefmt = "plain"
-            ) + "\n" + "-"*nb_of_cols + "\n"
+    objects = hcph.list_objects(path, name_only, files_only)
+    for obj in objects:
+        yield str(obj) + "\n"
 
 def object_is_folder(object_path : str, hcph : HCPHandler) -> bool:
     return (object_path[-1] == "/") and (hcph.get_object(object_path)["ContentLength"] == 0)

{ngpiris-5.3.0 → ngpiris-5.4.0}/NGPIris/hcp/hcp.py

@@ -256,51 +256,64 @@ class HCPHandler:
         response = self.get_response("/namespaces")
         list_of_buckets : list[str] = response["name"]
         return list_of_buckets
-    
+
     @check_mounted
     def list_objects(self, path_key : str = "", name_only : bool = False, files_only : bool = False) -> Generator:
         """
         List all objects in the mounted bucket as a generator. If one wishes to 
         get the result as a list, use :py:function:`list` to type cast the generator
 
-        :param path_key: Filter string for which keys to list, specifically for finding objects in certain folders.
+        :param path_key: Filter string for which keys to list, specifically for finding objects in certain folders. Defaults to \"the root\" of the bucket
         :type path_key: str, optional
         :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
         :type name_only: bool, optional
         :param files_only: If true, only yield file objects. Defaults to False
         :type files_only: bool, optional
-        :yield: A generator of all objects in a bucket
+        :yield: A generator of all objects in specified folder in a bucket
         :rtype: Generator
         """
         paginator : Paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key)
-
-        if files_only:
-            filter_string = "Contents[?!ends_with(Key, '/')][]"
-        else:
-            filter_string = "Contents[*][]"
+        pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key, Delimiter = "/")
 
-        split_path_key = len(path_key.split("/")) + 1
-
-        pages_filtered = pages.search(filter_string)
-        for object in pages_filtered:
-            # If there are no objects in the bucket, then `object` will be None,
-            # which means that we should break out of the for loop
-            if not object:
+        for page in pages:
+            page : dict | None
+            # Check if `page` is None
+            if not page:
                 break
 
-            # Split the object key by "/"
-            split_object = object["Key"].split("/")
-            # Check if the object is within the specified path_key depth
-            if len(split_object) <= split_path_key:
-                # Skip objects that are not at the desired depth
-                if (len(split_object) == split_path_key) and split_object[-1]:
-                    continue
-                
-                if name_only:
-                    yield str(object["Key"])
-                else:
-                    yield object
+            if files_only:
+                for file_object in page.get("Contents", []):
+                    file_object : dict
+                    if file_object["Key"] != path_key:
+                        file_object["IsFile"] = True
+                        if name_only:
+                            yield file_object["Key"]
+                        else:
+                            yield file_object
+            else:
+                for folder_object in page.get("CommonPrefixes", []):
+                    folder_object : dict
+                    folder_object_metadata = self.get_object(folder_object["Prefix"])
+                    
+                    if name_only: 
+                        yield folder_object["Prefix"]
+                    else:
+                        yield {
+                            "Key" : folder_object["Prefix"],
+                            "LastModified" : folder_object_metadata["LastModified"],
+                            "ETag" : folder_object_metadata["ETag"],
+                            "IsFile" : False,
+                        }
+
+                for file_object in page.get("Contents", []):
+                    file_object : dict
+                    if file_object["Key"] != path_key:
+                        file_object["IsFile"] = True
+                        if name_only:
+                            yield file_object["Key"]
+                        else:
+                            yield file_object
+        
                     
     @check_mounted
     def get_object(self, key : str) -> dict:
@@ -340,7 +353,7 @@ class HCPHandler:
             return False
 
     @check_mounted
-    def download_file(self, key : str, local_file_path : str) -> None:
+    def download_file(self, key : str, local_file_path : str, show_progress_bar : bool = True) -> None:
         """
         Download one object file from the mounted bucket
 
@@ -349,25 +362,38 @@ class HCPHandler:
 
         :param local_file_path: Path to a file on your local system where the contents of the object file can be put
         :type local_file_path: str
+
+        :param show_progress_bar: Boolean choice of displaying a progress bar. Defaults to True
+        :type show_progress_bar: bool, optional
+
+        :raises ObjectDoesNotExist: If the object does not exist in the bucket
         """
         try:
             self.get_object(key)
         except:
             raise ObjectDoesNotExist("Could not find object", "\"" + key + "\"", "in bucket", "\"" + str(self.bucket_name) + "\"")
         try:
-            file_size : int = self.s3_client.head_object(Bucket = self.bucket_name, Key = key)["ContentLength"]
-            with tqdm(
-                total = file_size, 
-                unit = "B", 
-                unit_scale = True, 
-                desc = key
-            ) as pbar:
+            if show_progress_bar:
+                file_size : int = self.s3_client.head_object(Bucket = self.bucket_name, Key = key)["ContentLength"]
+                with tqdm(
+                    total = file_size, 
+                    unit = "B", 
+                    unit_scale = True, 
+                    desc = key
+                ) as pbar:
+                    self.s3_client.download_file(
+                        Bucket = self.bucket_name, 
+                        Key = key, 
+                        Filename = local_file_path, 
+                        Config = self.transfer_config,
+                        Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
+                    )
+            else:
                 self.s3_client.download_file(
                     Bucket = self.bucket_name, 
                     Key = key, 
                     Filename = local_file_path, 
                     Config = self.transfer_config,
-                    Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
                 )
         except ClientError as e0: 
             print(str(e0))
@@ -376,7 +402,14 @@ class HCPHandler:
             raise Exception(e)
 
     @check_mounted
-    def download_folder(self, folder_key : str, local_folder_path : str, use_download_limit : bool = False, download_limit_in_bytes : Byte = TiB(1).to_Byte()) -> None:
+    def download_folder(
+            self, 
+            folder_key : str, 
+            local_folder_path : str, 
+            use_download_limit : bool = False, 
+            download_limit_in_bytes : Byte = TiB(1).to_Byte(), 
+            show_progress_bar : bool = True
+        ) -> None:  
         """
         Download multiple objects from a folder in the mounted bucket
 
@@ -392,7 +425,13 @@ class HCPHandler:
         :param download_limit_in_bytes: The optional download limit in Byte (from the package `bitmath`). Defaults to 1 TB (`TiB(1).to_Byte()`)
         :type download_limit_in_bytes: Byte, optional
 
+        :param show_progress_bar: Boolean choice of displaying a progress bar. Defaults to True
+        :type show_progress_bar: bool, optional
+        
+        :raises ObjectDoesNotExist: If the object does not exist in the bucket
+        
         :raises DownloadLimitReached: If download limit was reached while downloading files
+        
         :raises NotADirectory: If local_folder_path is not a directory
         """
         try:
@@ -409,12 +448,12 @@ class HCPHandler:
                     current_download_size_in_bytes += Byte(object["Size"])
                     if current_download_size_in_bytes >= download_limit_in_bytes and use_download_limit:
                         raise DownloadLimitReached("The download limit was reached when downloading files")
-                    self.download_file(object["Key"], p.as_posix())
+                    self.download_file(object["Key"], p.as_posix(), show_progress_bar = show_progress_bar)
         else:
             raise NotADirectory(local_folder_path + " is not a directory")
 
     @check_mounted
-    def upload_file(self, local_file_path : str, key : str = "") -> None:
+    def upload_file(self, local_file_path : str, key : str = "", show_progress_bar : bool = True) -> None:
         """
         Upload one file to the mounted bucket
 
@@ -423,6 +462,13 @@ class HCPHandler:
 
         :param key: An optional new name for the file object on the bucket. Defaults to the same name as the file
         :type key: str, optional
+
+        :param show_progress_bar: Boolean choice of displaying a progress bar. Defaults to True
+        :type show_progress_bar: bool, optional
+
+        :raises RuntimeError: If the \"\\\" is used in the file path 
+        
+        :raises ObjectAlreadyExist: If the object already exist on the mounted bucket
         """
         raise_path_error(local_file_path)
 
@@ -436,30 +482,43 @@ class HCPHandler:
         if self.object_exists(key):
             raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket")
         else:
-            file_size : int = stat(local_file_path).st_size
-            with tqdm(
-                total = file_size, 
-                unit = "B", 
-                unit_scale = True, 
-                desc = local_file_path
-            ) as pbar:
+            if show_progress_bar:
+                file_size : int = stat(local_file_path).st_size
+                with tqdm(
+                    total = file_size, 
+                    unit = "B", 
+                    unit_scale = True, 
+                    desc = local_file_path
+                ) as pbar:
+                    self.s3_client.upload_file(
+                        Filename = local_file_path, 
+                        Bucket = self.bucket_name, 
+                        Key = key,
+                        Config = self.transfer_config,
+                        Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
+                    )
+            else:
                 self.s3_client.upload_file(
                     Filename = local_file_path, 
                     Bucket = self.bucket_name, 
                     Key = key,
                     Config = self.transfer_config,
-                    Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
                 )
 
+
     @check_mounted
-    def upload_folder(self, local_folder_path : str, key : str = "") -> None:
+    def upload_folder(self, local_folder_path : str, key : str = "", show_progress_bar : bool = True) -> None:
         """
         Upload the contents of a folder to the mounted bucket
 
         :param local_folder_path: Path to the folder to be uploaded
         :type local_folder_path: str
+
         :param key: An optional new name for the folder path on the bucket. Defaults to the same name as the local folder path
         :type key: str, optional
+
+        :param show_progress_bar: Boolean choice of displaying a progress bar. Defaults to True
+        :type show_progress_bar: bool, optional
         """
         raise_path_error(local_folder_path)
 
@@ -468,7 +527,7 @@ class HCPHandler:
         filenames = listdir(local_folder_path)
 
         for filename in filenames:
-            self.upload_file(local_folder_path + filename, key + filename)
+            self.upload_file(local_folder_path + filename, key + filename, show_progress_bar = show_progress_bar)
 
     @check_mounted
     def delete_objects(self, keys : list[str], verbose : bool = True) -> None:

{ngpiris-5.3.0 → ngpiris-5.4.0/NGPIris.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: NGPIris
-Version: 5.3.0
+Version: 5.4.0
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE

{ngpiris-5.3.0/NGPIris.egg-info → ngpiris-5.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: NGPIris
-Version: 5.3.0
+Version: 5.4.0
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE

{ngpiris-5.3.0 → ngpiris-5.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "NGPIris"
-version = "5.3.0"
+version = "5.4.0"
 readme = "README.md"
 dependencies = [
     "requests >= 2.31.0",

{ngpiris-5.3.0 → ngpiris-5.4.0}/tests/test_hcp.py

@@ -56,7 +56,19 @@ def test_list_objects_without_mounting(custom_config : CustomConfig) -> None:
 
 def test_upload_file(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
-    custom_config.hcp_h.upload_file(custom_config.test_file_path) 
+    
+    # With progress bar
+    custom_config.hcp_h.upload_file(
+        custom_config.test_file_path, 
+        custom_config.test_file_path
+    )
+    
+    # Without progress bar
+    custom_config.hcp_h.upload_file(
+        custom_config.test_file_path, 
+        custom_config.test_file_path + "_no_progress_bar", 
+        show_progress_bar = False
+    )
 
 def test_upload_file_without_mounting(custom_config : CustomConfig) -> None:
     _hcp_h = custom_config.hcp_h 
@@ -104,16 +116,29 @@ def test_get_folder_without_mounting(custom_config : CustomConfig) -> None:
 
 def test_get_file_in_sub_directory(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
-    test_file = Path(custom_config.test_file_path).name 
-    assert custom_config.hcp_h.object_exists(test_file) 
-    assert custom_config.hcp_h.get_object(test_file) 
+    assert custom_config.hcp_h.object_exists(custom_config.test_file_path) 
+    assert custom_config.hcp_h.get_object(custom_config.test_file_path) 
+
+from icecream import ic
 
 def test_download_file(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
-    Path(custom_config.result_path).mkdir() 
-    test_file = Path(custom_config.test_file_path).name 
-    custom_config.hcp_h.download_file(test_file, custom_config.result_path + test_file) 
-    assert cmp(custom_config.result_path + test_file, custom_config.test_file_path) 
+    Path(custom_config.result_path).mkdir()
+    
+    test_folder_path = str(custom_config.test_file_path).rsplit("/", maxsplit = 1)[0] + "/"
+    Path(custom_config.result_path + test_folder_path).mkdir(parents = True, exist_ok = True)
+
+    # With progress bar
+    custom_config.hcp_h.download_file(custom_config.test_file_path, custom_config.result_path + custom_config.test_file_path)
+    assert cmp(custom_config.result_path + custom_config.test_file_path, custom_config.test_file_path) 
+
+    # Without progress bar
+    custom_config.hcp_h.download_file(
+        custom_config.test_file_path + "_no_progress_bar", 
+        custom_config.result_path + custom_config.test_file_path + "_no_progress_bar",
+        show_progress_bar = False
+    )
+    assert cmp(custom_config.result_path + custom_config.test_file_path, custom_config.test_file_path) 
 
 def test_download_file_without_mounting(custom_config : CustomConfig) -> None:
     _hcp_h = custom_config.hcp_h 
@@ -152,8 +177,7 @@ def test_fuzzy_search_in_bucket_without_mounting(custom_config : CustomConfig) -
 
 def test_get_object_acl(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
-    test_file = Path(custom_config.test_file_path).name 
-    custom_config.hcp_h.get_object_acl(test_file) 
+    custom_config.hcp_h.get_object_acl(custom_config.test_file_path) 
 
 def test_get_object_acl_without_mounting(custom_config : CustomConfig) -> None:
     _hcp_h = custom_config.hcp_h 
@@ -189,8 +213,8 @@ def test_get_bucket_acl_without_mounting(custom_config : CustomConfig) -> None:
 
 def test_delete_file(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
-    test_file = Path(custom_config.test_file_path).name 
-    custom_config.hcp_h.delete_object(test_file) 
+    custom_config.hcp_h.delete_object(custom_config.test_file_path) 
+    custom_config.hcp_h.delete_object(custom_config.test_file_path + "_no_progress_bar")
     custom_config.hcp_h.delete_object("a_sub_directory/a_file") 
     custom_config.hcp_h.delete_object("a_sub_directory")