NGPIris 5.3.1__tar.gz → 5.4.1__tar.gz

{ngpiris-5.3.1 → ngpiris-5.4.1}/NGPIris/cli/__init__.py

@@ -4,11 +4,7 @@ from click.core import Context
 from json import dump
 from pathlib import Path
 from boto3 import set_stream_logger
-from botocore.paginate import PageIterator, Paginator
 from typing import Any, Generator
-from os import get_terminal_size
-from math import floor
-from tabulate import tabulate
 from bitmath import Byte, TiB
 
 from NGPIris.hcp import HCPHandler
@@ -26,36 +22,9 @@ def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool, fil
     different from `list_objects` in `hcp.py` in order to make the output 
     printable in a terminal
     """
-    paginator : Paginator = hcph.s3_client.get_paginator("list_objects_v2")
-    pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name, Prefix = path)
-    (nb_of_cols, _) = get_terminal_size()
-    max_width = floor(nb_of_cols / 5)
-    if (not name_only):
-        yield tabulate(
-            [],
-            headers = ["Key", "LastModified", "ETag", "Size", "StorageClass"],
-            tablefmt = "plain",
-            stralign = "center"
-        ) + "\n" + "-"*nb_of_cols + "\n"
-    if files_only:
-        filter_string = "Contents[?!ends_with(Key, '/')][]" # filter objects that does not end with "/"
-    else:
-        filter_string = "Contents[*][]"
-    for object in pages.search(filter_string): 
-        if name_only:
-            yield str(object["Key"]) + "\n"
-        else:
-            yield tabulate(
-                [
-                    [str(object["Key"]), 
-                        str(object["LastModified"]), 
-                        str(object["ETag"]), 
-                        str(object["Size"]), 
-                        str(object["StorageClass"])]
-                ],
-                maxcolwidths = max_width,
-                tablefmt = "plain"
-            ) + "\n" + "-"*nb_of_cols + "\n"
+    objects = hcph.list_objects(path, name_only, files_only)
+    for obj in objects:
+        yield str(obj) + "\n"
 
 def object_is_folder(object_path : str, hcph : HCPHandler) -> bool:
     return (object_path[-1] == "/") and (hcph.get_object(object_path)["ContentLength"] == 0)
@@ -457,7 +426,7 @@ def iris_generate_credentials_file(path : str, name : str):
             file_path = path + name
     
         if not Path(path).is_dir():
-            Path(path).mkdir()
+            Path(path).mkdir(parents=True)
     else:
         file_path = name
         

{ngpiris-5.3.1 → ngpiris-5.4.1}/NGPIris/hcp/hcp.py

@@ -1,4 +1,3 @@
-
 from NGPIris.parse_credentials import CredentialsHandler
 from NGPIris.hcp.helpers import (
     raise_path_error,
@@ -41,7 +40,6 @@ from rapidfuzz import (
 from requests import get
 from urllib3 import disable_warnings
 from tqdm import tqdm
-
 from bitmath import TiB, Byte
 
 from typing import Generator
@@ -256,51 +254,75 @@ class HCPHandler:
         response = self.get_response("/namespaces")
         list_of_buckets : list[str] = response["name"]
         return list_of_buckets
-    
+
     @check_mounted
-    def list_objects(self, path_key : str = "", name_only : bool = False, files_only : bool = False) -> Generator:
+    def list_objects(
+        self, 
+        path_key : str = "", 
+        list_all_bucket_objects : bool = False,
+        name_only : bool = False, 
+        files_only : bool = False
+    ) -> Generator:
         """
         List all objects in the mounted bucket as a generator. If one wishes to 
         get the result as a list, use :py:function:`list` to type cast the generator
 
-        :param path_key: Filter string for which keys to list, specifically for finding objects in certain folders.
+        :param path_key: Filter string for which keys to list, specifically for finding objects in certain folders. Defaults to \"the root\" of the bucket
         :type path_key: str, optional
+        :param list_all_bucket_objects: If True, the value of `path_key` will be ignored and instead will list all objects in the bucket. Defaults to False
+        :type list_all_bucket_objects: bool, optional
         :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
         :type name_only: bool, optional
-        :param files_only: If true, only yield file objects. Defaults to False
+        :param files_only: If True, only yield file objects. Defaults to False
         :type files_only: bool, optional
-        :yield: A generator of all objects in a bucket
+        :yield: A generator of all objects in specified folder in a bucket
         :rtype: Generator
         """
         paginator : Paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key)
-
-        if files_only:
-            filter_string = "Contents[?!ends_with(Key, '/')][]"
+        if list_all_bucket_objects:
+            pages : PageIterator = paginator.paginate(Bucket = self.bucket_name)
         else:
-            filter_string = "Contents[*][]"
-
-        split_path_key = len(path_key.split("/")) + 1
+            pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key, Delimiter = "/")
 
-        pages_filtered = pages.search(filter_string)
-        for object in pages_filtered:
-            # If there are no objects in the bucket, then `object` will be None,
-            # which means that we should break out of the for loop
-            if not object:
+        for page in pages:
+            page : dict | None
+            # Check if `page` is None
+            if not page:
                 break
 
-            # Split the object key by "/"
-            split_object = object["Key"].split("/")
-            # Check if the object is within the specified path_key depth
-            if len(split_object) <= split_path_key:
-                # Skip objects that are not at the desired depth
-                if (len(split_object) == split_path_key) and split_object[-1]:
-                    continue
-                
-                if name_only:
-                    yield str(object["Key"])
-                else:
-                    yield object
+            if files_only:
+                for file_object in page.get("Contents", []):
+                    file_object : dict
+                    if file_object["Key"] != path_key:
+                        file_object["IsFile"] = True
+                        if name_only:
+                            yield file_object["Key"]
+                        else:
+                            yield file_object
+            else:
+                for folder_object in page.get("CommonPrefixes", []):
+                    folder_object : dict
+                    folder_object_metadata = self.get_object(folder_object["Prefix"])
+                    
+                    if name_only: 
+                        yield folder_object["Prefix"]
+                    else:
+                        yield {
+                            "Key" : folder_object["Prefix"],
+                            "LastModified" : folder_object_metadata["LastModified"],
+                            "ETag" : folder_object_metadata["ETag"],
+                            "IsFile" : False,
+                        }
+
+                for file_object in page.get("Contents", []):
+                    file_object : dict
+                    if file_object["Key"] != path_key:
+                        file_object["IsFile"] = True
+                        if name_only:
+                            yield file_object["Key"]
+                        else:
+                            yield file_object
+        
                     
     @check_mounted
     def get_object(self, key : str) -> dict:
@@ -427,10 +449,18 @@ class HCPHandler:
             raise ObjectDoesNotExist("Could not find object", "\"" + folder_key + "\"", "in bucket", "\"" + str(self.bucket_name) + "\"")
         if Path(local_folder_path).is_dir():
             current_download_size_in_bytes = Byte(0) # For tracking download limit
-            for object in self.list_objects(folder_key): # Build the tree with directories or add files
+            (Path(local_folder_path) / Path(folder_key)).mkdir(parents = True) # Create "base folder"
+            for object in self.list_objects(folder_key): # Build the tree with directories or add files:
                 p = Path(local_folder_path) / Path(object["Key"])
-                if object["Key"][-1] == "/": # If the object is a "folder"
-                    p.mkdir()
+                if not object["IsFile"]: # If the object is a "folder"
+                    p.mkdir(parents = True)
+                    self.download_folder(
+                        folder_key = str(object["Key"]), 
+                        local_folder_path = local_folder_path,
+                        use_download_limit = use_download_limit, 
+                        show_progress_bar = show_progress_bar,
+                        download_limit_in_bytes = download_limit_in_bytes - current_download_size_in_bytes
+                    )
                 else: # If the object is a file
                     current_download_size_in_bytes += Byte(object["Size"])
                     if current_download_size_in_bytes >= download_limit_in_bytes and use_download_limit:
@@ -573,6 +603,7 @@ class HCPHandler:
             key += "/"
 
         objects : list[str] = list(self.list_objects(key, name_only = True))
+        objects.append(key) # Include the object "folder" path to be deleted
 
         if not objects:
             raise RuntimeError("\"" + key + "\"" + " is not a valid path") #TODO: change this error
@@ -640,11 +671,11 @@ class HCPHandler:
             processor = utils.default_process 
 
         if not name_only:
-            full_list = list(self.list_objects())
+            full_list = list(self.list_objects(list_all_bucket_objects = True))
 
         for item, score, index in process.extract_iter(
                 search_string, 
-                self.list_objects(name_only = True), 
+                self.list_objects(list_all_bucket_objects = True, name_only = True), 
                 scorer = fuzz.partial_ratio,
                 processor = processor
             ):

{ngpiris-5.3.1 → ngpiris-5.4.1/NGPIris.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: NGPIris
-Version: 5.3.1
+Version: 5.4.1
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -13,6 +13,7 @@ Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
 Requires-Dist: tabulate==0.9.0
+Dynamic: license-file
 
 # NGP IRIS 👀
 NGP IRIS, or just Iris, is a tool for interacting with a Hitachi Content Platform (HCP) using S3 in the `boto3` package. NGP Iris is designed with two use cases in mind:

{ngpiris-5.3.1/NGPIris.egg-info → ngpiris-5.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: NGPIris
-Version: 5.3.1
+Version: 5.4.1
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -13,6 +13,7 @@ Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
 Requires-Dist: tabulate==0.9.0
+Dynamic: license-file
 
 # NGP IRIS 👀
 NGP IRIS, or just Iris, is a tool for interacting with a Hitachi Content Platform (HCP) using S3 in the `boto3` package. NGP Iris is designed with two use cases in mind:

{ngpiris-5.3.1 → ngpiris-5.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "NGPIris"
-version = "5.3.1"
+version = "5.4.1"
 readme = "README.md"
 dependencies = [
     "requests >= 2.31.0",