NGPIris 5.4.0__tar.gz → 5.4.2__tar.gz

{ngpiris-5.4.0 → ngpiris-5.4.2}/NGPIris/cli/__init__.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from boto3 import set_stream_logger
 from typing import Any, Generator
 from bitmath import Byte, TiB
+import sys
 
 from NGPIris.hcp import HCPHandler
 
@@ -41,19 +42,32 @@ def add_trailing_slash(path : str) -> str:
     help = "Get the debug log for running a command",
     is_flag = True
 )
+@click.option(
+    "-tc", 
+    "--transfer_config", 
+    help = "Use a custom transfer config for uploads or downloads", 
+)
 @click.version_option(package_name = "NGPIris")
 @click.pass_context
-def cli(context : Context, credentials : str, debug : bool):
+def cli(context : Context, credentials : str, debug : bool, transfer_config : str):
     """
     NGP Intelligence and Repository Interface Software, IRIS. 
     
     CREDENTIALS refers to the path to the JSON credentials file.
     """
+
+    if transfer_config:
+        context.ensure_object(dict)
+        context.obj["hcph"] = HCPHandler(credentials, custom_config_path = transfer_config)
+    else:    
+        context.ensure_object(dict)
+        context.obj["hcph"] = HCPHandler(credentials)
+
     if debug:
         set_stream_logger(name="")
-        
-    context.ensure_object(dict)
-    context.obj["hcph"] = HCPHandler(credentials)
+        click.echo(
+            context.obj["hcph"].transfer_config.__dict__
+        )
 
 @cli.command()
 @click.argument("bucket")
@@ -65,8 +79,25 @@ def cli(context : Context, credentials : str, debug : bool):
     help = "Simulate the command execution without making actual changes. Useful for testing and verification", 
     is_flag = True
 )
+@click.option(
+    "-um", 
+    "--upload_mode", 
+    help = "Choose an upload method. Default upload mode is STANDARD which uses a basic multipart upload. Use another mode than STANDARD if that mode misbehaves",
+    type = click.Choice(
+        ["STANDARD", "SIMPLE", "EQUAL_PARTS"],
+        case_sensitive = False
+    ),
+    default = "STANDARD"
+)
+@click.option(
+    "-ep", 
+    "--equal_parts", 
+    help = "Supplementary option when using the EQUAL_PARTS upload mode. Splits each file into a given number of parts. Must be a positive integer", 
+    type = int,
+    default = 5
+)
 @click.pass_context
-def upload(context : Context, bucket : str, source : str, destination : str, dry_run : bool):
+def upload(context : Context, bucket : str, source : str, destination : str, dry_run : bool, upload_mode : str, equal_parts : int):
     """
     Upload files to an HCP bucket/namespace. 
     
@@ -76,6 +107,13 @@ def upload(context : Context, bucket : str, source : str, destination : str, dry
     
     DESTINATION is the destination path on the HCP. 
     """
+
+    if equal_parts <= 0:
+        click.echo("Error: --equal_parts value must be a positive integer", err=True)
+        sys.exit(1)
+    
+    upload_mode_choice = HCPHandler.UploadMode(upload_mode.lower())
+
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
     destination = add_trailing_slash(destination)
@@ -84,14 +122,14 @@ def upload(context : Context, bucket : str, source : str, destination : str, dry
         if dry_run:
             click.echo("This command would have uploaded the folder \"" + source + "\" to \"" + destination + "\"")
         else:
-            hcph.upload_folder(source, destination)
+            hcph.upload_folder(source, destination, upload_mode = upload_mode_choice, equal_parts = equal_parts)
     else:
         file_name = Path(source).name
         destination += file_name
         if dry_run:
             click.echo("This command would have uploaded the file \"" + source + "\" to \"" + destination + "\"")
         else:
-            hcph.upload_file(source, destination)
+            hcph.upload_file(source, destination, upload_mode = upload_mode_choice, equal_parts = equal_parts)
 
 @cli.command()
 @click.argument("bucket")
@@ -426,7 +464,7 @@ def iris_generate_credentials_file(path : str, name : str):
             file_path = path + name
     
         if not Path(path).is_dir():
-            Path(path).mkdir()
+            Path(path).mkdir(parents=True)
     else:
         file_path = name
         

{ngpiris-5.4.0 → ngpiris-5.4.2}/NGPIris/hcp/hcp.py

@@ -1,4 +1,3 @@
-
 from NGPIris.parse_credentials import CredentialsHandler
 from NGPIris.hcp.helpers import (
     raise_path_error,
@@ -41,9 +40,9 @@ from rapidfuzz import (
 from requests import get
 from urllib3 import disable_warnings
 from tqdm import tqdm
-
 from bitmath import TiB, Byte
 
+from enum import Enum
 from typing import Generator
 
 _KB = 1024
@@ -258,22 +257,33 @@ class HCPHandler:
         return list_of_buckets
 
     @check_mounted
-    def list_objects(self, path_key : str = "", name_only : bool = False, files_only : bool = False) -> Generator:
+    def list_objects(
+        self, 
+        path_key : str = "", 
+        list_all_bucket_objects : bool = False,
+        name_only : bool = False, 
+        files_only : bool = False
+    ) -> Generator:
         """
         List all objects in the mounted bucket as a generator. If one wishes to 
         get the result as a list, use :py:function:`list` to type cast the generator
 
         :param path_key: Filter string for which keys to list, specifically for finding objects in certain folders. Defaults to \"the root\" of the bucket
         :type path_key: str, optional
+        :param list_all_bucket_objects: If True, the value of `path_key` will be ignored and instead will list all objects in the bucket. Defaults to False
+        :type list_all_bucket_objects: bool, optional
         :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
         :type name_only: bool, optional
-        :param files_only: If true, only yield file objects. Defaults to False
+        :param files_only: If True, only yield file objects. Defaults to False
         :type files_only: bool, optional
         :yield: A generator of all objects in specified folder in a bucket
         :rtype: Generator
         """
         paginator : Paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key, Delimiter = "/")
+        if list_all_bucket_objects:
+            pages : PageIterator = paginator.paginate(Bucket = self.bucket_name)
+        else:
+            pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key, Delimiter = "/")
 
         for page in pages:
             page : dict | None
@@ -440,10 +450,18 @@ class HCPHandler:
             raise ObjectDoesNotExist("Could not find object", "\"" + folder_key + "\"", "in bucket", "\"" + str(self.bucket_name) + "\"")
         if Path(local_folder_path).is_dir():
             current_download_size_in_bytes = Byte(0) # For tracking download limit
-            for object in self.list_objects(folder_key): # Build the tree with directories or add files
+            (Path(local_folder_path) / Path(folder_key)).mkdir(parents = True) # Create "base folder"
+            for object in self.list_objects(folder_key): # Build the tree with directories or add files:
                 p = Path(local_folder_path) / Path(object["Key"])
-                if object["Key"][-1] == "/": # If the object is a "folder"
-                    p.mkdir()
+                if not object["IsFile"]: # If the object is a "folder"
+                    p.mkdir(parents = True)
+                    self.download_folder(
+                        folder_key = str(object["Key"]), 
+                        local_folder_path = local_folder_path,
+                        use_download_limit = use_download_limit, 
+                        show_progress_bar = show_progress_bar,
+                        download_limit_in_bytes = download_limit_in_bytes - current_download_size_in_bytes
+                    )
                 else: # If the object is a file
                     current_download_size_in_bytes += Byte(object["Size"])
                     if current_download_size_in_bytes >= download_limit_in_bytes and use_download_limit:
@@ -451,9 +469,14 @@ class HCPHandler:
                     self.download_file(object["Key"], p.as_posix(), show_progress_bar = show_progress_bar)
         else:
             raise NotADirectory(local_folder_path + " is not a directory")
+    
+    class UploadMode(Enum):
+        STANDARD = "standard"
+        SIMPLE = "simple"
+        EQUAL_PARTS = "equal_parts"
 
     @check_mounted
-    def upload_file(self, local_file_path : str, key : str = "", show_progress_bar : bool = True) -> None:
+    def upload_file(self, local_file_path : str, key : str = "", show_progress_bar : bool = True, upload_mode : UploadMode = UploadMode.STANDARD, equal_parts : int = 5) -> None:
         """
         Upload one file to the mounted bucket
 
@@ -466,6 +489,16 @@ class HCPHandler:
         :param show_progress_bar: Boolean choice of displaying a progress bar. Defaults to True
         :type show_progress_bar: bool, optional
 
+        :param upload_mode: 
+            The upload mode of the transfer is any of the following:
+                HCPHandler.UploadMode.STANDARD,
+                HCPHandler.UploadMode.SIMPLE,
+                HCPHandler.UploadMode.EQUAL_PARTS\n
+        :type upload_mode: UploadMode, optional
+
+        :param equal_parts: The number of equal parts that each file should be divided into when using the HCPHandler.UploadMode.EQUAL_PARTS mode. Default is 5
+        :type equal_parts: int, optional
+
         :raises RuntimeError: If the \"\\\" is used in the file path 
         
         :raises ObjectAlreadyExist: If the object already exist on the mounted bucket
@@ -482,8 +515,17 @@ class HCPHandler:
         if self.object_exists(key):
             raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket")
         else:
+            file_size : int = stat(local_file_path).st_size
+
+            match upload_mode:
+                case HCPHandler.UploadMode.STANDARD:
+                    config = self.transfer_config
+                case HCPHandler.UploadMode.SIMPLE:
+                    config = TransferConfig(multipart_chunksize = file_size)
+                case HCPHandler.UploadMode.EQUAL_PARTS:
+                    config = TransferConfig(multipart_chunksize = round(file_size / equal_parts))
+
             if show_progress_bar:
-                file_size : int = stat(local_file_path).st_size
                 with tqdm(
                     total = file_size, 
                     unit = "B", 
@@ -494,7 +536,7 @@ class HCPHandler:
                         Filename = local_file_path, 
                         Bucket = self.bucket_name, 
                         Key = key,
-                        Config = self.transfer_config,
+                        Config = config,
                         Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
                     )
             else:
@@ -502,12 +544,11 @@ class HCPHandler:
                     Filename = local_file_path, 
                     Bucket = self.bucket_name, 
                     Key = key,
-                    Config = self.transfer_config,
+                    Config = config,
                 )
 
-
     @check_mounted
-    def upload_folder(self, local_folder_path : str, key : str = "", show_progress_bar : bool = True) -> None:
+    def upload_folder(self, local_folder_path : str, key : str = "", show_progress_bar : bool = True, upload_mode : UploadMode = UploadMode.STANDARD, equal_parts : int = 5) -> None:
         """
         Upload the contents of a folder to the mounted bucket
 
@@ -519,6 +560,16 @@ class HCPHandler:
 
         :param show_progress_bar: Boolean choice of displaying a progress bar. Defaults to True
         :type show_progress_bar: bool, optional
+
+        :param upload_mode: 
+            The upload mode of the transfer is any of the following:
+                HCPHandler.UploadMode.STANDARD,
+                HCPHandler.UploadMode.SIMPLE,
+                HCPHandler.UploadMode.EQUAL_PARTS\n
+        :type upload_mode: UploadMode, optional
+
+        :param equal_parts: The number of equal parts that each file should be divided into when using the HCPHandler.UploadMode.EQUAL_PARTS mode. Default is 5
+        :type equal_parts: int, optional
         """
         raise_path_error(local_folder_path)
 
@@ -527,7 +578,13 @@ class HCPHandler:
         filenames = listdir(local_folder_path)
 
         for filename in filenames:
-            self.upload_file(local_folder_path + filename, key + filename, show_progress_bar = show_progress_bar)
+            self.upload_file(
+                local_folder_path + filename, 
+                key + filename, 
+                show_progress_bar = show_progress_bar, 
+                upload_mode = upload_mode,
+                equal_parts = equal_parts
+            )
 
     @check_mounted
     def delete_objects(self, keys : list[str], verbose : bool = True) -> None:
@@ -586,6 +643,7 @@ class HCPHandler:
             key += "/"
 
         objects : list[str] = list(self.list_objects(key, name_only = True))
+        objects.append(key) # Include the object "folder" path to be deleted
 
         if not objects:
             raise RuntimeError("\"" + key + "\"" + " is not a valid path") #TODO: change this error
@@ -653,11 +711,11 @@ class HCPHandler:
             processor = utils.default_process 
 
         if not name_only:
-            full_list = list(self.list_objects())
+            full_list = list(self.list_objects(list_all_bucket_objects = True))
 
         for item, score, index in process.extract_iter(
                 search_string, 
-                self.list_objects(name_only = True), 
+                self.list_objects(list_all_bucket_objects = True, name_only = True), 
                 scorer = fuzz.partial_ratio,
                 processor = processor
             ):

{ngpiris-5.4.0 → ngpiris-5.4.2}/NGPIris.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: NGPIris
-Version: 5.4.0
+Version: 5.4.2
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -13,6 +13,7 @@ Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
 Requires-Dist: tabulate==0.9.0
+Dynamic: license-file
 
 # NGP IRIS 👀
 NGP IRIS, or just Iris, is a tool for interacting with a Hitachi Content Platform (HCP) using S3 in the `boto3` package. NGP Iris is designed with two use cases in mind:

{ngpiris-5.4.0 → ngpiris-5.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: NGPIris
-Version: 5.4.0
+Version: 5.4.2
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -13,6 +13,7 @@ Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
 Requires-Dist: tabulate==0.9.0
+Dynamic: license-file
 
 # NGP IRIS 👀
 NGP IRIS, or just Iris, is a tool for interacting with a Hitachi Content Platform (HCP) using S3 in the `boto3` package. NGP Iris is designed with two use cases in mind:

{ngpiris-5.4.0 → ngpiris-5.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "NGPIris"
-version = "5.4.0"
+version = "5.4.2"
 readme = "README.md"
 dependencies = [
     "requests >= 2.31.0",

{ngpiris-5.4.0 → ngpiris-5.4.2}/tests/test_hcp.py

@@ -1,12 +1,10 @@
 
-
 from pathlib import Path
 from filecmp import cmp
 from typing import Any, Callable
-
 from conftest import CustomConfig
-
 from NGPIris.hcp import HCPHandler
+from icecream import ic
 
 # --------------------------- Helper fucntions ---------------------------------
 
@@ -70,6 +68,15 @@ def test_upload_file(custom_config : CustomConfig) -> None:
         show_progress_bar = False
     )
 
+    # Test every upload mode
+    for mode in HCPHandler.UploadMode:
+        ic(mode, custom_config.test_file_path + "_" + str(mode).replace(".", "_"))
+        custom_config.hcp_h.upload_file(
+            custom_config.test_file_path, 
+            custom_config.test_file_path + "_" + str(mode).replace(".", "_"),
+            upload_mode = mode
+        )
+
 def test_upload_file_without_mounting(custom_config : CustomConfig) -> None:
     _hcp_h = custom_config.hcp_h 
     _without_mounting(_hcp_h, HCPHandler.upload_file)
@@ -119,8 +126,6 @@ def test_get_file_in_sub_directory(custom_config : CustomConfig) -> None:
     assert custom_config.hcp_h.object_exists(custom_config.test_file_path) 
     assert custom_config.hcp_h.get_object(custom_config.test_file_path) 
 
-from icecream import ic
-
 def test_download_file(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
     Path(custom_config.result_path).mkdir()
@@ -215,6 +220,8 @@ def test_delete_file(custom_config : CustomConfig) -> None:
     test_mount_bucket(custom_config)
     custom_config.hcp_h.delete_object(custom_config.test_file_path) 
     custom_config.hcp_h.delete_object(custom_config.test_file_path + "_no_progress_bar")
+    for mode in HCPHandler.UploadMode:
+        custom_config.hcp_h.delete_object(custom_config.test_file_path + "_" + str(mode).replace(".", "_"))
     custom_config.hcp_h.delete_object("a_sub_directory/a_file") 
     custom_config.hcp_h.delete_object("a_sub_directory")