NGPIris 5.4.2.dev4__py3-none-any.whl → 5.4.2.dev6__py3-none-any.whl

NGPIris/cli/__init__.py

@@ -78,8 +78,25 @@ def cli(context : Context, credentials : str, debug : bool, transfer_config : st
     help = "Simulate the command execution without making actual changes. Useful for testing and verification", 
     is_flag = True
 )
+@click.option(
+    "-um", 
+    "--upload_mode", 
+    help = "Choose an upload method. Default upload mode is STANDARD which uses a basic multipart upload. Use another mode than STANDARD if that mode misbehaves",
+    type = click.Choice(
+        ["STANDARD", "SIMPLE", "EQUAL_PARTS"],
+        case_sensitive = False
+    ),
+    default = "STANDARD"
+)
+@click.option(
+    "-ep", 
+    "--equal_parts", 
+    help = "Supplementary option when using the EQUAL_PARTS upload mode. Splits each file into a given number of parts. Must be a positive integer", 
+    type = int,
+    default = 5
+)
 @click.pass_context
-def upload(context : Context, bucket : str, source : str, destination : str, dry_run : bool):
+def upload(context : Context, bucket : str, source : str, destination : str, dry_run : bool, upload_mode : str, equal_parts : int):
     """
     Upload files to an HCP bucket/namespace. 
     
@@ -89,6 +106,12 @@ def upload(context : Context, bucket : str, source : str, destination : str, dry
     
     DESTINATION is the destination path on the HCP. 
     """
+
+    if equal_parts <= 0:
+        exit("Error: --equal_parts value must be a positive integer")
+    
+    upload_mode_choice = HCPHandler.UploadMode(upload_mode.lower())
+
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
     destination = add_trailing_slash(destination)
@@ -97,14 +120,14 @@ def upload(context : Context, bucket : str, source : str, destination : str, dry
         if dry_run:
             click.echo("This command would have uploaded the folder \"" + source + "\" to \"" + destination + "\"")
         else:
-            hcph.upload_folder(source, destination)
+            hcph.upload_folder(source, destination, upload_mode = upload_mode_choice, equal_parts = equal_parts)
     else:
         file_name = Path(source).name
         destination += file_name
         if dry_run:
             click.echo("This command would have uploaded the file \"" + source + "\" to \"" + destination + "\"")
         else:
-            hcph.upload_file(source, destination)
+            hcph.upload_file(source, destination, upload_mode = upload_mode_choice, equal_parts = equal_parts)
 
 @cli.command()
 @click.argument("bucket")

NGPIris/hcp/hcp.py

@@ -42,6 +42,7 @@ from urllib3 import disable_warnings
 from tqdm import tqdm
 from bitmath import TiB, Byte
 
+from enum import Enum
 from typing import Generator
 
 _KB = 1024
@@ -468,9 +469,14 @@ class HCPHandler:
                     self.download_file(object["Key"], p.as_posix(), show_progress_bar = show_progress_bar)
         else:
             raise NotADirectory(local_folder_path + " is not a directory")
+    
+    class UploadMode(Enum):
+        STANDARD = "standard"
+        SIMPLE = "simple"
+        EQUAL_PARTS = "equal_parts"
 
     @check_mounted
-    def upload_file(self, local_file_path : str, key : str = "", show_progress_bar : bool = True) -> None:
+    def upload_file(self, local_file_path : str, key : str = "", show_progress_bar : bool = True, upload_mode : UploadMode = UploadMode.STANDARD, equal_parts : int = 5) -> None:
         """
         Upload one file to the mounted bucket
 
@@ -483,6 +489,16 @@ class HCPHandler:
         :param show_progress_bar: Boolean choice of displaying a progress bar. Defaults to True
         :type show_progress_bar: bool, optional
 
+        :param upload_mode: 
+            The upload mode of the transfer is any of the following:
+                HCPHandler.UploadMode.STANDARD,
+                HCPHandler.UploadMode.SIMPLE,
+                HCPHandler.UploadMode.EQUAL_PARTS\n
+        :type upload_mode: UploadMode, optional
+
+        :param equal_parts: The number of equal parts that each file should be divided into when using the HCPHandler.UploadMode.EQUAL_PARTS mode. Default is 5
+        :type equal_parts: int, optional
+
         :raises RuntimeError: If the \"\\\" is used in the file path 
         
         :raises ObjectAlreadyExist: If the object already exist on the mounted bucket
@@ -496,13 +512,20 @@ class HCPHandler:
         if "\\" in local_file_path:
             raise RuntimeError("The \"\\\" character is not allowed in the file path")
 
-        #if self.object_exists(key):
-        #    raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket")
-        if False:
-            pass
+        if self.object_exists(key):
+            raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket")
         else:
+            file_size : int = stat(local_file_path).st_size
+
+            match upload_mode:
+                case HCPHandler.UploadMode.STANDARD:
+                    config = self.transfer_config
+                case HCPHandler.UploadMode.SIMPLE:
+                    config = TransferConfig(multipart_chunksize = file_size)
+                case HCPHandler.UploadMode.EQUAL_PARTS:
+                    config = TransferConfig(multipart_chunksize = round(file_size / equal_parts))
+
             if show_progress_bar:
-                file_size : int = stat(local_file_path).st_size
                 with tqdm(
                     total = file_size, 
                     unit = "B", 
@@ -513,7 +536,7 @@ class HCPHandler:
                         Filename = local_file_path, 
                         Bucket = self.bucket_name, 
                         Key = key,
-                        Config = self.transfer_config,
+                        Config = config,
                         Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
                     )
             else:
@@ -521,12 +544,11 @@ class HCPHandler:
                     Filename = local_file_path, 
                     Bucket = self.bucket_name, 
                     Key = key,
-                    Config = self.transfer_config,
+                    Config = config,
                 )
 
-
     @check_mounted
-    def upload_folder(self, local_folder_path : str, key : str = "", show_progress_bar : bool = True) -> None:
+    def upload_folder(self, local_folder_path : str, key : str = "", show_progress_bar : bool = True, upload_mode : UploadMode = UploadMode.STANDARD, equal_parts : int = 5) -> None:
         """
         Upload the contents of a folder to the mounted bucket
 
@@ -538,6 +560,16 @@ class HCPHandler:
 
         :param show_progress_bar: Boolean choice of displaying a progress bar. Defaults to True
         :type show_progress_bar: bool, optional
+
+        :param upload_mode: 
+            The upload mode of the transfer is any of the following:
+                HCPHandler.UploadMode.STANDARD,
+                HCPHandler.UploadMode.SIMPLE,
+                HCPHandler.UploadMode.EQUAL_PARTS\n
+        :type upload_mode: UploadMode, optional
+
+        :param equal_parts: The number of equal parts that each file should be divided into when using the HCPHandler.UploadMode.EQUAL_PARTS mode. Default is 5
+        :type equal_parts: int, optional
         """
         raise_path_error(local_folder_path)
 
@@ -546,7 +578,13 @@ class HCPHandler:
         filenames = listdir(local_folder_path)
 
         for filename in filenames:
-            self.upload_file(local_folder_path + filename, key + filename, show_progress_bar = show_progress_bar)
+            self.upload_file(
+                local_folder_path + filename, 
+                key + filename, 
+                show_progress_bar = show_progress_bar, 
+                upload_mode = upload_mode,
+                equal_parts = equal_parts
+            )
 
     @check_mounted
     def delete_objects(self, keys : list[str], verbose : bool = True) -> None:

{ngpiris-5.4.2.dev4.dist-info → ngpiris-5.4.2.dev6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: NGPIris
-Version: 5.4.2.dev4
+Version: 5.4.2.dev6
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE

{ngpiris-5.4.2.dev4.dist-info → ngpiris-5.4.2.dev6.dist-info}/RECORD

@@ -1,21 +1,21 @@
 NGPIris/__init__.py,sha256=eHaeRGl1JQ_hjsazEd4oQCXOkdpSxLIzg-ipYAAaa-s,69
-NGPIris/cli/__init__.py,sha256=yZsTn2noSZTmg1BWPKwTklGTEIqMCIse4vZVN1TORaY,14459
+NGPIris/cli/__init__.py,sha256=cvQy6XNBI-Iru5JTttr7tdclcreu-h-Lao-LeUvr3Os,15371
 NGPIris/hci/__init__.py,sha256=qZRzN2JZp8uIw46TQiqJHMIkleiVDErSoZ3-v0O_cZU,38
 NGPIris/hci/exceptions.py,sha256=wUbpUwY8u1QtuSvnzlBuql9LgGsxBl207IGsHkCt-JQ,47
 NGPIris/hci/hci.py,sha256=-7RBWPGqlINVNsl-GbjuXkcLdIF5cs4cu3OFt2FJi1E,4321
 NGPIris/hci/helpers.py,sha256=THZt9ItmEsEiuvB1v99orAx1oOMgBdGR62PQyOre5yo,2592
 NGPIris/hcp/__init__.py,sha256=yMDW3WpxsU89GNcBJ3ApeGHOmWvisfuuAsZxwlKWnpw,38
 NGPIris/hcp/exceptions.py,sha256=24irbs3JMWaVOM_gf4ByR1hfYPpEP1eXPgHvq77uuqU,361
-NGPIris/hcp/hcp.py,sha256=kmLVEvQiQRlisMj4_ObVJAfZmghV20q-P6ukRmC4Hjg,30655
+NGPIris/hcp/hcp.py,sha256=Rh9XSl21SNr-iOG26bPI0k8aZGFEfJh7bCYAQ2K2zpY,32433
 NGPIris/hcp/helpers.py,sha256=SRHX-B6TSDogK7Mz2co7VkzHtZdtMaMeH-_NoDWEDh8,1406
 NGPIris/hcp/statistics.py,sha256=8VgRC-7hWQOCRWwoKk4NHCtGTdY_4C2iHcAHEXKJ0kM,817
 NGPIris/parse_credentials/__init__.py,sha256=O9uq_F9uOtuW7giW-t10KV4VmTgfkvgaBEwQU43Sf-w,57
 NGPIris/parse_credentials/parse_credentials.py,sha256=E1NzQCMvjOa2mGLBkEYshZomNSxfSCkoc_iUhUfpBEM,2106
 NGPIris/utils/__init__.py,sha256=yPCEc1-_7r01dZ-31EKiA7Jova7ztX7XXuU95IKmKX8,59
 NGPIris/utils/utils.py,sha256=m9qwLuOoiqVNRy7B4raYNFJkigL_wE--PiWhyhVhHII,248
-ngpiris-5.4.2.dev4.dist-info/licenses/LICENSE,sha256=ZA2Q9u5AEkH_YoNNDRsz-DBJ6ZuL_foE7RsKFjXd4-c,18093
-ngpiris-5.4.2.dev4.dist-info/METADATA,sha256=MygYrvZeKbY4KH0cVwJhsFxVCx322kwFmtKgNp1eLk0,7102
-ngpiris-5.4.2.dev4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ngpiris-5.4.2.dev4.dist-info/entry_points.txt,sha256=t8ADhe3ovPm9tW3XDZq1_vKmfHY98Ei_o6WioJobNzU,117
-ngpiris-5.4.2.dev4.dist-info/top_level.txt,sha256=954WCuIJkle_SMtr7MNPapiiLLAFhlKbcz6Bp89-2V0,8
-ngpiris-5.4.2.dev4.dist-info/RECORD,,
+ngpiris-5.4.2.dev6.dist-info/licenses/LICENSE,sha256=ZA2Q9u5AEkH_YoNNDRsz-DBJ6ZuL_foE7RsKFjXd4-c,18093
+ngpiris-5.4.2.dev6.dist-info/METADATA,sha256=bcpmfub_sxQmms8g_6QefikLznHtnDRYUn8PZGpYjzU,7102
+ngpiris-5.4.2.dev6.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ngpiris-5.4.2.dev6.dist-info/entry_points.txt,sha256=t8ADhe3ovPm9tW3XDZq1_vKmfHY98Ei_o6WioJobNzU,117
+ngpiris-5.4.2.dev6.dist-info/top_level.txt,sha256=954WCuIJkle_SMtr7MNPapiiLLAFhlKbcz6Bp89-2V0,8
+ngpiris-5.4.2.dev6.dist-info/RECORD,,