accli 1.1.2__tar.gz → 1.1.4__tar.gz

{accli-1.1.2 → accli-1.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: accli
-Version: 1.1.2
+Version: 1.1.4
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)

{accli-1.1.2 → accli-1.1.4}/accli/AcceleratorTerminalCliProjectService.py

@@ -4,6 +4,7 @@ import json
 import os
 
 import requests
+from requests.adapters import HTTPAdapter
 import urllib3
 from typing import List, Tuple
 
@@ -21,7 +22,12 @@ class AccAPIError(Exception):
         self.status_code = status_code
 
 
-retries = urllib3.util.Retry(total=10, backoff_factor=1)
+retries = urllib3.util.Retry(
+    total=10,
+    backoff_factor=1,
+    status_forcelist=[408, 429, 500, 502, 503, 504],
+    allowed_methods=["HEAD", "GET", "OPTIONS", "PUT"]
+)
 
 http_client = urllib3.poolmanager.PoolManager(
     num_pools=20, retries=retries, maxsize=2000, block=True
@@ -41,8 +47,9 @@ class AcceleratorTerminalCliProjectService:
     ):
 
         self.user_token = user_token
+        self.verify_cert = verify_cert
 
-        if verify_cert:
+        if self.verify_cert:
             self.http_client = http_client
         else:
             self.http_client = http_client_wo_cert_verification
@@ -264,16 +271,22 @@ class AcceleratorTerminalCliProjectService:
             project_slug, app_bucket_id, uniqified_filename, upload_id, part_number
         )
 
-        part_upload_response = requests.put(
-            put_presigned_url,
-            data=part_data,
-            # headers=headers,
-            verify=False,
-        )
+        session = requests.Session()
+        adapter = HTTPAdapter(max_retries=retries)
+        session.mount("https://", adapter)
+        session.verify = self.verify_cert
+
+        with session.put(
+                put_presigned_url,
+                data=part_data,
+                timeout=120
+        ) as part_upload_response:
+            part_upload_response.raise_for_status()
+
+            etag = part_upload_response.headers.get("etag").replace('"', "")
 
         progress.update(task, advance=len(part_data))
 
-        etag = part_upload_response.headers.get("etag").replace('"', "")
         return part_number, etag
 
     def upload_filestream_to_accelerator(self, project_slug, filename, file_stream, progress, task,
@@ -281,7 +294,7 @@ class AcceleratorTerminalCliProjectService:
         headers = dict()
         headers["Content-Type"] = "application/octet-stream"
 
-        part_size, part_count = 200 * 1024 ** 2, -1
+        part_size, part_count = 50 * 1024 ** 2, -1
 
         upload_id = None
         app_bucket_id = None
@@ -294,13 +307,21 @@ class AcceleratorTerminalCliProjectService:
         uploaded_size = 0
         put_presigned_url = None
 
-        futures = []
-
         try:
 
             with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
 
+                active_futures = set()
+
                 while not stop:
+                    if len(active_futures) >= max_workers:
+                        done, active_futures = concurrent.futures.wait(
+                            active_futures,
+                            return_when=concurrent.futures.FIRST_COMPLETED
+                        )
+                        for f in done:
+                            parts.append(f.result())
+
                     part_number += 1
                     part_data = self.read_part_data(
                         file_stream,
@@ -331,14 +352,16 @@ class AcceleratorTerminalCliProjectService:
                             # headers=headers
                         )
 
-                    future = executor.submit(self.put_part, project_slug, app_bucket_id, uniqified_filename, upload_id,
-                                             part_number, part_data, progress, task)
-
-                    futures.append(future)
-
-            futures_results = concurrent.futures.wait(futures, return_when=concurrent.futures.ALL_COMPLETED)
+                    future = executor.submit(
+                        self.put_part, project_slug, app_bucket_id, uniqified_filename,
+                        upload_id, part_number, part_data, progress, task
+                    )
+                    active_futures.add(future)
 
-            parts = [f.result() for f in futures_results.done]
+                if active_futures:
+                    done, _ = concurrent.futures.wait(active_futures, return_when=concurrent.futures.ALL_COMPLETED)
+                    for f in done:
+                        parts.append(f.result())
 
             parts.sort(key=lambda x: x[0])
 

{accli-1.1.2 → accli-1.1.4}/accli/_version.py

@@ -1,3 +1,3 @@
 # Please strictly put double quote to use this info for git tag 
 # Read a developer guide for git tag command with regex
-VERSION = "v1.1.2"
+VERSION = "v1.1.4"

{accli-1.1.2 → accli-1.1.4}/accli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: accli
-Version: 1.1.2
+Version: 1.1.4
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)