ciocore 8.3.2__py2.py3-none-any.whl → 9.1.0b1__py2.py3-none-any.whl

ciocore/VERSION

@@ -1 +1 @@
-8.3.2
+9.1.0-beta.1

ciocore/uploader/_uploader.py

@@ -2,6 +2,7 @@ import datetime
 import json
 import logging
 import os
+import pathlib
 import requests.exceptions
 import sys
 import time
@@ -760,6 +761,31 @@ class Uploader(object):
         file_map = {path: None for path in processed_filepaths}
         self.handle_upload_response(project=None, upload_files=file_map)
 
+        if common.SIGINT_EXIT or self.cancel:
+            print("\nUpload cancelled\n")
+
+        else:
+            print("\nUpload of {} file completed\n".format(len(file_map)))
+
+        error_messages = []
+
+        for exception in self.error_messages:
+            error_messages.append(str(exception[1]))
+            print("".join(traceback.format_tb(exception[2])))
+            logger.error("".join(traceback.format_tb(exception[2])))
+
+        if error_messages:
+
+            log_file = loggeria.LOG_PATH
+            sys.stderr.write("\nError uploading files:\n")
+
+            for err_msg in error_messages:
+                sys.stderr.write("\t{}\n".format(err_msg))
+
+            sys.stderr.write("\nSee log {} for more details\n\n".format(log_file))
+
+        self.error_messages = []        
+
     def handle_upload_response(self, project, upload_files, upload_id=None):
         """
         This is a really confusing method and should probably be split into to clear logic
@@ -818,8 +844,7 @@ class Uploader(object):
                 time.sleep(5)
 
             # Shutdown the manager once all jobs are done
-            if not self.cancel and not self.manager.error:
-                logger.debug("Waiting for Manager to join")
+            if not (self.cancel or self.manager.error or common.SIGINT_EXIT):
                 self.manager.join()
 
             upload_stats = UploadStats.create(

ciocore/uploader/thread_queue_job.py

@@ -0,0 +1,101 @@
+import logging
+from ciocore import loggeria
+
+logger = logging.getLogger("{}.uploader".format(loggeria.CONDUCTOR_LOGGER_NAME))
+
+class ThreadQueueJob():
+    pass
+
+class UploadThreadQueueJob(ThreadQueueJob):
+    
+    def __init__(self, path, file_size, presigned_url, file_md5=None, upload_id=None, part_size=None, total_parts=1, part_index=1, kms_key_name=None):
+
+        super().__init__()
+
+        self.path = path
+        self.file_size = file_size 
+        self.upload_id = upload_id
+        self.presigned_url = presigned_url
+        self.file_md5 = file_md5
+        self.part_size = part_size
+        self.part_index = part_index
+        self.total_parts = total_parts
+        self.kms_key_name = kms_key_name
+
+        logger.info("Creating %s (%s): %s", str(self.__class__), str(self), str(self.__dict__))
+
+    def is_multipart(self):
+        return self.total_parts != 1
+
+    def is_vendor_aws(self):
+        return "amazonaws" in self.presigned_url
+    
+    def is_vendor_cw(self):
+        return "coreweave" in self.presigned_url    
+
+    @classmethod
+    def create_from_response(cls, response):
+
+        new_thread_queue_jobs = []
+
+        for part_type, file_request_list in response.items():
+
+            for file_request in file_request_list:
+                if part_type == "multiPartURLs":
+                        
+                        for part in file_request["parts"]:
+                            new_tqj = cls( path=file_request['filePath'],
+                                           file_size = file_request['filePath'],
+                                           presigned_url = file_request['preSignedURL'],
+                                           file_md5  = file_request['preSignedURL'],
+                                           upload_id = file_request['preSignedURL'],
+                                           part_size = file_request['preSignedURL'],
+                                           part_index = file_request['preSignedURL'])
+                            
+
+                else:
+                    new_tqj = cls( path=file_request['filePath'],
+                                   file_size = file_request['filePath'],
+                                   presigned_url = file_request['preSignedURL'])
+                    
+        new_thread_queue_jobs.append(new_tqj)
+
+
+
+class MultiPartThreadQueueJob(ThreadQueueJob):
+    
+    def __init__(self, path, md5, total_parts=1, part_index=1):
+
+        super().__init__()
+
+        self.upload_id = None
+        self.md5 = md5
+        self.project = None
+        self.path = path
+        self.part_index = part_index
+        self.etag = None
+        self.total_parts = total_parts
+
+        logger.info("Creating %s (%s): %s", str(self.__class__), str(self), str(self.__dict__))
+
+    def is_multipart(self):
+        return self.total_parts != 1
+    
+    # def __str__(self):
+    #     return
+
+    @staticmethod
+    def aggregate_parts(parts):
+        """
+        Helper function to take all the parts of a multipart upload and put 
+        them into a format that's expected for the HTTP call.
+        """
+
+        completed_parts_payload = []
+        
+        for part in parts:
+            completed_parts_payload.append({'partNumber': part.part,
+                                                       'etag': part.etag}
+                                                      )
+            
+        return completed_parts_payload

{ciocore-8.3.2.dist-info → ciocore-9.1.0b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ciocore
-Version: 8.3.2
+Version: 9.1.0b1
 Summary: Core functionality for Conductor's client tools
 Home-page: https://github.com/ConductorTechnologies/ciocore
 Author: conductor
@@ -51,6 +51,12 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 ## Changelog
 
+## Unreleased: 
+* Adds required changes to parallelize multi-part uploads
+* Cleans up the output when explicit paths are uploaded
+* Fixes logic so managers doesn't erroneously try and call join a second time if cancelled
+
+
 ## Version:8.3.2 -- 01 Oct 2024
 
 * Tweak to package order behavior for markdown package query

{ciocore-8.3.2.dist-info → ciocore-9.1.0b1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ciocore/VERSION,sha256=yMqzsb3D4LmOawN9712vzGxhLlrQgxoUp1YYCGk_0ng,5
+ciocore/VERSION,sha256=TCUlj36m9-VV6atKK8EETx-2I8BQY_6CSKTVDGTHeqo,12
 ciocore/__init__.py,sha256=aTP7LeeosQA8BZE67gDV4jgfTK5zxmwZRjiTRu_ZWj0,646
 ciocore/api_client.py,sha256=SBxEwAiwn2XtH7T_ipefUbWhczXjoNdNbQBur1RV-Bw,32810
 ciocore/cli.py,sha256=jZ1lOKQiUcrMhsVmD9SVmPMFwHtgDF4SaoAf2-PBS54,15449
@@ -26,7 +26,7 @@ ciocore/docsite/index.html,sha256=NEK4HaX2yaetTajVtQuTmS9C5cPnkAtxgeKEj7wQ9t0,20
 ciocore/docsite/logo.png,sha256=gArgFFWdw8w985-0TkuGIgU_pW9sziEMZdqytXb5WLo,2825
 ciocore/docsite/objects.inv,sha256=XwmLactPEWWC4fAWqHNPBXGsluRxLLTrwDLQqq51ONY,775
 ciocore/docsite/sitemap.xml,sha256=M_V85zl0y2adRvzJAnoCxlZH_Hl7TLnIb1A-6l_xGmI,109
-ciocore/docsite/sitemap.xml.gz,sha256=nmc2lveKFlofHpVpP6or7MslVmJqs4Y4nz-7cB32EoU,127
+ciocore/docsite/sitemap.xml.gz,sha256=mThGdpcFPBqhVOOM7T3lZOZMuSVSoDrSjqxs4psAmeY,127
 ciocore/docsite/apidoc/api_client/index.html,sha256=TvoSl4iqdXhBVfesDxe_sBPA6G-Jt1-gpWA40xXspa0,188372
 ciocore/docsite/apidoc/apidoc/index.html,sha256=GOSvv6KZPOtgekgshRE4j7aDvJkkaiBQLwA_By9J94g,26171
 ciocore/docsite/apidoc/config/index.html,sha256=WDqy4MLR3EMp9T_2-Z9Op61rTFkvb0aTWmtjiR8sbjA,72559
@@ -95,7 +95,8 @@ ciocore/downloader/perpetual_downloader.py,sha256=cD7lnBH75-c-ZVVPHZc1vSnDhgJOnG
 ciocore/downloader/registry.py,sha256=_JIOuqpWkJkgJGN33nt-DCvqN9Gw3xeFhzPq4RUxIoE,2903
 ciocore/downloader/reporter.py,sha256=p1NK9k6iQ-jt7lRvZR0xFz0cGb2yo8tQcjlvYKR9SWM,4501
 ciocore/uploader/__init__.py,sha256=hxRFJf5Lo86rtRObFXSjjot8nybQd-SebSfYCbgZwow,24
-ciocore/uploader/_uploader.py,sha256=40nzqO5DuFi4sx31VvjWxZPNkrWsWqM9jtFVxs_-o3o,37479
+ciocore/uploader/_uploader.py,sha256=Xz1sos76FK-BKsgPuErlKQ8HsCAtBKp3pLbCtRVrr9E,38250
+ciocore/uploader/thread_queue_job.py,sha256=MzOcetttfWtDfwy-M0_ARwUf8_OjaGjyy-dA_WgNTPE,3416
 ciocore/uploader/upload_stats/__init__.py,sha256=Lg1y4zq1i0cwc6Hh2K1TAQDYymLff49W-uIo1xjcvdI,5309
 ciocore/uploader/upload_stats/stats_formats.py,sha256=giNirtObU66VALWghPFSRhg3q_vw5MvESsnXhb_I3y8,2402
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -121,8 +122,8 @@ tests/test_uploader.py,sha256=B1llTJt_fqR6e_V_Jxfw9z73QgkFlEPU87xLYGzt-TQ,2914
 tests/test_validator.py,sha256=2fY66ayNc08PGyj2vTI-V_1yeCWJDngkj2zkUM5TTCI,1526
 tests/mocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/mocks/glob.py,sha256=J2MH7nqi6NJOHuGdVWxhfeBd700_Ckj6cLh_8jSNkfg,215
-ciocore-8.3.2.dist-info/METADATA,sha256=cKdUXW010yUKTuzdmz3u8iCoDekFby8TJRxpVdG31Dk,18860
-ciocore-8.3.2.dist-info/WHEEL,sha256=qUzzGenXXuJTzyjFah76kDVqDvnk-YDzY00svnrl84w,109
-ciocore-8.3.2.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
-ciocore-8.3.2.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
-ciocore-8.3.2.dist-info/RECORD,,
+ciocore-9.1.0b1.dist-info/METADATA,sha256=jd_61cbX9eiHy2SgX19hXCdf-gqvcvaOJP_gOwZVG-M,19085
+ciocore-9.1.0b1.dist-info/WHEEL,sha256=qUzzGenXXuJTzyjFah76kDVqDvnk-YDzY00svnrl84w,109
+ciocore-9.1.0b1.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
+ciocore-9.1.0b1.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
+ciocore-9.1.0b1.dist-info/RECORD,,