ciocore 9.1.0rc1__py2.py3-none-any.whl → 9.1.0rc2__py2.py3-none-any.whl

ciocore/VERSION

@@ -1 +1 @@
-9.1.0-rc.1
+9.1.0-rc.2

ciocore/uploader/_uploader.py

@@ -2,7 +2,6 @@ import datetime
 import json
 import logging
 import os
-import pathlib
 import requests.exceptions
 import sys
 import time
@@ -26,12 +25,13 @@ from ciocore import (
     exceptions,
 )
 
-from . import thread_queue_job
-
 from .upload_stats import UploadStats
 
 logger = logging.getLogger("{}.uploader".format(loggeria.CONDUCTOR_LOGGER_NAME))
 
+SINGLEPART = "singlepart"
+MULTIPART = "multipart"
+
 
 class MD5Worker(worker.ThreadWorker):
     """
@@ -329,61 +329,28 @@ class FileStatWorker(worker.ThreadWorker):
         """
 
         if job:
-
-            kms_key_name = job.get('kmsKeyName')
-
             # iterate through singlepart urls
             for singlepart_upload in job.get("singlePartURLs", []):
                 path = singlepart_upload["filePath"]
                 file_size = singlepart_upload["fileSize"]
                 upload_url = singlepart_upload["preSignedURL"]
-                md5 = self.metric_store.get_dict("file_md5s", path)
 
                 self.metric_store.increment("bytes_to_upload", file_size, path)
                 self.metric_store.increment("num_files_to_upload")
                 logger.debug("Singlepart, adding task %s", path)
 
-                upload_tq_job = thread_queue_job.UploadThreadQueueJob(path,
-                                                                      file_size,
-                                                                      presigned_url=upload_url,
-                                                                      file_md5=md5,
-                                                                      upload_id=None,
-                                                                      part_size=file_size,
-                                                                      part_index=1,
-                                                                      kms_key_name=kms_key_name)
-
-                self.put_job(upload_tq_job)
+                self.put_job((path, file_size, upload_url, SINGLEPART))
 
             # iterate through multipart
             for multipart_upload in job.get("multiPartURLs", []):
                 path = multipart_upload["filePath"]
                 file_size = multipart_upload["fileSize"]
 
-                part = multipart_upload
-                total_parts = len(multipart_upload['parts'])
-                md5 = self.metric_store.get_dict("file_md5s", path)
-
-                for chunk in multipart_upload['parts']:
-                    logger.debug("Multipart, adding task %s (part %s)", path, chunk['partNumber'])
-
-                    upload_tq_job = thread_queue_job.UploadThreadQueueJob(
-                                                        path=path,
-                                                        file_size=file_size,
-                                                        presigned_url=chunk['url'],
-                                                        file_md5=md5,
-                                                        upload_id=multipart_upload['uploadID'],
-                                                        part_size=multipart_upload['partSize'],
-                                                        total_parts=total_parts,
-                                                        part_index=chunk['partNumber'],
-                                                        kms_key_name=kms_key_name)
-
-
-                    part['parts'] = chunk
-                    self.put_job(upload_tq_job)
-
                 self.metric_store.increment("bytes_to_upload", file_size, path)
                 self.metric_store.increment("num_files_to_upload")
- 
+                logger.debug("Multipart, adding task %s", path)
+                self.put_job((path, file_size, multipart_upload, MULTIPART))
+
         # make sure we return None, so no message is automatically added to the out_queue
         return None
 
@@ -415,30 +382,45 @@ class UploadWorker(worker.ThreadWorker):
                 self.metric_store.increment("bytes_uploaded", len(data), filename)
 
     def do_work(self, job, thread_int):
-        
         if job:
-        
-            md5 = self.metric_store.get_dict("file_md5s", job.path)
+            kms_key_name = None
 
             try:
-                if job.is_multipart():
-                    return self.do_multipart_upload(job)
-                    
-                else:
-                    return self.do_singlepart_upload(job)
+                filename = job[0]
+                file_size = job[1]
+                upload = job[2]
+                upload_type = job[3]
 
-            except Exception as err_msg:
-                real_md5 = common.get_base64_md5(job.path)
+            except Exception:
+                logger.error("Issue with job (%s): %s", len(job), job)
+                raise
+
+            if len(job) > 4:
+                kms_key_name = job[4]
+
+            md5 = self.metric_store.get_dict("file_md5s", filename)
+
+            try:
+                if upload_type == SINGLEPART:
+                    return self.do_singlepart_upload(
+                        upload, filename, file_size, md5, kms_key_name
+                    )
+                elif upload_type == MULTIPART:
+                    return self.do_multipart_upload(upload, filename, md5)
 
-                exc_tb = sys.exc_info()[2]
-                exception_line_num = exc_tb.tb_lineno
-                exception_file = pathlib.Path(exc_tb.tb_frame.f_code.co_filename).name
+                raise Exception(
+                    "upload_type is '%s' expected %s or %s"
+                    % (upload_type, SINGLEPART, MULTIPART)
+                )
+
+            except Exception as err_msg:
+                real_md5 = common.get_base64_md5(filename)
 
                 if isinstance(err_msg, requests.exceptions.HTTPError):
-                    error_message = f"Upload of {job.path} failed with a response code {err_msg.response.status_code} ({err_msg.response.reason}) (expected '{job.md5}', got '{real_md5}')"
+                    error_message = f"Upload of {filename} failed with a response code {err_msg.response.status_code} ({err_msg.response.reason}) (expected '{md5}', got '{real_md5}')"
                 else:
                     error_message = (
-                        f"Upload of {job.path} failed. (expected '{job.file_md5}', got '{real_md5}') {str(err_msg)} [{exception_file}-{exception_line_num}]"
+                        f"Upload of {filename} failed. (expected '{md5}', got '{real_md5}') {str(err_msg)}"
                     )
 
                 logger.error(error_message)
@@ -447,7 +429,9 @@ class UploadWorker(worker.ThreadWorker):
         return worker.EMPTY_JOB
 
     @common.DecRetry(retry_exceptions=api_client.CONNECTION_EXCEPTIONS, tries=5)
-    def do_singlepart_upload(self, job):
+    def do_singlepart_upload(
+        self, upload_url, filename, file_size, md5, kms_key_name=None
+    ):
         """
         Note that for GCS we don't rely on the make_request's own retry mechanism because we need to
         recreate the chunked_reader generator before retrying the request. Instead, we wrap this
@@ -457,23 +441,19 @@ class UploadWorker(worker.ThreadWorker):
         headers that S3 does not accept.
         """
 
-        tq_job = thread_queue_job.MultiPartThreadQueueJob( md5=job.file_md5,
-                                                           path=job.path,
-                                                           total_parts=job.total_parts)
-
-        if job.is_vendor_aws() or job.is_vendor_cw():
+        if ("amazonaws" in upload_url) or ("coreweave" in upload_url):
             # must declare content-length ourselves due to zero byte bug in requests library.
             # api_client.make_prepared_request docstring.
             headers = {
                 "Content-Type": "application/octet-stream",
-                "Content-Length": str(job.file_size),
+                "Content-Length": str(file_size),
             }
 
-            with open(job.path, "rb") as fh:
+            with open(filename, "rb") as fh:
                 # TODO: support chunked
                 response = self.api_client.make_prepared_request(
                     verb="PUT",
-                    url=job.presigned_url,
+                    url=upload_url,
                     headers=headers,
                     params=None,
                     data=fh,
@@ -487,26 +467,25 @@ class UploadWorker(worker.ThreadWorker):
                 response.close()
 
                 # report upload progress
-                self.metric_store.increment("bytes_uploaded", job.file_size, job.path)
-                
+                self.metric_store.increment("bytes_uploaded", file_size, filename)
+
+                return response
         else:
             headers = {"Content-MD5": md5, "Content-Type": "application/octet-stream"}
 
-            if job.kms_key_name is not None:
-                headers["x-goog-encryption-kms-key-name"] = job.kms_key_name
+            if kms_key_name:
+                headers["x-goog-encryption-kms-key-name"] = kms_key_name
 
-            self.api_client.make_request(
-                conductor_url=job.presigned_url,
+            return self.api_client.make_request(
+                conductor_url=upload_url,
                 headers=headers,
-                data=self.chunked_reader(job.path),
+                data=self.chunked_reader(filename),
                 verb="PUT",
                 tries=1,
                 use_api_key=True,
             )
-        
-        return tq_job
 
-    def do_multipart_upload(self, job):
+    def do_multipart_upload(self, upload, filename, md5):
         """
         Files will be split into partSize returned by the FileAPI and hydrated once all parts are
         uploaded. On successful part upload, response headers will contain an ETag. This value must
@@ -514,32 +493,42 @@ class UploadWorker(worker.ThreadWorker):
         """
         uploads = []
         complete_payload = {
-            "uploadID": job.upload_id,
-            "hash": job.file_md5,
+            "uploadID": upload["uploadID"],
+            "hash": md5,
             "completedParts": [],
             "project": self.project,
         }
 
-        tq_job = thread_queue_job.MultiPartThreadQueueJob(path=job.path,
-                                                          md5=job.file_md5,
-                                                          total_parts=job.total_parts,
-                                                          part_index=job.part_index)
-        tq_job.upload_id = job.upload_id
-        tq_job.project = self.project
+        # iterate over parts and upload
+        for part in upload["parts"]:
+            resp_headers = self._do_multipart_upload(
+                upload_url=part["url"],
+                filename=filename,
+                part_number=part["partNumber"],
+                part_size=upload["partSize"],
+            )
 
+            if resp_headers:
+                uploads.append(upload["uploadID"])
+                completed_part = {
+                    "partNumber": part["partNumber"],
+                    "etag": resp_headers["ETag"].strip('"'),
+                }
+                complete_payload["completedParts"].append(completed_part)
 
-        resp_headers = self._do_multipart_upload(
-            upload_url=job.presigned_url,
-            filename=job.path,
-            part_number=job.part_index,
-            part_size=job.part_size,
+        # Complete multipart upload in order to hydrate file for availability
+        uri_path = "/api/v2/files/multipart/complete"
+        headers = {"Content-Type": "application/json"}
+        self.api_client.make_request(
+            uri_path=uri_path,
+            verb="POST",
+            headers=headers,
+            data=json.dumps(complete_payload),
+            raise_on_error=True,
+            use_api_key=True,
         )
 
-        if resp_headers:
-            tq_job.part = job.part_index
-            tq_job.etag = resp_headers["ETag"].strip('"')
-
-        return tq_job
+        return uploads
 
     @common.DecRetry(retry_exceptions=api_client.CONNECTION_EXCEPTIONS, tries=5)
     def _do_multipart_upload(self, upload_url, filename, part_number, part_size):
@@ -574,73 +563,6 @@ class UploadWorker(worker.ThreadWorker):
 
             return response.headers
 
-
-class MultiPartSiphonWorker(worker.ThreadWorker):
-    def __init__(self, *args, **kwargs):
-        super(MultiPartSiphonWorker, self).__init__(*args, **kwargs)
-        
-        self.api_client = api_client.ApiClient()
-        self.multipart_siphon = {}
-
-    def do_work(self, job, thread_int):
-        """
-        Process files that have already been uploaded.
-
-        If it's a single-part file, add the job to the out queue, so that it can
-        be used to determine if the Upload entity is complete.
-
-        If it's a multi-part upload, collect all the parts together. Once all the
-        parts have been accumulated, mark it as complete and add the file to the
-        out queue.
-        """
-
-        if job:
-
-            if not job.is_multipart():
-                logger.debug("Job is not multipart (%s, %s)", job.total_parts, job.part_index)
-
-            else:
-
-                if job.md5 not in self.multipart_siphon:
-                    self.multipart_siphon[job.md5] = []
-                    
-                    # Add to the task count for this worker.
-                    # -1 because a task has already been added for a single file
-                    # but not all its parts.
-                    old_task_count = self.task_count
-                    self.task_count += job.total_parts - 1
-                    logger.debug("Incrementing task count to %s from %s", self.task_count, old_task_count)
-                
-                self.multipart_siphon[job.md5].append(job)
-
-                if len(self.multipart_siphon[job.md5]) == job.total_parts:
-
-                    complete_payload = {
-                        "uploadID": job.upload_id,
-                        "hash": job.md5,
-                        "completedParts": thread_queue_job.MultiPartThreadQueueJob.aggregate_parts(self.multipart_siphon[job.md5]),
-                        "project": job.project,
-                    }
-
-                    # Complete multipart upload in order to hydrate file for availability
-                    uri_path = "/api/v2/files/multipart/complete"
-                    headers = {"Content-Type": "application/json"}
-                    self.api_client.make_request(
-                        uri_path=uri_path,
-                        verb="POST",
-                        headers=headers,
-                        data=json.dumps(complete_payload),
-                        raise_on_error=True,
-                        use_api_key=True,
-                    )
-
-                    logger.debug("JSON payload: '%s'", json.dumps(complete_payload))
-
-            return job
- 
-        # make sure we return None, so no message is automatically added to the out_queue
-        return None
-
     def is_complete(self):
         # Get the number of files already uploaded as they are not passed to the Upload
         # worker
@@ -658,11 +580,11 @@ class MultiPartSiphonWorker(worker.ThreadWorker):
                 self.task_count,
             )
 
-            return (queue_size) >= self.task_count
+            return (queue_size + already_completed_uploads) >= self.task_count
 
         else:
             logger.debug("Is complete?: files not initialized yet")
-            return False         
+            return False
 
 
 class Uploader(object):
@@ -716,7 +638,6 @@ class Uploader(object):
             ),
             (FileStatWorker, [], {"thread_count": 1}),
             (UploadWorker, [], {"thread_count": self.args["thread_count"]}),
-            (MultiPartSiphonWorker, [], {"thread_count": 1})
         ]
 
         manager = worker.JobManager(job_description)        
@@ -839,31 +760,6 @@ class Uploader(object):
         file_map = {path: None for path in processed_filepaths}
         self.handle_upload_response(project=None, upload_files=file_map)
 
-        if common.SIGINT_EXIT or self.cancel:
-            print("\nUpload cancelled\n")
-
-        else:
-            print("\nUpload of {} file completed\n".format(len(file_map)))
-
-        error_messages = []
-
-        for exception in self.error_messages:
-            error_messages.append(str(exception[1]))
-            print("".join(traceback.format_tb(exception[2])))
-            logger.error("".join(traceback.format_tb(exception[2])))
-
-        if error_messages:
-
-            log_file = loggeria.LOG_PATH
-            sys.stderr.write("\nError uploading files:\n")
-
-            for err_msg in error_messages:
-                sys.stderr.write("\t{}\n".format(err_msg))
-
-            sys.stderr.write("\nSee log {} for more details\n\n".format(log_file))
-
-        self.error_messages = []        
-
     def handle_upload_response(self, project, upload_files, upload_id=None):
         """
         This is a really confusing method and should probably be split into to clear logic
@@ -922,7 +818,8 @@ class Uploader(object):
                 time.sleep(5)
 
             # Shutdown the manager once all jobs are done
-            if not (self.cancel or self.manager.error or common.SIGINT_EXIT):
+            if not self.cancel and not self.manager.error:
+                logger.debug("Waiting for Manager to join")
                 self.manager.join()
 
             upload_stats = UploadStats.create(

{ciocore-9.1.0rc1.dist-info → ciocore-9.1.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ciocore
-Version: 9.1.0rc1
+Version: 9.1.0rc2
 Summary: Core functionality for Conductor's client tools
 Home-page: https://github.com/ConductorTechnologies/ciocore
 Author: conductor
@@ -50,18 +50,15 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 ## Changelog
 
-## Version:9.1.0-rc.1 -- 11 Dec 2024
+## Version:9.1.0-rc.2 -- 12 Dec 2024
 
-* Adds required changes to parallelize multi-part uploads
-* Cleans up the output when explicit paths are uploaded
-* Fixes logic so managers doesn't erroneously try and call join a second time if cancelled
-* Use the new required jwt parameters
-* Removing py2.7 compatibility
+  * Use the new required jwt parameters
+  * Removing py2.7 compatibility
 
 ## Version:8.3.3 -- 04 Dec 2024
 
 * Adds filter option for API data query
- 
+
 ## Version:8.3.2 -- 01 Oct 2024
 
 * Tweak to package order behavior for markdown package query

{ciocore-9.1.0rc1.dist-info → ciocore-9.1.0rc2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ciocore/VERSION,sha256=ax2aQwModu65ZstFnKm4dTChIjzu_0ljZcaKGDaGkSA,10
+ciocore/VERSION,sha256=rK2e-XPKebjaWLwloD4dWey9U5CMdVkZh1nVx-IOI3s,10
 ciocore/__init__.py,sha256=aTP7LeeosQA8BZE67gDV4jgfTK5zxmwZRjiTRu_ZWj0,646
 ciocore/api_client.py,sha256=KKL7TsYygNcfkFZDPPq1CSJsrVN_QLK4PqP44vXsCQg,33101
 ciocore/cli.py,sha256=jZ1lOKQiUcrMhsVmD9SVmPMFwHtgDF4SaoAf2-PBS54,15449
@@ -94,8 +94,7 @@ ciocore/downloader/perpetual_downloader.py,sha256=cD7lnBH75-c-ZVVPHZc1vSnDhgJOnG
 ciocore/downloader/registry.py,sha256=_JIOuqpWkJkgJGN33nt-DCvqN9Gw3xeFhzPq4RUxIoE,2903
 ciocore/downloader/reporter.py,sha256=p1NK9k6iQ-jt7lRvZR0xFz0cGb2yo8tQcjlvYKR9SWM,4501
 ciocore/uploader/__init__.py,sha256=hxRFJf5Lo86rtRObFXSjjot8nybQd-SebSfYCbgZwow,24
-ciocore/uploader/_uploader.py,sha256=Kt4toITJHZDMjRLqRyw_lwe_HOoWz2AigMp2k5heHBI,42291
-ciocore/uploader/thread_queue_job.py,sha256=MzOcetttfWtDfwy-M0_ARwUf8_OjaGjyy-dA_WgNTPE,3416
+ciocore/uploader/_uploader.py,sha256=40nzqO5DuFi4sx31VvjWxZPNkrWsWqM9jtFVxs_-o3o,37479
 ciocore/uploader/upload_stats/__init__.py,sha256=Lg1y4zq1i0cwc6Hh2K1TAQDYymLff49W-uIo1xjcvdI,5309
 ciocore/uploader/upload_stats/stats_formats.py,sha256=giNirtObU66VALWghPFSRhg3q_vw5MvESsnXhb_I3y8,2402
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -119,8 +118,8 @@ tests/test_uploader.py,sha256=JGp6GEyqRXRtbQSb-IW-cCX_BzNotWdCbnJnLwZvpUM,2869
 tests/test_validator.py,sha256=2fY66ayNc08PGyj2vTI-V_1yeCWJDngkj2zkUM5TTCI,1526
 tests/mocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/mocks/glob.py,sha256=J2MH7nqi6NJOHuGdVWxhfeBd700_Ckj6cLh_8jSNkfg,215
-ciocore-9.1.0rc1.dist-info/METADATA,sha256=R_i2XrMj2RbhWLQrczzrbq71Ufz_3Q37WY3d0qIe2m8,19201
-ciocore-9.1.0rc1.dist-info/WHEEL,sha256=qUzzGenXXuJTzyjFah76kDVqDvnk-YDzY00svnrl84w,109
-ciocore-9.1.0rc1.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
-ciocore-9.1.0rc1.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
-ciocore-9.1.0rc1.dist-info/RECORD,,
+ciocore-9.1.0rc2.dist-info/METADATA,sha256=5ckG7OoC_iQTAE6keOl-Sc3GgfEihq4xc-2BvA10ZnU,18999
+ciocore-9.1.0rc2.dist-info/WHEEL,sha256=qUzzGenXXuJTzyjFah76kDVqDvnk-YDzY00svnrl84w,109
+ciocore-9.1.0rc2.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
+ciocore-9.1.0rc2.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
+ciocore-9.1.0rc2.dist-info/RECORD,,

ciocore/uploader/thread_queue_job.py

@@ -1,101 +0,0 @@
-import logging
-from ciocore import loggeria
-
-logger = logging.getLogger("{}.uploader".format(loggeria.CONDUCTOR_LOGGER_NAME))
-
-class ThreadQueueJob():
-    pass
-
-class UploadThreadQueueJob(ThreadQueueJob):
-    
-    def __init__(self, path, file_size, presigned_url, file_md5=None, upload_id=None, part_size=None, total_parts=1, part_index=1, kms_key_name=None):
-
-        super().__init__()
-
-        self.path = path
-        self.file_size = file_size 
-        self.upload_id = upload_id
-        self.presigned_url = presigned_url
-        self.file_md5 = file_md5
-        self.part_size = part_size
-        self.part_index = part_index
-        self.total_parts = total_parts
-        self.kms_key_name = kms_key_name
-
-        logger.info("Creating %s (%s): %s", str(self.__class__), str(self), str(self.__dict__))
-
-    def is_multipart(self):
-        return self.total_parts != 1
-
-    def is_vendor_aws(self):
-        return "amazonaws" in self.presigned_url
-    
-    def is_vendor_cw(self):
-        return "coreweave" in self.presigned_url    
-
-    @classmethod
-    def create_from_response(cls, response):
-
-        new_thread_queue_jobs = []
-
-        for part_type, file_request_list in response.items():
-
-            for file_request in file_request_list:
-                if part_type == "multiPartURLs":
-                        
-                        for part in file_request["parts"]:
-                            new_tqj = cls( path=file_request['filePath'],
-                                           file_size = file_request['filePath'],
-                                           presigned_url = file_request['preSignedURL'],
-                                           file_md5  = file_request['preSignedURL'],
-                                           upload_id = file_request['preSignedURL'],
-                                           part_size = file_request['preSignedURL'],
-                                           part_index = file_request['preSignedURL'])
-                            
-
-                else:
-                    new_tqj = cls( path=file_request['filePath'],
-                                   file_size = file_request['filePath'],
-                                   presigned_url = file_request['preSignedURL'])
-                    
-        new_thread_queue_jobs.append(new_tqj)
-
-
-
-class MultiPartThreadQueueJob(ThreadQueueJob):
-    
-    def __init__(self, path, md5, total_parts=1, part_index=1):
-
-        super().__init__()
-
-        self.upload_id = None
-        self.md5 = md5
-        self.project = None
-        self.path = path
-        self.part_index = part_index
-        self.etag = None
-        self.total_parts = total_parts
-
-        logger.info("Creating %s (%s): %s", str(self.__class__), str(self), str(self.__dict__))
-
-    def is_multipart(self):
-        return self.total_parts != 1
-    
-    # def __str__(self):
-    #     return
-
-    @staticmethod
-    def aggregate_parts(parts):
-        """
-        Helper function to take all the parts of a multipart upload and put 
-        them into a format that's expected for the HTTP call.
-        """
-
-        completed_parts_payload = []
-        
-        for part in parts:
-            completed_parts_payload.append({'partNumber': part.part,
-                                                       'etag': part.etag}
-                                                      )
-            
-        return completed_parts_payload