geoseeq 0.5.6a7__py3-none-any.whl → 0.5.6a8__py3-none-any.whl

geoseeq/cli/main.py

@@ -53,7 +53,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.5.6a7')  # remember to update setup
+    click.echo('0.5.6a8')  # remember to update setup
 
 
 @main.group('advanced')

geoseeq/cli/upload/upload.py

@@ -107,7 +107,8 @@ def cli_upload_file(state, cores, yes, private, link_type, recursive, hidden, ge
         link_type=link_type,
         progress_tracker_factory=PBarManager().get_new_bar,
         log_level=state.log_level,
-        overwrite=True
+        overwrite=True,
+        use_cache=state.use_cache,
     )
     for geoseeq_file_name, file_path in name_pairs:
         if isfile(file_path):
@@ -140,7 +141,8 @@ def cli_upload_folder(state, cores, yes, private, recursive, hidden, project_or_
         link_type='upload',
         progress_tracker_factory=PBarManager().get_new_bar,
         log_level=logging.INFO,
-        overwrite=True
+        overwrite=True,
+        use_cache=state.use_cache,
     )
     for folder_name in folder_names:
         result_folder = root_obj.result_folder(folder_name).idem()

geoseeq/cli/upload/upload_reads.py

@@ -95,6 +95,7 @@ def _do_upload(groups, module_name, link_type, lib, filepaths, overwrite, cores,
             log_level=state.log_level,
             overwrite=overwrite,
             progress_tracker_factory=PBarManager().get_new_bar,
+            use_cache=state.use_cache,
         )
         for group in groups:
             sample = lib.sample(group['sample_name']).idem()

geoseeq/file_system_cache.py

@@ -15,7 +15,7 @@ CACHE_DIR = join(
     "geoseeq"
 )
 USE_GEOSEEQ_CACHE = None
-
+GEOSEEQ_CACHE_DIR = abspath(f'{CACHE_DIR}/geoseeq_api_cache/v1/')
 
 def hash_obj(obj):
     val = obj
@@ -41,7 +41,7 @@ class FileSystemCache:
 
     @property
     def cache_dir_path(self):
-        return abspath(f'{CACHE_DIR}/geoseeq_api_cache/v1/')
+        return GEOSEEQ_CACHE_DIR
 
     def setup(self):
         if self.no_cache:

geoseeq/result/file_upload.py

@@ -1,7 +1,8 @@
 
 import time
 import json
-from os.path import basename, getsize
+import os
+from os.path import basename, getsize, join, dirname, isfile
 from pathlib import Path
 
 import requests
@@ -11,7 +12,7 @@ from geoseeq.constants import FIVE_MB
 from geoseeq.utils import md5_checksum
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from .utils import *
-
+from geoseeq.file_system_cache import GEOSEEQ_CACHE_DIR
 
 class FileChunker:
 
@@ -38,6 +39,77 @@ class FileChunker:
     def get_chunk_size(self, num):
         self.load_all_chunks()
         return len(self.loaded_parts[num])
+    
+
+class ResumableUploadTracker:
+
+    def __init__(self, filepath, chunk_size, tracker_file_prefix="gs_resumable_upload_tracker"):
+        self.open, self.upload_started = True, False
+        self.upload_id, self.urls = None, None
+        self.filepath = filepath
+        self.tracker_file = join(
+            GEOSEEQ_CACHE_DIR, 'upload',
+            tracker_file_prefix + f".{chunk_size}." + basename(filepath)
+        )
+        try:
+            os.makedirs(dirname(self.tracker_file), exist_ok=True)
+        except Exception as e:
+            logger.warning(f'Could not create resumable upload tracker directory. {e}')
+            self.open = False
+        self._loaded_parts = {}
+        self._load_parts_from_file()
+
+    def start_upload(self, upload_id, urls):
+        if not self.open:
+            return
+        if self.upload_started:
+            raise GeoseeqGeneralError("Upload has already started.")
+        blob = dict(upload_id=upload_id, urls=urls)
+        serialized = json.dumps(blob)
+        with open(self.tracker_file, "w") as f:
+            f.write(serialized + "\n")
+        self.upload_id, self.urls = upload_id, urls
+        self.upload_started = True
+    
+    def add_part(self, part_upload_info):
+        if not self.open:
+            return
+        part_id = part_upload_info["PartNumber"]
+        serialized = json.dumps(part_upload_info)
+        with open(self.tracker_file, "a") as f:
+            f.write(serialized + "\n")
+        self._loaded_parts[part_id] = part_upload_info
+        if len(self._loaded_parts) == len(self.urls):
+            self.cleanup()
+            self.open = False
+    
+    def _load_parts_from_file(self):
+        if not isfile(self.tracker_file):
+            return
+        with open(self.tracker_file, "r") as f:
+            header_blob = json.loads(f.readline())
+            self.upload_id, self.urls = header_blob["upload_id"], header_blob["urls"]
+            self.upload_started = True
+            for line in f:
+                blob = json.loads(line)
+                part_id = blob["PartNumber"]
+                self._loaded_parts[part_id] = blob
+    
+    def part_has_been_uploaded(self, part_number):
+        if not self.open:
+            return False
+        return part_number in self._loaded_parts
+    
+    def get_part_info(self, part_number):
+        return self._loaded_parts[part_number]
+    
+    def cleanup(self):
+        if not self.open:
+            return
+        try:
+            os.remove(self.tracker_file)
+        except FileNotFoundError:
+            pass
 
 
 class ResultFileUpload:
@@ -74,7 +146,10 @@ class ResultFileUpload:
         urls = response
         return upload_id, urls
     
-    def _upload_one_part(self, file_chunker, url, num, max_retries, session=None):
+    def _upload_one_part(self, file_chunker, url, num, max_retries, session=None, resumable_upload_tracker=None):
+        if resumable_upload_tracker and resumable_upload_tracker.part_has_been_uploaded(num + 1):
+            logger.info(f"Part {num + 1} has already been uploaded. Skipping.")
+            return resumable_upload_tracker.get_part_info(num + 1)
         file_chunk = file_chunker.get_chunk(num)
         attempts = 0
         while attempts < max_retries:
@@ -94,7 +169,12 @@ class ResultFileUpload:
                 if attempts == max_retries:
                     raise
                 time.sleep(10**attempts)  # exponential backoff, (10 ** 2)s default max
-        return {"ETag": http_response.headers["ETag"], "PartNumber": num + 1}
+        etag = http_response.headers["ETag"].replace('"', "")
+        blob = {"ETag": etag, "PartNumber": num + 1}
+        if resumable_upload_tracker:
+            # TODO technically not thread safe, but should be fine for now
+            resumable_upload_tracker.add_part(blob)
+        return blob
     
     def _finish_multipart_upload(self, upload_id, complete_parts):
         response = self.knex.post(
@@ -108,12 +188,12 @@ class ResultFileUpload:
         )
         response.raise_for_status()
 
-    def _upload_parts(self, file_chunker, urls, max_retries, session, progress_tracker, threads):
+    def _upload_parts(self, file_chunker, urls, max_retries, session, progress_tracker, threads, resumable_upload_tracker=None):
         if threads == 1:
             logger.info(f"Uploading parts in series for {file_chunker.filepath}")
             complete_parts = []
             for num, url in enumerate(list(urls.values())):
-                response_part = self._upload_one_part(file_chunker, url, num, max_retries, session)
+                response_part = self._upload_one_part(file_chunker, url, num, max_retries, session, resumable_upload_tracker)
                 complete_parts.append(response_part)
                 if progress_tracker: progress_tracker.update(file_chunker.get_chunk_size(num))
                 logger.info(f'Uploaded part {num + 1} of {len(urls)} for "{file_chunker.filepath}"')
@@ -124,7 +204,7 @@ class ResultFileUpload:
             futures = []
             for num, url in enumerate(list(urls.values())):
                 future = executor.submit(
-                    self._upload_one_part, file_chunker, url, num, max_retries, session
+                    self._upload_one_part, file_chunker, url, num, max_retries, session, resumable_upload_tracker
                 )
                 futures.append(future)
             complete_parts = []
@@ -148,15 +228,34 @@ class ResultFileUpload:
         session=None,
         progress_tracker=None,
         threads=1,
+        use_cache=True,
     ):
         """Upload a file to S3 using the multipart upload process."""
         logger.info(f"Uploading {filepath} to S3 using multipart upload.")
-        upload_id, urls = self._prep_multipart_upload(filepath, file_size, chunk_size, optional_fields)
+        resumable_upload_tracker = None
+        if use_cache and file_size > 10 * FIVE_MB:  # only use resumable upload tracker for larger files
+            resumable_upload_tracker = ResumableUploadTracker(filepath, chunk_size)
+        if resumable_upload_tracker and resumable_upload_tracker.upload_started:
+            upload_id, urls = resumable_upload_tracker.upload_id, resumable_upload_tracker.urls
+            logger.info(f'Resuming upload for "{filepath}", upload_id: "{upload_id}"')
+        else:
+            upload_id, urls = self._prep_multipart_upload(filepath, file_size, chunk_size, optional_fields)
+            if resumable_upload_tracker:
+                logger.info(f'Creating new resumable upload for "{filepath}", upload_id: "{upload_id}"')
+                resumable_upload_tracker.start_upload(upload_id, urls)
         logger.info(f'Starting upload for "{filepath}"')
         complete_parts = []
         file_chunker = FileChunker(filepath, chunk_size).load_all_chunks()
         if progress_tracker: progress_tracker.set_num_chunks(file_chunker.file_size)
-        complete_parts = self._upload_parts(file_chunker, urls, max_retries, session, progress_tracker, threads)
+        complete_parts = self._upload_parts(
+            file_chunker,
+            urls,
+            max_retries,
+            session,
+            progress_tracker,
+            threads,
+            resumable_upload_tracker=resumable_upload_tracker
+        )
         self._finish_multipart_upload(upload_id, complete_parts)
         logger.info(f'Finished Upload for "{filepath}"')
         return self

geoseeq/upload_download_manager.py

@@ -19,12 +19,12 @@ def _make_in_process_logger(log_level):
 
 
 def _upload_one_file(args):
-    result_file, filepath, session, progress_tracker, link_type, overwrite, log_level, parallel_uploads = args
+    result_file, filepath, session, progress_tracker, link_type, overwrite, log_level, parallel_uploads, use_cache = args
     if parallel_uploads:
         _make_in_process_logger(log_level)
     if link_type == 'upload':
         # TODO: check checksums to see if the file is the same
-        result_file.upload_file(filepath, session=session, overwrite=overwrite, progress_tracker=progress_tracker, threads=4)
+        result_file.upload_file(filepath, session=session, overwrite=overwrite, progress_tracker=progress_tracker, threads=4, use_cache=use_cache)
     else:
         result_file.link_file(link_type, filepath)
     return result_file
@@ -38,7 +38,8 @@ class GeoSeeqUploadManager:
                  link_type='upload',
                  progress_tracker_factory=None,
                  log_level=logging.WARNING,
-                 overwrite=True):
+                 overwrite=True,
+                 use_cache=True):
         self.session = session
         self.n_parallel_uploads = n_parallel_uploads
         self.progress_tracker_factory = progress_tracker_factory if progress_tracker_factory else lambda x: None
@@ -46,6 +47,7 @@ class GeoSeeqUploadManager:
         self.link_type = link_type
         self.overwrite = overwrite
         self._result_files = []
+        self.use_cache = use_cache
 
     def add_result_file(self, result_file, local_path):
         self._result_files.append((result_file, local_path))
@@ -70,7 +72,7 @@ class GeoSeeqUploadManager:
                 result_file, local_path,
                 self.session, self.progress_tracker_factory(local_path),
                 self.link_type, self.overwrite, self.log_level,
-                self.n_parallel_uploads > 1
+                self.n_parallel_uploads > 1, self.use_cache
             ) for result_file, local_path in self._result_files
         ]
         out = []

{geoseeq-0.5.6a7.dist-info → geoseeq-0.5.6a8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a7
+Version: 0.5.6a8
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a7.dist-info → geoseeq-0.5.6a8.dist-info}/RECORD

@@ -3,7 +3,7 @@ geoseeq/app.py,sha256=Y6d1UzxFLfE3RNccATbFCVi6kH3eFmzwoUbeR2Ry09A,2387
 geoseeq/blob_constructors.py,sha256=AkWpDQY0EdGMxF1p6eRspyHKubcUdiW4it-_Q7S2QWk,188
 geoseeq/bulk_creators.py,sha256=pdn-Dv7yv5SFv-PfDuQbuOnw2W4-BfIfRJVRAhM8U6s,2115
 geoseeq/constants.py,sha256=h9RURz4xs2bZyDrSGocej7ANJvRLr_H1H7JRxpNUXJM,431
-geoseeq/file_system_cache.py,sha256=fZrvqWmUtTLFOpm_qG0fz1Q0GWnw_yVOAcFVPloc75c,4101
+geoseeq/file_system_cache.py,sha256=HzVZWtwLD2fjWWSo_UfWmGeBltm9He4lP_OqzKwNGWg,4138
 geoseeq/knex.py,sha256=6fPO8F8yxgBgBXZiliMJvYYjgf_16chfJPyWLe-kpPk,7898
 geoseeq/organization.py,sha256=a9xmGDE0tQsjPJfyFkYnWagxZ8xpdeckkwvkhH6LNIk,2462
 geoseeq/pipeline.py,sha256=89mhWaecsKnm6tyRkdkaVp4dmZh62_v42Ze0oXf8OTY,9873
@@ -11,7 +11,7 @@ geoseeq/project.py,sha256=-9Y2ik0-BpT3iqh89v8VQBbdadhI58oaUP9oZK8oetc,13741
 geoseeq/remote_object.py,sha256=Es-JlAz8iLRmCpAzh1MOwUh2MqtbuQM-p8wHIBAqNlQ,7131
 geoseeq/sample.py,sha256=whgEVk6GnDJJLjn5uTOqFqRtVxZD3BgjTo7brAC5noU,7981
 geoseeq/search.py,sha256=gawad6Cx5FxJBPlYkXWb-UKAO-UC0_yhvyU9Ca1kaNI,3388
-geoseeq/upload_download_manager.py,sha256=Xk5VB5qkDRZGQj6DqXOEzUmphXltF66FAVjxKWaweUE,7090
+geoseeq/upload_download_manager.py,sha256=Nh2cahwn7qvjfuWJSru40_k7q65NBSRQBu0EMdx-Gqc,7206
 geoseeq/user.py,sha256=tol8i1UGLRrbMw5jeJDnna1ikRgrCDd50Jxz0a1lSgg,690
 geoseeq/utils.py,sha256=PDRiEQIZYTcfEV9AYvloQVvfqs5JaebcFZodAa2SUW8,3577
 geoseeq/work_orders.py,sha256=5uLVVfdKE8qh4gGaHkdBpXJGRTujuSg59knWCqEET4A,8071
@@ -22,7 +22,7 @@ geoseeq/cli/detail.py,sha256=q8Suu-j2k18knfSVFG-SWWGNsKM-n8y9RMA3LcIIi9Y,4132
 geoseeq/cli/download.py,sha256=ldpqpnRe00utb1EL1T_5CyPbFrZbtauIvOSOHtxz9qc,17656
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
 geoseeq/cli/get_eula.py,sha256=79mbUwyiF7O1r0g6UTxG9kJGQEqKuH805E6eLkPC6Y4,997
-geoseeq/cli/main.py,sha256=ONd6tMTFSLtEQ8MiL53Ig2Gncindxmpcgm8MXBWWCE8,3259
+geoseeq/cli/main.py,sha256=Suhx91FX_oXb2Zd923oln3Bwk9W1gL5WTUQYk4CgjEk,3259
 geoseeq/cli/manage.py,sha256=wGXAcVaXqE5JQEU8Jh6OlHr02nB396bpS_SFcOZdrEo,5929
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/run.py,sha256=bx2AV6VIqOSTlxUda78xl0XxcZ8TXlQx02-e7iLQPwI,3838
@@ -37,9 +37,9 @@ geoseeq/cli/shared_params/id_handlers.py,sha256=501K9sCVkI0YGDQ62vXk_DM5lMMDrdB5
 geoseeq/cli/shared_params/obj_getters.py,sha256=ZSkt6LnDkVFlNVYKgLrjzg60-6BthZMr3eeD3HNqzac,2741
 geoseeq/cli/shared_params/opts_and_args.py,sha256=4ZnXe0MfxHPIh1pqkpAHwoi1I0XXGGSBkAr0ltfSvEk,1948
 geoseeq/cli/upload/__init__.py,sha256=3C9_S9t7chmYU-2ot89NV03x-EtmsjibulErKaU9w1k,627
-geoseeq/cli/upload/upload.py,sha256=F8n7eo1lblWHPABjlUSBVmU4XLyBSPbCt1wXTDp-INE,8963
+geoseeq/cli/upload/upload.py,sha256=JsQnVyk4NLxgBtYfytORs8hgAeMGEE2o8atx1g2-Mek,9035
 geoseeq/cli/upload/upload_advanced.py,sha256=Jq5eGe-wOdrzxGWVwaFPg0BAJcW0YSx_eHEmYjJeKuA,3434
-geoseeq/cli/upload/upload_reads.py,sha256=Wh6Fvl5Ob4C5JdKow3otnnVdII1VY5G6q2VaOAWIcW0,7264
+geoseeq/cli/upload/upload_reads.py,sha256=y9qsiBLF708QwQmZXqQIH2M1Ubhz7lbviupa3lbLFTw,7303
 geoseeq/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 geoseeq/contrib/ncbi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 geoseeq/contrib/ncbi/api.py,sha256=WQeLoGA_-Zha-QeSO8_i7HpvXyD8UkV0qc5okm11KiA,1056
@@ -64,7 +64,7 @@ geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEY
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
 geoseeq/result/file_download.py,sha256=XQA5bdQJJSZIgbFcN09OvVdLq12fnA98kPCIONAkLk0,5568
-geoseeq/result/file_upload.py,sha256=dl5T3bUNfb4JdotldAQxZV0wwYcA_XUTlHOApFl8Bgw,7390
+geoseeq/result/file_upload.py,sha256=uAPPIouCbLjS5qgU0QsJZ7xFj4QrvCpmskVYQAI6ApA,11496
 geoseeq/result/result_file.py,sha256=HG1gKpgIcWImWuf6LduVLEOyW_auuQ-dWld8MNOXGLE,8433
 geoseeq/result/result_folder.py,sha256=6porOXPh7Tpxw3oX5yMRPYQzNCGYqszqmFJd3SwQmTc,11122
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
@@ -80,9 +80,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.5.6a7.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.5.6a7.dist-info/METADATA,sha256=2FR5sXg1kzhWv9LZHlcaKxgLUmGSeLpRdxqVZJN861A,4805
-geoseeq-0.5.6a7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-geoseeq-0.5.6a7.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.5.6a7.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.5.6a7.dist-info/RECORD,,
+geoseeq-0.5.6a8.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.5.6a8.dist-info/METADATA,sha256=yKLoJOHbteUBODSqcT9MpvbjKazjex-YIJjoHunZKuk,4805
+geoseeq-0.5.6a8.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+geoseeq-0.5.6a8.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.5.6a8.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.5.6a8.dist-info/RECORD,,