geoseeq 0.5.6a12__py3-none-any.whl → 0.5.6a14__py3-none-any.whl

geoseeq/cli/main.py

@@ -53,7 +53,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.5.6a12')  # remember to update setup
+    click.echo('0.5.6a14')  # remember to update setup
 
 
 @main.group('advanced')

geoseeq/cli/upload/upload.py

@@ -25,6 +25,7 @@ from geoseeq.cli.shared_params import (
     project_or_sample_id_arg,
     handle_project_or_sample_id,
     no_new_versions_option,
+    ignore_errors_option,
 )
 from geoseeq.upload_download_manager import GeoSeeqUploadManager
 
@@ -39,6 +40,7 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
 @click.option('--cores', default=1, help='Number of uploads to run in parallel', show_default=True)
 @click.option('--threads-per-upload', default=4, help='Number of threads used to upload each file', show_default=True)
 @click.option('--num-retries', default=3, help='Number of times to retry a failed upload', show_default=True)
+@ignore_errors_option
 @yes_option
 @private_option
 @link_option
@@ -50,7 +52,7 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
               show_default=True)
 @folder_id_arg
 @click.argument('file_paths', type=click.Path(exists=True), nargs=-1)
-def cli_upload_file(state, cores, threads_per_upload, num_retries, yes, private, link_type, recursive, hidden, no_new_versions, geoseeq_file_name, folder_id, file_paths):
+def cli_upload_file(state, cores, threads_per_upload, num_retries, ignore_errors, yes, private, link_type, recursive, hidden, no_new_versions, geoseeq_file_name, folder_id, file_paths):
     """Upload files to GeoSeeq.
 
     This command uploads files to either a sample or project on GeoSeeq. It can be used to upload
@@ -118,6 +120,8 @@ def cli_upload_file(state, cores, threads_per_upload, num_retries, yes, private,
         no_new_versions=no_new_versions,
         use_cache=state.use_cache,
         num_retries=num_retries,
+        ignore_errors=ignore_errors,
+        session=knex.new_session(),
     )
     for geoseeq_file_name, file_path in name_pairs:
         if isfile(file_path):

geoseeq/knex.py

@@ -96,6 +96,9 @@ class Knex:
         sess.auth = self.auth
         sess.verify = self._verify
         return sess
+    
+    def new_session(self):
+        return self._new_session()
 
     def _set_verify(self):
         try:

geoseeq/result/file_upload.py

@@ -4,7 +4,7 @@ import json
 import os
 from os.path import basename, getsize, join, dirname, isfile, getctime
 from pathlib import Path
-
+from random import random
 import requests
 
 from geoseeq.knex import GeoseeqGeneralError
@@ -184,14 +184,16 @@ class ResultFileUpload:
                 logger.debug(f"Upload for part {num + 1} succeeded.")
                 break
             except (requests.exceptions.HTTPError, requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e:
+                attempts += 1
                 logger.debug(
-                    f"Upload for part {num + 1} failed. Attempt {attempts + 1} of {max_retries}."
+                    f"Upload for part {num + 1} failed. Attempt {attempts} of {max_retries}. Error: {e}"
                 )
-                attempts += 1
                 if attempts >= max_retries:
                     raise e
 
                 retry_time = min(8 ** attempts, 120)  # exponential backoff, max 120s
+                retry_time *= 0.8 + (random() * 0.4)  # randomize to avoid thundering herd
+                logger.debug(f"Retrying upload for part {num + 1} in {retry_time} seconds.")
                 time.sleep(retry_time)
             
         etag = http_response.headers["ETag"].replace('"', "")

geoseeq/upload_download_manager.py

@@ -22,19 +22,25 @@ def _upload_one_file(args):
     (result_file, filepath, session, progress_tracker,
      link_type, overwrite, log_level, parallel_uploads,
      use_cache, no_new_versions, threads_per_upload,
-     num_retries) = args
+     num_retries, ignore_errors) = args
     if parallel_uploads:
         _make_in_process_logger(log_level)
-    if link_type == 'upload':
-        # TODO: check checksums to see if the file is the same
-        result_file.upload_file(
-            filepath,
-            session=session, overwrite=overwrite, progress_tracker=progress_tracker,
-            threads=threads_per_upload, use_cache=use_cache,
-            no_new_versions=no_new_versions, max_retries=num_retries,
-        )
-    else:
-        result_file.link_file(link_type, filepath)
+    try:
+        if link_type == 'upload':
+            # TODO: check checksums to see if the file is the same
+            result_file.upload_file(
+                filepath,
+                session=session, overwrite=overwrite, progress_tracker=progress_tracker,
+                threads=threads_per_upload, use_cache=use_cache,
+                no_new_versions=no_new_versions, max_retries=num_retries,
+            )
+        else:
+            result_file.link_file(link_type, filepath)
+    except Exception as e:
+        if ignore_errors:
+            logger.error(f"Error uploading {filepath}: {e}")
+        else:
+            raise e
     return result_file
 
 
@@ -50,6 +56,7 @@ class GeoSeeqUploadManager:
                  overwrite=True,
                  no_new_versions=False,
                  num_retries=3,
+                 ignore_errors=False,
                  use_cache=True):
         self.session = session
         self.n_parallel_uploads = n_parallel_uploads
@@ -62,6 +69,7 @@ class GeoSeeqUploadManager:
         self.use_cache = use_cache
         self.threads_per_upload = threads_per_upload
         self.num_retries = num_retries
+        self.ignore_errors = ignore_errors
 
     def add_result_file(self, result_file, local_path):
         self._result_files.append((result_file, local_path))
@@ -87,7 +95,7 @@ class GeoSeeqUploadManager:
                 self.session, self.progress_tracker_factory(local_path),
                 self.link_type, self.overwrite, self.log_level,
                 self.n_parallel_uploads > 1, self.use_cache, self.no_new_versions,
-                self.threads_per_upload, self.num_retries
+                self.threads_per_upload, self.num_retries, self.ignore_errors
             ) for result_file, local_path in self._result_files
         ]
         out = []

{geoseeq-0.5.6a12.dist-info → geoseeq-0.5.6a14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a12
+Version: 0.5.6a14
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a12.dist-info → geoseeq-0.5.6a14.dist-info}/RECORD

@@ -4,14 +4,14 @@ geoseeq/blob_constructors.py,sha256=AkWpDQY0EdGMxF1p6eRspyHKubcUdiW4it-_Q7S2QWk,
 geoseeq/bulk_creators.py,sha256=pdn-Dv7yv5SFv-PfDuQbuOnw2W4-BfIfRJVRAhM8U6s,2115
 geoseeq/constants.py,sha256=h9RURz4xs2bZyDrSGocej7ANJvRLr_H1H7JRxpNUXJM,431
 geoseeq/file_system_cache.py,sha256=HzVZWtwLD2fjWWSo_UfWmGeBltm9He4lP_OqzKwNGWg,4138
-geoseeq/knex.py,sha256=6fPO8F8yxgBgBXZiliMJvYYjgf_16chfJPyWLe-kpPk,7898
+geoseeq/knex.py,sha256=SlK3Z9Y51APecIeJep4eNvFqlwKpQzvtokBnKe0L5Oc,7965
 geoseeq/organization.py,sha256=a9xmGDE0tQsjPJfyFkYnWagxZ8xpdeckkwvkhH6LNIk,2462
 geoseeq/pipeline.py,sha256=89mhWaecsKnm6tyRkdkaVp4dmZh62_v42Ze0oXf8OTY,9873
 geoseeq/project.py,sha256=-9Y2ik0-BpT3iqh89v8VQBbdadhI58oaUP9oZK8oetc,13741
 geoseeq/remote_object.py,sha256=Es-JlAz8iLRmCpAzh1MOwUh2MqtbuQM-p8wHIBAqNlQ,7131
 geoseeq/sample.py,sha256=whgEVk6GnDJJLjn5uTOqFqRtVxZD3BgjTo7brAC5noU,7981
 geoseeq/search.py,sha256=gawad6Cx5FxJBPlYkXWb-UKAO-UC0_yhvyU9Ca1kaNI,3388
-geoseeq/upload_download_manager.py,sha256=-s2Zdru_O2YWThAdmXRksIx2q8RwSOEZ2nLvm1VM4EM,7738
+geoseeq/upload_download_manager.py,sha256=OOTLJQ0dHAPGzO0koF6hr4Be_lxWVc4YADjPi_uLAkY,8051
 geoseeq/user.py,sha256=tol8i1UGLRrbMw5jeJDnna1ikRgrCDd50Jxz0a1lSgg,690
 geoseeq/utils.py,sha256=PDRiEQIZYTcfEV9AYvloQVvfqs5JaebcFZodAa2SUW8,3577
 geoseeq/work_orders.py,sha256=5uLVVfdKE8qh4gGaHkdBpXJGRTujuSg59knWCqEET4A,8071
@@ -22,7 +22,7 @@ geoseeq/cli/detail.py,sha256=q8Suu-j2k18knfSVFG-SWWGNsKM-n8y9RMA3LcIIi9Y,4132
 geoseeq/cli/download.py,sha256=_upzZo08K0fAPbEsyi1uN0HGNUaY1pl6OoGPcWmvSUY,17765
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
 geoseeq/cli/get_eula.py,sha256=79mbUwyiF7O1r0g6UTxG9kJGQEqKuH805E6eLkPC6Y4,997
-geoseeq/cli/main.py,sha256=lqRT8Y62SqLQs9J_P7TCZIZ9CGk6sMphKOALz9TSQ0o,3260
+geoseeq/cli/main.py,sha256=pvnUIUYphiA6YJ4DcZ2Zz3_UZMwV-rPZYHDRU2jrj_I,3260
 geoseeq/cli/manage.py,sha256=wGXAcVaXqE5JQEU8Jh6OlHr02nB396bpS_SFcOZdrEo,5929
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/run.py,sha256=bx2AV6VIqOSTlxUda78xl0XxcZ8TXlQx02-e7iLQPwI,3838
@@ -37,7 +37,7 @@ geoseeq/cli/shared_params/id_handlers.py,sha256=501K9sCVkI0YGDQ62vXk_DM5lMMDrdB5
 geoseeq/cli/shared_params/obj_getters.py,sha256=ZSkt6LnDkVFlNVYKgLrjzg60-6BthZMr3eeD3HNqzac,2741
 geoseeq/cli/shared_params/opts_and_args.py,sha256=LrDkv9WtUryM4uUMXPRk04-EBcTQ7q5V6Yu-XRDUvvA,2083
 geoseeq/cli/upload/__init__.py,sha256=3C9_S9t7chmYU-2ot89NV03x-EtmsjibulErKaU9w1k,627
-geoseeq/cli/upload/upload.py,sha256=whhj-cX324_N_IQKuR3t-Zw4BFzDz7NHzMrmWg5aqiE,9688
+geoseeq/cli/upload/upload.py,sha256=qlIkqpTRwLlyt41XHcUFJlEPLGQHaBYZi-hpJjeD88U,9824
 geoseeq/cli/upload/upload_advanced.py,sha256=Jq5eGe-wOdrzxGWVwaFPg0BAJcW0YSx_eHEmYjJeKuA,3434
 geoseeq/cli/upload/upload_reads.py,sha256=EMGqyZf11xwN4v2j8gNxMagTbE4kaOd-_hwupmg5I-8,10670
 geoseeq/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -64,7 +64,7 @@ geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEY
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
 geoseeq/result/file_download.py,sha256=vbYo2B4JshTIqLaklcgcBb7NY9cD5pMkas95GuQxW8s,5776
-geoseeq/result/file_upload.py,sha256=qQ2F28W6ieLoEsbqCWr61cQ8hYlNaKkhn4DBQ5xqR-I,13214
+geoseeq/result/file_upload.py,sha256=qkJug2ptmfBwmuz577DtPG_Z7eLTtatJzycSjE1UTXk,13430
 geoseeq/result/result_file.py,sha256=1Yj9fkZhds3J-tay6eNH2-EHi00MovHGV1M80_ckHD8,8677
 geoseeq/result/result_folder.py,sha256=6porOXPh7Tpxw3oX5yMRPYQzNCGYqszqmFJd3SwQmTc,11122
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
@@ -80,9 +80,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.5.6a12.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.5.6a12.dist-info/METADATA,sha256=DTEadRCMzytqJtg9vPNagyrMCY8l8gIslgIr7PaOA1o,4806
-geoseeq-0.5.6a12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-geoseeq-0.5.6a12.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.5.6a12.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.5.6a12.dist-info/RECORD,,
+geoseeq-0.5.6a14.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.5.6a14.dist-info/METADATA,sha256=Y8gauUJ3fr2WUwnpXiYEOdFayG523uz2mUY0bTIXdyk,4806
+geoseeq-0.5.6a14.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+geoseeq-0.5.6a14.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.5.6a14.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.5.6a14.dist-info/RECORD,,