geoseeq 0.5.6a11__tar.gz → 0.5.6a13__tar.gz

{geoseeq-0.5.6a11 → geoseeq-0.5.6a13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a11
+Version: 0.5.6a13
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a11 → geoseeq-0.5.6a13}/geoseeq/cli/main.py

@@ -53,7 +53,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.5.6a11')  # remember to update setup
+    click.echo('0.5.6a13')  # remember to update setup
 
 
 @main.group('advanced')

{geoseeq-0.5.6a11 → geoseeq-0.5.6a13}/geoseeq/cli/upload/upload.py

@@ -25,6 +25,7 @@ from geoseeq.cli.shared_params import (
     project_or_sample_id_arg,
     handle_project_or_sample_id,
     no_new_versions_option,
+    ignore_errors_option,
 )
 from geoseeq.upload_download_manager import GeoSeeqUploadManager
 
@@ -36,8 +37,10 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
 
 @click.command('files')
 @use_common_state
-@click.option('--cores', default=1, help='Number of uploads to run in parallel')
-@click.option('--threads-per-upload', default=4, help='Number of threads used to upload each file')
+@click.option('--cores', default=1, help='Number of uploads to run in parallel', show_default=True)
+@click.option('--threads-per-upload', default=4, help='Number of threads used to upload each file', show_default=True)
+@click.option('--num-retries', default=3, help='Number of times to retry a failed upload', show_default=True)
+@ignore_errors_option
 @yes_option
 @private_option
 @link_option
@@ -45,10 +48,11 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
 @hidden_option
 @no_new_versions_option
 @click.option('-n', '--geoseeq-file-name', default=None, multiple=True,
-              help='Specify a different name for the file on GeoSeeq than the local file name.')
+              help='Specify a different name for the file on GeoSeeq than the local file name.',
+              show_default=True)
 @folder_id_arg
 @click.argument('file_paths', type=click.Path(exists=True), nargs=-1)
-def cli_upload_file(state, cores, threads_per_upload, yes, private, link_type, recursive, hidden, no_new_versions, geoseeq_file_name, folder_id, file_paths):
+def cli_upload_file(state, cores, threads_per_upload, num_retries, ignore_errors, yes, private, link_type, recursive, hidden, no_new_versions, geoseeq_file_name, folder_id, file_paths):
     """Upload files to GeoSeeq.
 
     This command uploads files to either a sample or project on GeoSeeq. It can be used to upload
@@ -93,6 +97,8 @@ def cli_upload_file(state, cores, threads_per_upload, yes, private, link_type, r
 
     ---
     """
+    if num_retries < 1:
+        raise click.UsageError('--num-retries must be at least 1')
     knex = state.get_knex()
     result_folder = handle_folder_id(knex, folder_id, yes=yes, private=private, create=True)
     if geoseeq_file_name:
@@ -113,6 +119,8 @@ def cli_upload_file(state, cores, threads_per_upload, yes, private, link_type, r
         log_level=state.log_level,
         no_new_versions=no_new_versions,
         use_cache=state.use_cache,
+        num_retries=num_retries,
+        ignore_errors=ignore_errors,
     )
     for geoseeq_file_name, file_path in name_pairs:
         if isfile(file_path):

{geoseeq-0.5.6a11 → geoseeq-0.5.6a13}/geoseeq/result/file_upload.py

@@ -4,7 +4,7 @@ import json
 import os
 from os.path import basename, getsize, join, dirname, isfile, getctime
 from pathlib import Path
-
+from random import random
 import requests
 
 from geoseeq.knex import GeoseeqGeneralError
@@ -188,9 +188,12 @@ class ResultFileUpload:
                     f"Upload for part {num + 1} failed. Attempt {attempts + 1} of {max_retries}."
                 )
                 attempts += 1
-                if attempts == max_retries:
+                if attempts >= max_retries:
                     raise e
-                time.sleep(10**attempts)  # exponential backoff, (10 ** 2)s default max
+
+                retry_time = min(8 ** attempts, 120)  # exponential backoff, max 120s
+                retry_time *= 0.8 + (random() * 0.4)  # randomize to avoid thundering herd
+                time.sleep(retry_time)
             
         etag = http_response.headers["ETag"].replace('"', "")
         blob = {"ETag": etag, "PartNumber": num + 1}

{geoseeq-0.5.6a11 → geoseeq-0.5.6a13}/geoseeq/upload_download_manager.py

@@ -21,19 +21,26 @@ def _make_in_process_logger(log_level):
 def _upload_one_file(args):
     (result_file, filepath, session, progress_tracker,
      link_type, overwrite, log_level, parallel_uploads,
-     use_cache, no_new_versions, threads_per_upload) = args
+     use_cache, no_new_versions, threads_per_upload,
+     num_retries, ignore_errors) = args
     if parallel_uploads:
         _make_in_process_logger(log_level)
-    if link_type == 'upload':
-        # TODO: check checksums to see if the file is the same
-        result_file.upload_file(
-            filepath,
-            session=session, overwrite=overwrite, progress_tracker=progress_tracker,
-            threads=threads_per_upload, use_cache=use_cache,
-            no_new_versions=no_new_versions
-        )
-    else:
-        result_file.link_file(link_type, filepath)
+    try:
+        if link_type == 'upload':
+            # TODO: check checksums to see if the file is the same
+            result_file.upload_file(
+                filepath,
+                session=session, overwrite=overwrite, progress_tracker=progress_tracker,
+                threads=threads_per_upload, use_cache=use_cache,
+                no_new_versions=no_new_versions, max_retries=num_retries,
+            )
+        else:
+            result_file.link_file(link_type, filepath)
+    except Exception as e:
+        if ignore_errors:
+            logger.error(f"Error uploading {filepath}: {e}")
+        else:
+            raise e
     return result_file
 
 
@@ -48,6 +55,8 @@ class GeoSeeqUploadManager:
                  log_level=logging.WARNING,
                  overwrite=True,
                  no_new_versions=False,
+                 num_retries=3,
+                 ignore_errors=False,
                  use_cache=True):
         self.session = session
         self.n_parallel_uploads = n_parallel_uploads
@@ -59,6 +68,8 @@ class GeoSeeqUploadManager:
         self.no_new_versions = no_new_versions
         self.use_cache = use_cache
         self.threads_per_upload = threads_per_upload
+        self.num_retries = num_retries
+        self.ignore_errors = ignore_errors
 
     def add_result_file(self, result_file, local_path):
         self._result_files.append((result_file, local_path))
@@ -84,7 +95,7 @@ class GeoSeeqUploadManager:
                 self.session, self.progress_tracker_factory(local_path),
                 self.link_type, self.overwrite, self.log_level,
                 self.n_parallel_uploads > 1, self.use_cache, self.no_new_versions,
-                self.threads_per_upload
+                self.threads_per_upload, self.num_retries, self.ignore_errors
             ) for result_file, local_path in self._result_files
         ]
         out = []

{geoseeq-0.5.6a11 → geoseeq-0.5.6a13}/geoseeq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a11
+Version: 0.5.6a13
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a11 → geoseeq-0.5.6a13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "geoseeq"
-version = "0.5.6a11"
+version = "0.5.6a13"
 authors = [
   { name="David C. Danko", email="dcdanko@biotia.io" },
 ]