geoseeq 0.5.6a15__py3-none-any.whl → 0.5.6a16__py3-none-any.whl

geoseeq/cli/main.py

@@ -53,7 +53,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.5.6a15')  # remember to update setup
+    click.echo('0.5.6a16')  # remember to update setup
 
 
 @main.group('advanced')
@@ -101,4 +101,20 @@ def cli_config(yes, api_token, endpoint, profile, overwrite):
             click.echo('You must accept the EULA to use the GeoSeeq API.')
             return
     set_profile(api_token, endpoint=endpoint, profile=profile, overwrite=overwrite)
-    click.echo(f'Profile configured.')
+    click.echo(f'Profile configured.')
+
+
+@main.command('clear-cache')
+@yes_option
+def cli_clear_cache(yes):
+    """Clear the local cache.
+
+    ---
+    
+    Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
+    Run `geoseeq eula show` to view the EULA.
+    """
+    from geoseeq.file_system_cache import GEOSEEQ_CACHE_DIR
+    import shutil
+    if yes or click.confirm('Are you sure you want to clear the cache?'):
+        shutil.rmtree(GEOSEEQ_CACHE_DIR, ignore_errors=True)

geoseeq/cli/upload/upload.py

@@ -40,7 +40,7 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
 @click.option('--cores', default=1, help='Number of uploads to run in parallel', show_default=True)
 @click.option('--threads-per-upload', default=4, help='Number of threads used to upload each file', show_default=True)
 @click.option('--num-retries', default=3, help='Number of times to retry a failed upload', show_default=True)
-@click.option('--chunk-size-mb', default=5, help='Size of chunks to upload in MB', show_default=True)
+@click.option('--chunk-size-mb', default=-1, help='Size of chunks to upload in MB', show_default=True)
 @ignore_errors_option
 @yes_option
 @private_option
@@ -122,8 +122,8 @@ def cli_upload_file(state, cores, threads_per_upload, num_retries, chunk_size_mb
         use_cache=state.use_cache,
         num_retries=num_retries,
         ignore_errors=ignore_errors,
-        session=knex.new_session(),
-        chunk_size_mb=chunk_size_mb,
+        session=None, #knex.new_session(),
+        chunk_size_mb=chunk_size_mb if chunk_size_mb > 0 else None,
     )
     for geoseeq_file_name, file_path in name_pairs:
         if isfile(file_path):

geoseeq/result/file_upload.py

@@ -175,6 +175,7 @@ class ResultFileUpload:
         attempts = 0
         while attempts < max_retries:
             try:
+                # url = url.replace("s3.wasabisys.com", "s3.us-east-1.wasabisys.com")
                 logger.debug(f"Uploading part {num + 1} to {url}. Size: {len(file_chunk)} bytes.")
                 if session:
                     http_response = session.put(url, data=file_chunk)
@@ -192,7 +193,7 @@ class ResultFileUpload:
                     raise e
 
                 retry_time = min(8 ** attempts, 120)  # exponential backoff, max 120s
-                retry_time *= 0.8 + (random() * 0.4)  # randomize to avoid thundering herd
+                retry_time *= 0.6 + (random() * 0.8)  # randomize to avoid thundering herd
                 logger.debug(f"Retrying upload for part {num + 1} in {retry_time} seconds.")
                 time.sleep(retry_time)
             
@@ -250,7 +251,7 @@ class ResultFileUpload:
         filepath,
         file_size,
         optional_fields=None,
-        chunk_size=FIVE_MB,
+        chunk_size=None,
         max_retries=3,
         session=None,
         progress_tracker=None,
@@ -259,6 +260,11 @@ class ResultFileUpload:
     ):
         """Upload a file to S3 using the multipart upload process."""
         logger.info(f"Uploading {filepath} to S3 using multipart upload.")
+        if not chunk_size:
+            chunk_size = FIVE_MB
+            if file_size >= 10 * FIVE_MB:
+                chunk_size = 5 * FIVE_MB
+        logger.debug(f"Using chunk size of {chunk_size} bytes.")
         resumable_upload_tracker = None
         if use_cache and file_size > 10 * FIVE_MB:  # only use resumable upload tracker for larger files
             resumable_upload_tracker = ResumableUploadTracker(filepath, chunk_size)

geoseeq/upload_download_manager.py

@@ -23,7 +23,7 @@ def _upload_one_file(args):
      link_type, overwrite, log_level, parallel_uploads,
      use_cache, no_new_versions, threads_per_upload,
      num_retries, ignore_errors, chunk_size_mb) = args
-    chunk_size = chunk_size_mb * 1024 * 1024
+    chunk_size = chunk_size_mb * 1024 * 1024 if chunk_size_mb else None
     if parallel_uploads:
         _make_in_process_logger(log_level)
     try:

{geoseeq-0.5.6a15.dist-info → geoseeq-0.5.6a16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a15
+Version: 0.5.6a16
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a15.dist-info → geoseeq-0.5.6a16.dist-info}/RECORD

@@ -11,7 +11,7 @@ geoseeq/project.py,sha256=-9Y2ik0-BpT3iqh89v8VQBbdadhI58oaUP9oZK8oetc,13741
 geoseeq/remote_object.py,sha256=Es-JlAz8iLRmCpAzh1MOwUh2MqtbuQM-p8wHIBAqNlQ,7131
 geoseeq/sample.py,sha256=whgEVk6GnDJJLjn5uTOqFqRtVxZD3BgjTo7brAC5noU,7981
 geoseeq/search.py,sha256=gawad6Cx5FxJBPlYkXWb-UKAO-UC0_yhvyU9Ca1kaNI,3388
-geoseeq/upload_download_manager.py,sha256=2WM1yvEseMxjrLsE_tNMvt0cCldMScAOuHxj2l0ICrc,8248
+geoseeq/upload_download_manager.py,sha256=_I_wJHKoeumMhX6cq_WDna9Il6MYatSfSSZ9DUwTW-E,8275
 geoseeq/user.py,sha256=tol8i1UGLRrbMw5jeJDnna1ikRgrCDd50Jxz0a1lSgg,690
 geoseeq/utils.py,sha256=PDRiEQIZYTcfEV9AYvloQVvfqs5JaebcFZodAa2SUW8,3577
 geoseeq/work_orders.py,sha256=5uLVVfdKE8qh4gGaHkdBpXJGRTujuSg59knWCqEET4A,8071
@@ -22,7 +22,7 @@ geoseeq/cli/detail.py,sha256=q8Suu-j2k18knfSVFG-SWWGNsKM-n8y9RMA3LcIIi9Y,4132
 geoseeq/cli/download.py,sha256=_upzZo08K0fAPbEsyi1uN0HGNUaY1pl6OoGPcWmvSUY,17765
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
 geoseeq/cli/get_eula.py,sha256=79mbUwyiF7O1r0g6UTxG9kJGQEqKuH805E6eLkPC6Y4,997
-geoseeq/cli/main.py,sha256=ExPXiBu0SIphQx1pjpmTVGyPPrkEVM7pHKwyi_U5Eao,3260
+geoseeq/cli/main.py,sha256=KL4TcaypRtrSoP0Om62T5cVZ7uOzjR6hcr2WqUm4qSU,3723
 geoseeq/cli/manage.py,sha256=wGXAcVaXqE5JQEU8Jh6OlHr02nB396bpS_SFcOZdrEo,5929
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/run.py,sha256=bx2AV6VIqOSTlxUda78xl0XxcZ8TXlQx02-e7iLQPwI,3838
@@ -37,7 +37,7 @@ geoseeq/cli/shared_params/id_handlers.py,sha256=501K9sCVkI0YGDQ62vXk_DM5lMMDrdB5
 geoseeq/cli/shared_params/obj_getters.py,sha256=ZSkt6LnDkVFlNVYKgLrjzg60-6BthZMr3eeD3HNqzac,2741
 geoseeq/cli/shared_params/opts_and_args.py,sha256=LrDkv9WtUryM4uUMXPRk04-EBcTQ7q5V6Yu-XRDUvvA,2083
 geoseeq/cli/upload/__init__.py,sha256=3C9_S9t7chmYU-2ot89NV03x-EtmsjibulErKaU9w1k,627
-geoseeq/cli/upload/upload.py,sha256=iZT_4M2I-muUo8cD5exLE49DrKjYuJA-xVKV69N0PB8,9978
+geoseeq/cli/upload/upload.py,sha256=e0tI68OxVLgS6sauUS-m-AJgza13n0XgMhxd86fq0Gw,10017
 geoseeq/cli/upload/upload_advanced.py,sha256=Jq5eGe-wOdrzxGWVwaFPg0BAJcW0YSx_eHEmYjJeKuA,3434
 geoseeq/cli/upload/upload_reads.py,sha256=EMGqyZf11xwN4v2j8gNxMagTbE4kaOd-_hwupmg5I-8,10670
 geoseeq/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -64,7 +64,7 @@ geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEY
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
 geoseeq/result/file_download.py,sha256=vbYo2B4JshTIqLaklcgcBb7NY9cD5pMkas95GuQxW8s,5776
-geoseeq/result/file_upload.py,sha256=qkJug2ptmfBwmuz577DtPG_Z7eLTtatJzycSjE1UTXk,13430
+geoseeq/result/file_upload.py,sha256=3RhFEdCFP64A3PY1AwkWz5Qp9at78GlNhzP-OWtHsq8,13721
 geoseeq/result/result_file.py,sha256=1Yj9fkZhds3J-tay6eNH2-EHi00MovHGV1M80_ckHD8,8677
 geoseeq/result/result_folder.py,sha256=6porOXPh7Tpxw3oX5yMRPYQzNCGYqszqmFJd3SwQmTc,11122
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
@@ -80,9 +80,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.5.6a15.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.5.6a15.dist-info/METADATA,sha256=REa9myn8L8WcbxLg0BXH1ucW-A6OtEclEHEb2Usr-0g,4806
-geoseeq-0.5.6a15.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-geoseeq-0.5.6a15.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.5.6a15.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.5.6a15.dist-info/RECORD,,
+geoseeq-0.5.6a16.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.5.6a16.dist-info/METADATA,sha256=04xNdeQsTinp3ka0HCZVdy4-4uiCpAx2HA-vtlzBSSA,4806
+geoseeq-0.5.6a16.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+geoseeq-0.5.6a16.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.5.6a16.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.5.6a16.dist-info/RECORD,,