geoseeq 0.5.6a9__py3-none-any.whl → 0.5.6a10__py3-none-any.whl

geoseeq/cli/download.py

@@ -97,13 +97,14 @@ def cli_download_metadata(state, sample_ids):
 
 
 cores_option = click.option('--cores', default=1, help='Number of downloads to run in parallel')
-
+head_option = click.option('--head', default=None, type=int, help='Download the first N bytes of each file')
 
 @cli_download.command("files")
 @use_common_state
 @cores_option
 @click.option("--target-dir", default=".")
 @yes_option
+@head_option
 @click.option("--download/--urls-only", default=True, help="Download files or just print urls")
 @click.option("--folder-type", type=click.Choice(['all', 'sample', 'project'], case_sensitive=False), default="all", help='Download files from sample folders, project folders, or both')
 @click.option("--folder-name", multiple=True, help='Filter folders for names that include this string. Case insensitive.')
@@ -120,6 +121,7 @@ def cli_download_files(
     sample_name_includes,
     target_dir,
     yes,
+    head,
     folder_type,
     folder_name,
     file_name,
@@ -213,6 +215,7 @@ def cli_download_files(
             ignore_errors=ignore_errors,
             log_level=state.log_level,
             progress_tracker_factory=PBarManager().get_new_bar,
+            head=head,
         )
         for fname, url in response["links"].items():
             download_manager.add_download(url, join(target_dir, fname))
@@ -230,11 +233,12 @@ def cli_download_files(
 @cores_option
 @click.option("-t", "--target-dir", default=".")
 @yes_option
+@head_option
 @click.option("--download/--urls-only", default=True, help="Download files or just print urls")
 @ignore_errors_option
 @click.option('--hidden/--no-hidden', default=True, help='Download hidden files in folder')
 @folder_ids_arg
-def cli_download_folders(state, cores, target_dir, yes, download, ignore_errors, hidden, folder_ids):
+def cli_download_folders(state, cores, target_dir, yes, head, download, ignore_errors, hidden, folder_ids):
     """Download entire folders from GeoSeeq.
     
     This command downloads folders directly based on their ID. This is used for "manual"
@@ -267,6 +271,7 @@ def cli_download_folders(state, cores, target_dir, yes, download, ignore_errors,
         ignore_errors=ignore_errors,
         log_level=state.log_level,
         progress_tracker_factory=PBarManager().get_new_bar,
+        head=head,
     )
     for result_folder in result_folders:
         download_manager.add_result_folder_download(
@@ -286,7 +291,7 @@ def cli_download_folders(state, cores, target_dir, yes, download, ignore_errors,
 @click.option("-n", "--file-name", multiple=True, help="File name to use for downloaded files. If set you must specify once per ID.")
 @yes_option
 @click.option("--download/--urls-only", default=True, help="Download files or just print urls")
-@click.option('--head', default=None, type=int, help='Download the first N bytes of each file')
+@head_option
 @ignore_errors_option
 @click.argument("ids", nargs=-1)
 def cli_download_ids(state, cores, target_dir, file_name, yes, download, head, ignore_errors, ids):

geoseeq/cli/main.py

@@ -53,7 +53,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.5.6a9')  # remember to update setup
+    click.echo('0.5.6a10')  # remember to update setup
 
 
 @main.group('advanced')

geoseeq/result/file_upload.py

@@ -2,7 +2,7 @@
 import time
 import json
 import os
-from os.path import basename, getsize, join, dirname, isfile
+from os.path import basename, getsize, join, dirname, isfile, getctime
 from pathlib import Path
 
 import requests
@@ -49,7 +49,7 @@ class ResumableUploadTracker:
         self.filepath = filepath
         self.tracker_file = join(
             GEOSEEQ_CACHE_DIR, 'upload',
-            tracker_file_prefix + f".{chunk_size}." + basename(filepath)
+            tracker_file_prefix + f".{chunk_size}.{getsize(filepath)}." + basename(filepath)
         )
         try:
             os.makedirs(dirname(self.tracker_file), exist_ok=True)
@@ -64,7 +64,7 @@ class ResumableUploadTracker:
             return
         if self.upload_started:
             raise GeoseeqGeneralError("Upload has already started.")
-        blob = dict(upload_id=upload_id, urls=urls)
+        blob = dict(upload_id=upload_id, urls=urls, start_time=time.time())
         serialized = json.dumps(blob)
         with open(self.tracker_file, "w") as f:
             f.write(serialized + "\n")
@@ -89,6 +89,11 @@ class ResumableUploadTracker:
         with open(self.tracker_file, "r") as f:
             header_blob = json.loads(f.readline())
             self.upload_id, self.urls = header_blob["upload_id"], header_blob["urls"]
+            start_time = header_blob["start_time"]
+            if (time.time() - start_time) > (60 * 60 * 23):
+                logger.warning(f"Tracker file {self.tracker_file} is too old. Deleting.")
+                os.remove(self.tracker_file)
+                return
             self.upload_started = True
             for line in f:
                 blob = json.loads(line)
@@ -161,14 +166,15 @@ class ResultFileUpload:
                 http_response.raise_for_status()
                 logger.debug(f"Upload for part {num + 1} succeeded.")
                 break
-            except requests.exceptions.HTTPError:
-                logger.warn(
+            except (requests.exceptions.HTTPError, requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e:
+                logger.debug(
                     f"Upload for part {num + 1} failed. Attempt {attempts + 1} of {max_retries}."
                 )
                 attempts += 1
                 if attempts == max_retries:
-                    raise
+                    raise e
                 time.sleep(10**attempts)  # exponential backoff, (10 ** 2)s default max
+            
         etag = http_response.headers["ETag"].replace('"', "")
         blob = {"ETag": etag, "PartNumber": num + 1}
         if resumable_upload_tracker:

{geoseeq-0.5.6a9.dist-info → geoseeq-0.5.6a10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a9
+Version: 0.5.6a10
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a9.dist-info → geoseeq-0.5.6a10.dist-info}/RECORD

@@ -19,10 +19,10 @@ geoseeq/cli/__init__.py,sha256=4WnK87K5seRK3SGJAxNWnQTqyg5uBhdhrOrzB1D4b3M,24
 geoseeq/cli/constants.py,sha256=Do5AUf9lMO9_P8KpFJ3XwwFBAWsxSjZ6sx9_QEGyC_c,176
 geoseeq/cli/copy.py,sha256=02U9kdrAIbbM8MlRMLL6p-LMYFSuRObE3h5jyvcL__M,2275
 geoseeq/cli/detail.py,sha256=q8Suu-j2k18knfSVFG-SWWGNsKM-n8y9RMA3LcIIi9Y,4132
-geoseeq/cli/download.py,sha256=ldpqpnRe00utb1EL1T_5CyPbFrZbtauIvOSOHtxz9qc,17656
+geoseeq/cli/download.py,sha256=_upzZo08K0fAPbEsyi1uN0HGNUaY1pl6OoGPcWmvSUY,17765
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
 geoseeq/cli/get_eula.py,sha256=79mbUwyiF7O1r0g6UTxG9kJGQEqKuH805E6eLkPC6Y4,997
-geoseeq/cli/main.py,sha256=7-OOKwxZ6npzer0KlnGDjjOzFLHlzQYZIgCpxICrkFg,3259
+geoseeq/cli/main.py,sha256=xbSTOVLf1iFdw-aUu6l_UEVeiVrPgi9Liylt5UjNRzU,3260
 geoseeq/cli/manage.py,sha256=wGXAcVaXqE5JQEU8Jh6OlHr02nB396bpS_SFcOZdrEo,5929
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/run.py,sha256=bx2AV6VIqOSTlxUda78xl0XxcZ8TXlQx02-e7iLQPwI,3838
@@ -64,7 +64,7 @@ geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEY
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
 geoseeq/result/file_download.py,sha256=XQA5bdQJJSZIgbFcN09OvVdLq12fnA98kPCIONAkLk0,5568
-geoseeq/result/file_upload.py,sha256=60flgzJ8KZ-EhDfKJEiBf305JmELaJdwoYRXzFIyQWk,11696
+geoseeq/result/file_upload.py,sha256=D8gbdsyzw5ztjT7Vmq4InxKhyl7l8n2leH4GYVD-seM,12109
 geoseeq/result/result_file.py,sha256=1Yj9fkZhds3J-tay6eNH2-EHi00MovHGV1M80_ckHD8,8677
 geoseeq/result/result_folder.py,sha256=6porOXPh7Tpxw3oX5yMRPYQzNCGYqszqmFJd3SwQmTc,11122
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
@@ -80,9 +80,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.5.6a9.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.5.6a9.dist-info/METADATA,sha256=kej6OxkIxWfdo77ei6qzYJ1kO8R9lV7llK-Kd8FkXwA,4805
-geoseeq-0.5.6a9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-geoseeq-0.5.6a9.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.5.6a9.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.5.6a9.dist-info/RECORD,,
+geoseeq-0.5.6a10.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.5.6a10.dist-info/METADATA,sha256=RKguQ4reNa4rH6Pv20JBMCbyo2inLjKcO5VFkcCgyxg,4806
+geoseeq-0.5.6a10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+geoseeq-0.5.6a10.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.5.6a10.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.5.6a10.dist-info/RECORD,,