geoseeq 0.5.6a10__py3-none-any.whl → 0.5.6a12__py3-none-any.whl

geoseeq/cli/main.py

@@ -53,7 +53,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.5.6a10')  # remember to update setup
+    click.echo('0.5.6a12')  # remember to update setup
 
 
 @main.group('advanced')

geoseeq/cli/upload/upload.py

@@ -36,7 +36,9 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
 
 @click.command('files')
 @use_common_state
-@click.option('--cores', default=1, help='Number of uploads to run in parallel')
+@click.option('--cores', default=1, help='Number of uploads to run in parallel', show_default=True)
+@click.option('--threads-per-upload', default=4, help='Number of threads used to upload each file', show_default=True)
+@click.option('--num-retries', default=3, help='Number of times to retry a failed upload', show_default=True)
 @yes_option
 @private_option
 @link_option
@@ -44,10 +46,11 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
 @hidden_option
 @no_new_versions_option
 @click.option('-n', '--geoseeq-file-name', default=None, multiple=True,
-              help='Specify a different name for the file on GeoSeeq than the local file name.')
+              help='Specify a different name for the file on GeoSeeq than the local file name.',
+              show_default=True)
 @folder_id_arg
 @click.argument('file_paths', type=click.Path(exists=True), nargs=-1)
-def cli_upload_file(state, cores, yes, private, link_type, recursive, hidden, no_new_versions, geoseeq_file_name, folder_id, file_paths):
+def cli_upload_file(state, cores, threads_per_upload, num_retries, yes, private, link_type, recursive, hidden, no_new_versions, geoseeq_file_name, folder_id, file_paths):
     """Upload files to GeoSeeq.
 
     This command uploads files to either a sample or project on GeoSeeq. It can be used to upload
@@ -92,6 +95,8 @@ def cli_upload_file(state, cores, yes, private, link_type, recursive, hidden, no
 
     ---
     """
+    if num_retries < 1:
+        raise click.UsageError('--num-retries must be at least 1')
     knex = state.get_knex()
     result_folder = handle_folder_id(knex, folder_id, yes=yes, private=private, create=True)
     if geoseeq_file_name:
@@ -106,11 +111,13 @@ def cli_upload_file(state, cores, yes, private, link_type, recursive, hidden, no
     
     upload_manager = GeoSeeqUploadManager(
         n_parallel_uploads=cores,
+        threads_per_upload=threads_per_upload,
         link_type=link_type,
         progress_tracker_factory=PBarManager().get_new_bar,
         log_level=state.log_level,
         no_new_versions=no_new_versions,
         use_cache=state.use_cache,
+        num_retries=num_retries,
     )
     for geoseeq_file_name, file_path in name_pairs:
         if isfile(file_path):

geoseeq/cli/upload/upload_reads.py

@@ -132,6 +132,28 @@ def flatten_list_of_fastqs(filepaths):
     return flattened
 
 
+def _is_bam(path):
+    for ext in ['.bam', '.bai']:
+        if path.endswith(ext):
+            return True
+    return False
+
+
+def flatten_list_of_bams(filepaths):
+    """Turn a list of bam filepaths and txt files containing bam filepaths into a single list of bam filepaths."""
+    flattened = []
+    for path in filepaths:
+        if _is_bam(path):
+            flattened.append(path)
+        else:
+            with open(path) as f:
+                for line in f:
+                    line = line.strip()
+                    if line and not line.startswith('#'):
+                        flattened.append(line)
+    return flattened
+
+
 
 @click.command('reads')
 @use_common_state
@@ -200,3 +222,67 @@ def cli_upload_reads_wizard(state, cores, overwrite, yes, regex, private, link_t
     regex = _get_regex(knex, filepaths, module_name, proj, regex)
     groups = _group_files(knex, filepaths, module_name, regex, yes)
     _do_upload(groups, module_name, link_type, proj, filepaths, overwrite, no_new_versions, cores, state)
+
+
+# @click.command('bam')
+# @use_common_state
+# @click.option('--genome', default=None, help='The genome aligned to the BAM files. Should be in 2bit format.')
+# @click.option('--cores', default=1, help='Number of uploads to run in parallel')
+# @overwrite_option
+# @yes_option
+# @click.option('--regex', default=None, help='An optional regex to use to extract sample names from the file names')
+# @private_option
+# @link_option
+# @no_new_versions_option
+# @project_id_arg
+# @click.argument('files', type=click.Path(exists=True), nargs=-1)
+# def cli_upload_bams(state, genome, cores, overwrite, yes, regex, private, link_type, no_new_versions, project_id, files):
+    """Upload BAM files to GeoSeeq.
+
+    This command automatically groups bams with their index files.
+
+    ---
+
+    Example Usage:
+
+    \b
+    # Upload a list of BAM files to a project, useful if you have hundreds of files
+    $ ls -1 path/to/bam/files/*.bam > file_list.txt
+    $ geoseeq upload bams "GeoSeeq/Example CLI Project" file_list.txt
+
+    \b
+    # Upload all the BAM files in a directory to a project with BAM indexes
+    $ geoseeq upload bams ed59b913-91ec-489b-a1b9-4ea137a6e5cf path/to/bam/files/*.bam path/to/bam/files/*.bam.bai
+
+    \b
+    # Upload all the BAM files in a directory to a project, performing 4 uploads in parallel
+    $ geoseeq upload bams --cores 4 ed59b913-91ec-489b-a1b9-4ea137a6e5cf path/to/bam/files/*.bam
+
+    \b
+    # Upload a list of BAM files to a project, automatically creating a new project and overwriting existing files
+    $ ls -1 path/to/bam/files/*.bam > file_list.txt
+    $ geoseeq upload bams --yes --overwrite "GeoSeeq/Example CLI Project" file_list.txt
+
+    ---
+
+    Command Arguments:
+    
+    [PROJECT_ID] Can be a project UUID, GeoSeeq Resource Number (GRN), or an
+    organization name and project name separated by a slash.
+
+    \b
+    Examples: 
+     - Name pair: "GeoSeeq/Example CLI Project"
+     - UUID: "ed59b913-91ec-489b-a1b9-4ea137a6e5cf"
+     - GRN: "grn:gs1:project:ed59b913-91ec-489b-a1b9-4ea137a6e5cf"
+
+    \b
+    [FILES...] can be paths to BAM files or a file containing a list of paths, or a mix of both.
+    Example: "path/to/bam/files
+    """
+    knex = state.get_knex()
+    proj = handle_project_id(knex, project_id, yes, private)
+    filepaths = {basename(line): line for line in flatten_list_of_bams(files)}
+    click.echo(f'Found {len(filepaths)} files to upload.', err=True)
+    groups = _group_files(knex, filepaths, 'bam::bam', regex, yes)
+    _do_upload(groups, 'bam::bam', link_type, proj, filepaths, overwrite, no_new_versions, cores, state)

geoseeq/result/file_download.py

@@ -46,6 +46,11 @@ def guess_download_kind(url):
 
 def download_url(url, kind='guess', filename=None, head=None, progress_tracker=None):
     """Return a local filepath to the downloaded file. Download the file."""
+    if filename and isfile(filename):
+        file_size = getsize(filename)
+        if file_size > 0:
+            logger.info(f"File already exists: {filename}. Not overwriting.")
+            return filename
     if kind == 'guess':
         kind = guess_download_kind(url)
         logger.info(f"Guessed download kind: {kind} for {url}")

geoseeq/result/file_upload.py

@@ -31,14 +31,30 @@ class FileChunker:
                     chunk = f.read(self.chunk_size)
                     self.loaded_parts.append(chunk)
         return self  # convenience for chaining
+    
+    def chunk_is_preloaded(self, num):
+        return len(self.loaded_parts) > num and self.loaded_parts[num]
+    
+    def read_one_chunk(self, num):
+        if not self.chunk_is_preloaded(num):
+            logger.debug(f"Reading chunk {num} from {self.filepath}")
+            with open(self.filepath, "rb") as f:
+                f.seek(num * self.chunk_size)
+                chunk = f.read(self.chunk_size)
+                return chunk
+        return self.loaded_parts[num]
 
     def get_chunk(self, num):
-        self.load_all_chunks()
-        return self.loaded_parts[num]
+        if self.chunk_is_preloaded(num):
+            return self.loaded_parts[num]
+        return self.read_one_chunk(num)
     
     def get_chunk_size(self, num):
-        self.load_all_chunks()
-        return len(self.loaded_parts[num])
+        if num < (self.n_parts - 1):  # all but the last chunk
+            return self.chunk_size
+        if self.chunk_is_preloaded(num):  # last chunk, pre-loaded
+            return len(self.loaded_parts[num])
+        return len(self.read_one_chunk(num))  # last chunk, not pre-loaded
     
 
 class ResumableUploadTracker:
@@ -159,6 +175,7 @@ class ResultFileUpload:
         attempts = 0
         while attempts < max_retries:
             try:
+                logger.debug(f"Uploading part {num + 1} to {url}. Size: {len(file_chunk)} bytes.")
                 if session:
                     http_response = session.put(url, data=file_chunk)
                 else:
@@ -171,9 +188,11 @@ class ResultFileUpload:
                     f"Upload for part {num + 1} failed. Attempt {attempts + 1} of {max_retries}."
                 )
                 attempts += 1
-                if attempts == max_retries:
+                if attempts >= max_retries:
                     raise e
-                time.sleep(10**attempts)  # exponential backoff, (10 ** 2)s default max
+
+                retry_time = min(8 ** attempts, 120)  # exponential backoff, max 120s
+                time.sleep(retry_time)
             
         etag = http_response.headers["ETag"].replace('"', "")
         blob = {"ETag": etag, "PartNumber": num + 1}
@@ -251,7 +270,12 @@ class ResultFileUpload:
                 resumable_upload_tracker.start_upload(upload_id, urls)
         logger.info(f'Starting upload for "{filepath}"')
         complete_parts = []
-        file_chunker = FileChunker(filepath, chunk_size).load_all_chunks()
+        file_chunker = FileChunker(filepath, chunk_size)
+        if file_chunker.file_size < 10 * FIVE_MB:
+            file_chunker.load_all_chunks()
+            logger.debug(f"Preloaded all chunks for {filepath}")
+        else:
+            logger.debug(f"Did not preload chunks for {filepath}")
         if progress_tracker: progress_tracker.set_num_chunks(file_chunker.file_size)
         complete_parts = self._upload_parts(
             file_chunker,

geoseeq/upload_download_manager.py

@@ -21,7 +21,8 @@ def _make_in_process_logger(log_level):
 def _upload_one_file(args):
     (result_file, filepath, session, progress_tracker,
      link_type, overwrite, log_level, parallel_uploads,
-     use_cache, no_new_versions) = args
+     use_cache, no_new_versions, threads_per_upload,
+     num_retries) = args
     if parallel_uploads:
         _make_in_process_logger(log_level)
     if link_type == 'upload':
@@ -29,7 +30,8 @@ def _upload_one_file(args):
         result_file.upload_file(
             filepath,
             session=session, overwrite=overwrite, progress_tracker=progress_tracker,
-            threads=4, use_cache=use_cache, no_new_versions=no_new_versions
+            threads=threads_per_upload, use_cache=use_cache,
+            no_new_versions=no_new_versions, max_retries=num_retries,
         )
     else:
         result_file.link_file(link_type, filepath)
@@ -40,12 +42,14 @@ class GeoSeeqUploadManager:
 
     def __init__(self,
                  n_parallel_uploads=1,
+                 threads_per_upload=4,
                  session=None,
                  link_type='upload',
                  progress_tracker_factory=None,
                  log_level=logging.WARNING,
                  overwrite=True,
                  no_new_versions=False,
+                 num_retries=3,
                  use_cache=True):
         self.session = session
         self.n_parallel_uploads = n_parallel_uploads
@@ -56,6 +60,8 @@ class GeoSeeqUploadManager:
         self._result_files = []
         self.no_new_versions = no_new_versions
         self.use_cache = use_cache
+        self.threads_per_upload = threads_per_upload
+        self.num_retries = num_retries
 
     def add_result_file(self, result_file, local_path):
         self._result_files.append((result_file, local_path))
@@ -80,7 +86,8 @@ class GeoSeeqUploadManager:
                 result_file, local_path,
                 self.session, self.progress_tracker_factory(local_path),
                 self.link_type, self.overwrite, self.log_level,
-                self.n_parallel_uploads > 1, self.use_cache, self.no_new_versions
+                self.n_parallel_uploads > 1, self.use_cache, self.no_new_versions,
+                self.threads_per_upload, self.num_retries
             ) for result_file, local_path in self._result_files
         ]
         out = []

{geoseeq-0.5.6a10.dist-info → geoseeq-0.5.6a12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a10
+Version: 0.5.6a12
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a10.dist-info → geoseeq-0.5.6a12.dist-info}/RECORD

@@ -11,7 +11,7 @@ geoseeq/project.py,sha256=-9Y2ik0-BpT3iqh89v8VQBbdadhI58oaUP9oZK8oetc,13741
 geoseeq/remote_object.py,sha256=Es-JlAz8iLRmCpAzh1MOwUh2MqtbuQM-p8wHIBAqNlQ,7131
 geoseeq/sample.py,sha256=whgEVk6GnDJJLjn5uTOqFqRtVxZD3BgjTo7brAC5noU,7981
 geoseeq/search.py,sha256=gawad6Cx5FxJBPlYkXWb-UKAO-UC0_yhvyU9Ca1kaNI,3388
-geoseeq/upload_download_manager.py,sha256=BGaEBAKu05CqftwRu3BjzL6FvcHp_w122yOS3LVVzd4,7423
+geoseeq/upload_download_manager.py,sha256=-s2Zdru_O2YWThAdmXRksIx2q8RwSOEZ2nLvm1VM4EM,7738
 geoseeq/user.py,sha256=tol8i1UGLRrbMw5jeJDnna1ikRgrCDd50Jxz0a1lSgg,690
 geoseeq/utils.py,sha256=PDRiEQIZYTcfEV9AYvloQVvfqs5JaebcFZodAa2SUW8,3577
 geoseeq/work_orders.py,sha256=5uLVVfdKE8qh4gGaHkdBpXJGRTujuSg59knWCqEET4A,8071
@@ -22,7 +22,7 @@ geoseeq/cli/detail.py,sha256=q8Suu-j2k18knfSVFG-SWWGNsKM-n8y9RMA3LcIIi9Y,4132
 geoseeq/cli/download.py,sha256=_upzZo08K0fAPbEsyi1uN0HGNUaY1pl6OoGPcWmvSUY,17765
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
 geoseeq/cli/get_eula.py,sha256=79mbUwyiF7O1r0g6UTxG9kJGQEqKuH805E6eLkPC6Y4,997
-geoseeq/cli/main.py,sha256=xbSTOVLf1iFdw-aUu6l_UEVeiVrPgi9Liylt5UjNRzU,3260
+geoseeq/cli/main.py,sha256=lqRT8Y62SqLQs9J_P7TCZIZ9CGk6sMphKOALz9TSQ0o,3260
 geoseeq/cli/manage.py,sha256=wGXAcVaXqE5JQEU8Jh6OlHr02nB396bpS_SFcOZdrEo,5929
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/run.py,sha256=bx2AV6VIqOSTlxUda78xl0XxcZ8TXlQx02-e7iLQPwI,3838
@@ -37,9 +37,9 @@ geoseeq/cli/shared_params/id_handlers.py,sha256=501K9sCVkI0YGDQ62vXk_DM5lMMDrdB5
 geoseeq/cli/shared_params/obj_getters.py,sha256=ZSkt6LnDkVFlNVYKgLrjzg60-6BthZMr3eeD3HNqzac,2741
 geoseeq/cli/shared_params/opts_and_args.py,sha256=LrDkv9WtUryM4uUMXPRk04-EBcTQ7q5V6Yu-XRDUvvA,2083
 geoseeq/cli/upload/__init__.py,sha256=3C9_S9t7chmYU-2ot89NV03x-EtmsjibulErKaU9w1k,627
-geoseeq/cli/upload/upload.py,sha256=vbUslOSPf8EDYFzFxukE4EFK_h8nfMYNbSczR2AJNxk,9203
+geoseeq/cli/upload/upload.py,sha256=whhj-cX324_N_IQKuR3t-Zw4BFzDz7NHzMrmWg5aqiE,9688
 geoseeq/cli/upload/upload_advanced.py,sha256=Jq5eGe-wOdrzxGWVwaFPg0BAJcW0YSx_eHEmYjJeKuA,3434
-geoseeq/cli/upload/upload_reads.py,sha256=iInPDnfgEuLa8E4gfENFaZ_Uhxfv1Zfkc38iPvTsTwg,7450
+geoseeq/cli/upload/upload_reads.py,sha256=EMGqyZf11xwN4v2j8gNxMagTbE4kaOd-_hwupmg5I-8,10670
 geoseeq/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 geoseeq/contrib/ncbi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 geoseeq/contrib/ncbi/api.py,sha256=WQeLoGA_-Zha-QeSO8_i7HpvXyD8UkV0qc5okm11KiA,1056
@@ -63,8 +63,8 @@ geoseeq/plotting/map/map.py,sha256=h2QPLGqe-SamhfaTij53S9cQIiO8orCJUAUh0hRicSM,3
 geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEYwE,1795
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
-geoseeq/result/file_download.py,sha256=XQA5bdQJJSZIgbFcN09OvVdLq12fnA98kPCIONAkLk0,5568
-geoseeq/result/file_upload.py,sha256=D8gbdsyzw5ztjT7Vmq4InxKhyl7l8n2leH4GYVD-seM,12109
+geoseeq/result/file_download.py,sha256=vbYo2B4JshTIqLaklcgcBb7NY9cD5pMkas95GuQxW8s,5776
+geoseeq/result/file_upload.py,sha256=qQ2F28W6ieLoEsbqCWr61cQ8hYlNaKkhn4DBQ5xqR-I,13214
 geoseeq/result/result_file.py,sha256=1Yj9fkZhds3J-tay6eNH2-EHi00MovHGV1M80_ckHD8,8677
 geoseeq/result/result_folder.py,sha256=6porOXPh7Tpxw3oX5yMRPYQzNCGYqszqmFJd3SwQmTc,11122
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
@@ -80,9 +80,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.5.6a10.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.5.6a10.dist-info/METADATA,sha256=RKguQ4reNa4rH6Pv20JBMCbyo2inLjKcO5VFkcCgyxg,4806
-geoseeq-0.5.6a10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-geoseeq-0.5.6a10.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.5.6a10.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.5.6a10.dist-info/RECORD,,
+geoseeq-0.5.6a12.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.5.6a12.dist-info/METADATA,sha256=DTEadRCMzytqJtg9vPNagyrMCY8l8gIslgIr7PaOA1o,4806
+geoseeq-0.5.6a12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+geoseeq-0.5.6a12.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.5.6a12.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.5.6a12.dist-info/RECORD,,