geoseeq 0.2.22__py3-none-any.whl → 0.3.1__py3-none-any.whl

geoseeq/cli/detail.py

@@ -0,0 +1,39 @@
+import click
+import json
+from .shared_params import (
+    use_common_state,
+    project_id_arg,
+    sample_ids_arg,
+    yes_option,
+    private_option,
+    org_id_arg,
+    handle_project_id,
+    handle_multiple_sample_ids,
+    handle_org_id,
+)
+from geoseeq.id_constructors import resolve_id
+
+
+@click.group('detail')
+def cli_detail():
+    """Detail objects on GeoSeeq."""
+    pass
+
+
+@cli_detail.command('folder')
+@use_common_state
+@click.argument('grn')
+def detail_folder(state, grn):
+    kind, rfolder = resolve_id(state.get_knex(), grn)
+    assert kind == 'folder'
+    click.echo('Folder:')
+    click.echo(rfolder)
+    click.echo('Created at: {}'.format(rfolder.created_at))
+    click.echo('Updated at: {}'.format(rfolder.updated_at))
+    click.echo('Files:')
+    for rfile in rfolder.get_result_files():
+        click.echo(rfile)
+        click.echo('Created at: {}'.format(rfile.created_at))
+        click.echo('Updated at: {}'.format(rfile.updated_at))
+        click.echo(json.dumps(rfile.stored_data, indent=2))
+        click.echo('--')

geoseeq/cli/download.py

@@ -166,22 +166,22 @@ def cli_download_files(
     """
     knex = state.get_knex()
     proj = handle_project_id(knex, project_id)
+    logger.info(f"Found project \"{proj.name}\"")
     samples = []
     if sample_ids:
         logger.info(f"Fetching info for {len(sample_ids)} samples.")
         samples = handle_multiple_sample_ids(knex, sample_ids, proj=proj)
 
-    data = {
-        "sample_uuids": [s.uuid for s in samples],
-        "sample_names": sample_name_includes,
-        "folder_type": folder_type,
-        "folder_names": folder_name,
-        "file_names": file_name,
-        "extensions": extension,
-        "with_versions": with_versions
-    }
-    url = f"sample_groups/{proj.uuid}/download"
-    response = knex.post(url, data)
+    response = proj.bulk_find_files(
+        sample_uuids=[s.uuid for s in samples],
+        sample_name_includes=sample_name_includes,
+        folder_types=folder_type,
+        folder_names=folder_name,
+        file_names=file_name,
+        extensions=extension,
+        with_versions=with_versions,
+    )
+
 
     if not download:
         data = json.dumps(response["links"])

geoseeq/cli/main.py

@@ -7,7 +7,7 @@ import click
 from .copy import cli_copy
 from .manage import cli_manage
 from .download import cli_download
-from .upload import cli_upload
+from .upload import cli_upload, cli_upload_advanced
 from .user import cli_user
 from .view import cli_view
 from .search import cli_search
@@ -15,6 +15,7 @@ from geoseeq.vc.cli import cli_vc
 from geoseeq.knex import DEFAULT_ENDPOINT
 from .shared_params.config import set_profile
 from .shared_params.opts_and_args import overwrite_option
+from .detail import cli_detail
 
 logger = logging.getLogger('geoseeq_api')
 handler = logging.StreamHandler()
@@ -35,7 +36,7 @@ main.add_command(cli_search)
 @main.command()
 def version():
     """Print the version of the Geoseeq API being used."""
-    click.echo('0.2.22')  # remember to update setup
+    click.echo('0.3.1')  # remember to update setup
 
 
 @main.group('advanced')
@@ -45,6 +46,8 @@ def cli_advanced():
 
 cli_advanced.add_command(cli_copy)
 cli_advanced.add_command(cli_user)
+cli_advanced.add_command(cli_detail)
+cli_advanced.add_command(cli_upload_advanced)
 
 @cli_advanced.group('experimental')
 def cli_experimental():

geoseeq/cli/upload/__init__.py

@@ -5,7 +5,7 @@ from .upload import (
     cli_metadata,
 )
 from .upload_reads import cli_upload_reads_wizard
-
+from .upload_advanced import cli_find_urls_for_reads
 
 @click.group('upload')
 def cli_upload():
@@ -15,3 +15,10 @@ def cli_upload():
 cli_upload.add_command(cli_upload_reads_wizard)
 cli_upload.add_command(cli_upload_file)
 cli_upload.add_command(cli_metadata)
+
+@click.group('upload')
+def cli_upload_advanced():
+    """Advanced tools to upload files to GeoSeeq."""
+    pass
+
+cli_upload_advanced.add_command(cli_find_urls_for_reads)

geoseeq/cli/upload/upload_advanced.py

@@ -0,0 +1,92 @@
+import logging
+import click
+import requests
+from os.path import basename, getsize
+from .upload_reads import (
+    _make_in_process_logger,
+    _get_regex,
+    _group_files,
+    flatten_list_of_fastqs,
+)
+
+from multiprocessing import Pool, current_process
+
+from geoseeq.cli.constants import *
+from geoseeq.cli.shared_params import (
+    handle_project_id,
+    private_option,
+    module_option,
+    project_id_arg,
+    overwrite_option,
+    yes_option,
+    use_common_state,
+)
+
+from geoseeq.constants import FASTQ_MODULE_NAMES
+from geoseeq.cli.progress_bar import PBarManager
+
+logger = logging.getLogger('geoseeq_api')
+
+
+def _keep_only_authentication_url_args(url):
+    """Return a url with only the S3 authentication args"""
+    root, args = url.split('?')
+    args = args.split('&')
+    args = [arg for arg in args if arg.startswith('AWSAccessKeyId=') or arg.startswith('Signature=')]
+    return root + '?' + '&'.join(args)
+
+
+def _get_url_for_one_file(args):
+    """Return a tuple of the filepath and the url to upload it to"""
+    result_file, filepath, overwrite, log_level = args
+    _make_in_process_logger(log_level)
+    if result_file.exists() and not overwrite:  
+        return
+    result_file = result_file.idem()
+    file_size = getsize(filepath)
+    _, urls = result_file._prep_multipart_upload(filepath, file_size, file_size + 1, {})
+    url = _keep_only_authentication_url_args(urls['1'])
+    return filepath, url
+
+
+def _find_target_urls(groups, module_name, lib, filepaths, overwrite, cores, state):
+    """Use GeoSeeq to get target urls for a set of files"""
+    with requests.Session() as session:
+        find_url_args = []
+        for group in groups:
+            sample = lib.sample(group['sample_name']).idem()
+            read_folder = sample.result_folder(module_name).idem()
+
+            for field_name, path in group['fields'].items():
+                result_file = read_folder.read_file(field_name)
+                filepath = filepaths[path]
+                find_url_args.append((
+                    result_file, filepath, overwrite, state.log_level
+                ))
+
+        with Pool(cores) as p:
+            for (file_name, target_url) in p.imap_unordered(_get_url_for_one_file, find_url_args):
+                yield file_name, target_url
+
+
+@click.command('read-links')
+@use_common_state
+@click.option('--cores', default=1, help='Number of uploads to run in parallel')
+@overwrite_option
+@yes_option
+@click.option('--regex', default=None, help='An optional regex to use to extract sample names from the file names')
+@private_option
+@module_option(FASTQ_MODULE_NAMES)
+@project_id_arg
+@click.argument('fastq_files', type=click.Path(exists=True), nargs=-1)
+def cli_find_urls_for_reads(state, cores, overwrite, yes, regex, private, module_name, project_id, fastq_files):
+    """Print a two column list with filenames and a target storage URL
+    """
+    knex = state.get_knex()
+    proj = handle_project_id(knex, project_id, yes, private)
+    filepaths = {basename(line): line for line in flatten_list_of_fastqs(fastq_files)}
+    click.echo(f'Found {len(filepaths)} files to upload.', err=True)
+    regex = _get_regex(knex, filepaths, module_name, proj, regex)
+    groups = _group_files(knex, filepaths, module_name, regex, yes)
+    for file_name, target_url in _find_target_urls(groups, module_name, proj, filepaths, overwrite, cores, state):
+        print(f'{file_name}\t{target_url}', file=state.outfile)

geoseeq/project.py

@@ -4,7 +4,9 @@ from .sample import Sample
 from .utils import paginated_iterator
 import json
 import pandas as pd
+import logging
 
+logger = logging.getLogger("geoseeq_api")
 
 
 class Project(RemoteObject):
@@ -14,9 +16,11 @@ class Project(RemoteObject):
         "name",
         "privacy_level",
         "description",
+        "samples_count",
     ]
     optional_remote_fields = [
         "privacy_level",
+        "samples_count",
     ]
     parent_field = "org"
     url_prefix = "sample_groups"
@@ -195,6 +199,18 @@ class Project(RemoteObject):
         for sample_blob in paginated_iterator(self.knex, url, error_handler=error_handler):
             yield sample_blob['uuid']
 
+    def _batch_sample_uuids(self, batch_size, input_sample_uuids=[]):
+        """Yield batches of sample uuids."""
+        uuids_to_batch = input_sample_uuids if input_sample_uuids else self.get_sample_uuids()
+        sample_uuids = []
+        for sample_uuid in uuids_to_batch:
+            sample_uuids.append(sample_uuid)
+            if len(sample_uuids) == batch_size:
+                yield sample_uuids
+                sample_uuids = []
+        if sample_uuids:
+            yield sample_uuids
+
     def get_analysis_results(self, cache=True):
         """Yield ProjectResultFolder objects for this project fetched from the server.
         
@@ -239,6 +255,76 @@ class Project(RemoteObject):
         url = f"sample_groups/{self.uuid}/metadata"
         blob = self.knex.get(url)
         return pd.DataFrame.from_dict(blob, orient="index")
+    
+    @property
+    def n_samples(self):
+        """Return the number of samples in this project."""
+        if self.hasattr('samples_count') and self.samples_count is not None:
+            return self.samples_count
+        return len(list(self.get_sample_uuids()))
+    
+    def bulk_find_files(self,
+                        sample_uuids=[],
+                        sample_name_includes=[],
+                        folder_types="all",
+                        folder_names=[],
+                        file_names=[],
+                        extensions=[],
+                        with_versions=False,
+                        use_batches_cutoff=500):
+        """Return a dict with links to download files that match the given criteria.
+
+        Options:
+        - sample_uuids: list of sample uuids; if blank search all samples in project
+        - sample_name_includes: list of strings; finds samples with names that include these strings
+        - folder_types: "all", "project", "sample"; finds files in folders of these types
+        - folder_names: list of strings; finds files in folders that have these strings in their names
+        - file_names: list of strings; finds files that have these strings in their names
+        - extensions: list of strings; finds files with these file extensions
+        - with_versions: bool; if True, include all versions of files in results
+        """
+        def _my_bulk_find(sample_uuids=[]):  # curry to save typing
+            return self._bulk_find_files_batch(sample_uuids=sample_uuids,
+                                             sample_name_includes=sample_name_includes,
+                                             folder_types=folder_types,
+                                             folder_names=folder_names,
+                                             file_names=file_names,
+                                             extensions=extensions,
+                                             with_versions=with_versions)
+        n_samples = len(sample_uuids) if sample_uuids else self.n_samples
+        if n_samples < use_batches_cutoff:
+            logger.debug(f"Using single batch bulk_find for {n_samples} samples")
+            return _my_bulk_find()
+        else:
+            logger.debug(f"Using multi batch bulk_find for {n_samples} samples")
+            merged_response = {'file_size_bytes': 0, 'links': {}, 'no_size_info_count': 0}
+            for batch in self._batch_sample_uuids(use_batches_cutoff - 1, input_sample_uuids=sample_uuids):
+                response = _my_bulk_find(sample_uuids=batch)
+                merged_response['file_size_bytes'] += response['file_size_bytes']
+                merged_response['links'].update(response['links'])
+                merged_response['no_size_info_count'] += response['no_size_info_count']
+            return merged_response
+                
+    def _bulk_find_files_batch(self,
+                               sample_uuids=[],
+                               sample_name_includes=[],
+                               folder_types=[],
+                               folder_names=[],
+                               file_names=[],
+                               extensions=[],
+                               with_versions=False):
+        data = {
+            "sample_uuids": sample_uuids,
+            "sample_names": sample_name_includes,
+            "folder_type": folder_types,
+            "folder_names": folder_names,
+            "file_names": file_names,
+            "extensions": extensions,
+            "with_versions": with_versions
+        }
+        url = f"sample_groups/{self.uuid}/download"
+        response = self.knex.post(url, data)
+        return response
 
     def __str__(self):
         return f"<Geoseeq::Project {self.name} {self.uuid} />"

geoseeq/result/file_upload.py

@@ -41,14 +41,13 @@ class FileChunker:
 
 class ResultFileUpload:
     """Abstract class that handles upload methods for result files."""
-    
-    def _prep_multipart_upload(self, filepath, file_size, chunk_size, optional_fields):
-        n_parts = int(file_size / chunk_size) + 1
+
+    def _create_multipart_upload(self, filepath, file_size, optional_fields):
         optional_fields = optional_fields if optional_fields else {}
         optional_fields.update(
             {
                 "md5_checksum": md5_checksum(filepath),
-                "file_size_bytes": getsize(filepath),
+                "file_size_bytes": file_size,
             }
         )
         data = {
@@ -57,6 +56,11 @@ class ResultFileUpload:
             "result_type": "sample" if self.is_sample_result else "group",
         }
         response = self.knex.post(f"/ar_fields/{self.uuid}/create_upload", json=data)
+        return response
+    
+    def _prep_multipart_upload(self, filepath, file_size, chunk_size, optional_fields):
+        n_parts = int(file_size / chunk_size) + 1
+        response = self._create_multipart_upload(filepath, file_size, optional_fields)
         upload_id = response["upload_id"]
         parts = list(range(1, n_parts + 1))
         data = {
@@ -105,6 +109,7 @@ class ResultFileUpload:
 
     def _upload_parts(self, file_chunker, urls, max_retries, session, progress_tracker, threads):
         if threads == 1:
+            logger.info(f"Uploading parts in series for {file_chunker.filepath}")
             complete_parts = []
             for num, url in enumerate(list(urls.values())):
                 response_part = self._upload_one_part(file_chunker, url, num, max_retries, session)
@@ -114,6 +119,7 @@ class ResultFileUpload:
             return complete_parts
         
         with ThreadPoolExecutor(max_workers=threads) as executor:
+            logger.info(f"Uploading parts in parallel for {file_chunker.filepath} with {threads} threads.")
             futures = []
             for num, url in enumerate(list(urls.values())):
                 future = executor.submit(
@@ -128,6 +134,7 @@ class ResultFileUpload:
                 logger.info(
                     f'Uploaded part {response_part["PartNumber"]} of {len(urls)} for "{file_chunker.filepath}"'
                 )
+        complete_parts = sorted(complete_parts, key=lambda x: x["PartNumber"])
         return complete_parts
 
     def multipart_upload_file(

{geoseeq-0.2.22.dist-info → geoseeq-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.2.22
+Version: 0.3.1
 Summary: # Geoseeq API Client
 Author: David C. Danko
 Author-email: dcdanko@biotia.io

{geoseeq-0.2.22.dist-info → geoseeq-0.3.1.dist-info}/RECORD

@@ -7,7 +7,7 @@ geoseeq/file_system_cache.py,sha256=7pdM-1VncOvjkYv2pOocPHOXnEWz0jKhH6sm1LaXtoU,
 geoseeq/knex.py,sha256=WWYV3rwGuCdWL6zucgeExbaMmTg-I7IYqRHuvxhc6Q4,6007
 geoseeq/organization.py,sha256=a9xmGDE0tQsjPJfyFkYnWagxZ8xpdeckkwvkhH6LNIk,2462
 geoseeq/pipeline.py,sha256=RuAHyJM0wb0Z7WUPT7_8O6wB4UH5VUiBaPmWsLVeIKo,6819
-geoseeq/project.py,sha256=dYawsv-K55WL5aAZTXRi7BQd34Ydo1ebVebbHYvrFvc,8795
+geoseeq/project.py,sha256=8HrK9JRhhXV03E9i7FQHPNj18vaEn7W81M7Ak0_S0-o,13080
 geoseeq/remote_object.py,sha256=6owZTIubA2wwbA4AiL0HdfWDMpZjOhUgqv_KxNG2XZg,6613
 geoseeq/sample.py,sha256=348NgY7wQBmFpoTq24T0ffVQ66iKB-hsCx2YdsBGTlc,4854
 geoseeq/search.py,sha256=gawad6Cx5FxJBPlYkXWb-UKAO-UC0_yhvyU9Ca1kaNI,3388
@@ -17,9 +17,10 @@ geoseeq/work_orders.py,sha256=5uLVVfdKE8qh4gGaHkdBpXJGRTujuSg59knWCqEET4A,8071
 geoseeq/cli/__init__.py,sha256=4WnK87K5seRK3SGJAxNWnQTqyg5uBhdhrOrzB1D4b3M,24
 geoseeq/cli/constants.py,sha256=Do5AUf9lMO9_P8KpFJ3XwwFBAWsxSjZ6sx9_QEGyC_c,176
 geoseeq/cli/copy.py,sha256=ocJ-T6vZBTPELUSBA65KndzAugrBWMq-3ux0dbffacE,1997
-geoseeq/cli/download.py,sha256=XJHvcYrGxbsVf5khkbg8pGyhuRh57yYa1NnlaR3CPMk,11879
+geoseeq/cli/detail.py,sha256=uWxJ3v2GTpfGCxXkFHRqYilEUpZjJTkzES-Mpa2HEe4,1040
+geoseeq/cli/download.py,sha256=Hn2RhRgUpy1QDZuVkG6JgKlthGLnnfVYXIYrVsYcYL4,11859
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
-geoseeq/cli/main.py,sha256=A6-4Rd7nCnDUEBhjMVS-Wky8LhqtoQA1rQREpIJ1TsE,1916
+geoseeq/cli/main.py,sha256=JgaUKKK_rm1CuGdUBzg2m_WJHV5fWSRj5DPTSRlf1xM,2050
 geoseeq/cli/manage.py,sha256=xy_iMfqSSMaSTQNYiTw2xEWpH4mpTAq4Cf_6QemEMnI,5409
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/search.py,sha256=KpqKiq4-wbgO6xrx58NxomYZcx451NqzA_MecgiND1g,881
@@ -32,8 +33,9 @@ geoseeq/cli/shared_params/config.py,sha256=06KrZhC3zljveo_21YvaZPcKIzyDBGHT_Yr79
 geoseeq/cli/shared_params/id_handlers.py,sha256=7s2g-UfKoTfwmDX_FI96-M_g_U1yYBUEj-8yy8NC_c8,6493
 geoseeq/cli/shared_params/obj_getters.py,sha256=ZSkt6LnDkVFlNVYKgLrjzg60-6BthZMr3eeD3HNqzac,2741
 geoseeq/cli/shared_params/opts_and_args.py,sha256=Jy2rVQydlvkgjoA_BYjSNuI-wUSPmE2S1C1rN6bycBI,1399
-geoseeq/cli/upload/__init__.py,sha256=QqBh0WXD9BaEDcKsKpb-WgWwQ0WzZdQUgYXYwyowW_c,340
+geoseeq/cli/upload/__init__.py,sha256=Wf30XGLllFFnnfeOnpYsO3SXyaq8yclHcYxGUNdSh6o,562
 geoseeq/cli/upload/upload.py,sha256=Ikly-vfLzDXpaMpyFTim1CfgqNjtrnRQHgOMi_7JubU,6367
+geoseeq/cli/upload/upload_advanced.py,sha256=Jq5eGe-wOdrzxGWVwaFPg0BAJcW0YSx_eHEmYjJeKuA,3434
 geoseeq/cli/upload/upload_reads.py,sha256=f5NPbnlZg93zjDUIX_OsjyOyvvEtkIn_ioYLFfClfjw,7583
 geoseeq/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 geoseeq/contrib/ncbi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -59,7 +61,7 @@ geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEY
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
 geoseeq/result/file_download.py,sha256=R087ypOAu2pMZ9vXvkO9babPBNtRXTF3ZhpPldcT_jM,4593
-geoseeq/result/file_upload.py,sha256=D5PpmraEnF1sNTJDitKnPiHwo1ak1s1UW_TBwYpd_ko,6707
+geoseeq/result/file_upload.py,sha256=g29WES3IjV8pmB8g3VRzxUk3wsSNgkCB6HIqZ_4ctuY,7158
 geoseeq/result/result_file.py,sha256=YMCAivUXgHUNEhH6yTGCHJh9gXHd8OLjjbnccdvW7iw,7138
 geoseeq/result/result_folder.py,sha256=qmxm-Z6DI1ohGnIJgQfh27AVJ1dbHM_p2XprepwUKEg,7782
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
@@ -75,9 +77,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.2.22.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.2.22.dist-info/METADATA,sha256=JUu4rlER6ebyt5r6HBvGoS5rYMYsSjAgqtBRRMSUFII,455
-geoseeq-0.2.22.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-geoseeq-0.2.22.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.2.22.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.2.22.dist-info/RECORD,,
+geoseeq-0.3.1.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.3.1.dist-info/METADATA,sha256=10uUQhfABpFPFN4KwAcA8NyABrFKgStOwecU65HlHvI,454
+geoseeq-0.3.1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+geoseeq-0.3.1.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.3.1.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.3.1.dist-info/RECORD,,