geoseeq 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

geoseeq/cli/detail.py

@@ -0,0 +1,39 @@
+import click
+import json
+from .shared_params import (
+    use_common_state,
+    project_id_arg,
+    sample_ids_arg,
+    yes_option,
+    private_option,
+    org_id_arg,
+    handle_project_id,
+    handle_multiple_sample_ids,
+    handle_org_id,
+)
+from geoseeq.id_constructors import resolve_id
+
+
+@click.group('detail')
+def cli_detail():
+    """Detail objects on GeoSeeq."""
+    pass
+
+
+@cli_detail.command('folder')
+@use_common_state
+@click.argument('grn')
+def detail_folder(state, grn):
+    kind, rfolder = resolve_id(state.get_knex(), grn)
+    assert kind == 'folder'
+    click.echo('Folder:')
+    click.echo(rfolder)
+    click.echo('Created at: {}'.format(rfolder.created_at))
+    click.echo('Updated at: {}'.format(rfolder.updated_at))
+    click.echo('Files:')
+    for rfile in rfolder.get_result_files():
+        click.echo(rfile)
+        click.echo('Created at: {}'.format(rfile.created_at))
+        click.echo('Updated at: {}'.format(rfile.updated_at))
+        click.echo(json.dumps(rfile.stored_data, indent=2))
+        click.echo('--')

geoseeq/cli/main.py

@@ -7,7 +7,7 @@ import click
 from .copy import cli_copy
 from .manage import cli_manage
 from .download import cli_download
-from .upload import cli_upload
+from .upload import cli_upload, cli_upload_advanced
 from .user import cli_user
 from .view import cli_view
 from .search import cli_search
@@ -15,6 +15,7 @@ from geoseeq.vc.cli import cli_vc
 from geoseeq.knex import DEFAULT_ENDPOINT
 from .shared_params.config import set_profile
 from .shared_params.opts_and_args import overwrite_option
+from .detail import cli_detail
 
 logger = logging.getLogger('geoseeq_api')
 handler = logging.StreamHandler()
@@ -35,7 +36,7 @@ main.add_command(cli_search)
 @main.command()
 def version():
     """Print the version of the Geoseeq API being used."""
-    click.echo('0.3.0')  # remember to update setup
+    click.echo('0.3.1')  # remember to update setup
 
 
 @main.group('advanced')
@@ -45,6 +46,8 @@ def cli_advanced():
 
 cli_advanced.add_command(cli_copy)
 cli_advanced.add_command(cli_user)
+cli_advanced.add_command(cli_detail)
+cli_advanced.add_command(cli_upload_advanced)
 
 @cli_advanced.group('experimental')
 def cli_experimental():

geoseeq/cli/upload/__init__.py

@@ -5,7 +5,7 @@ from .upload import (
     cli_metadata,
 )
 from .upload_reads import cli_upload_reads_wizard
-
+from .upload_advanced import cli_find_urls_for_reads
 
 @click.group('upload')
 def cli_upload():
@@ -15,3 +15,10 @@ def cli_upload():
 cli_upload.add_command(cli_upload_reads_wizard)
 cli_upload.add_command(cli_upload_file)
 cli_upload.add_command(cli_metadata)
+
+@click.group('upload')
+def cli_upload_advanced():
+    """Advanced tools to upload files to GeoSeeq."""
+    pass
+
+cli_upload_advanced.add_command(cli_find_urls_for_reads)

geoseeq/cli/upload/upload_advanced.py

@@ -0,0 +1,92 @@
+import logging
+import click
+import requests
+from os.path import basename, getsize
+from .upload_reads import (
+    _make_in_process_logger,
+    _get_regex,
+    _group_files,
+    flatten_list_of_fastqs,
+)
+
+from multiprocessing import Pool, current_process
+
+from geoseeq.cli.constants import *
+from geoseeq.cli.shared_params import (
+    handle_project_id,
+    private_option,
+    module_option,
+    project_id_arg,
+    overwrite_option,
+    yes_option,
+    use_common_state,
+)
+
+from geoseeq.constants import FASTQ_MODULE_NAMES
+from geoseeq.cli.progress_bar import PBarManager
+
+logger = logging.getLogger('geoseeq_api')
+
+
+def _keep_only_authentication_url_args(url):
+    """Return a url with only the S3 authentication args"""
+    root, args = url.split('?')
+    args = args.split('&')
+    args = [arg for arg in args if arg.startswith('AWSAccessKeyId=') or arg.startswith('Signature=')]
+    return root + '?' + '&'.join(args)
+
+
+def _get_url_for_one_file(args):
+    """Return a tuple of the filepath and the url to upload it to"""
+    result_file, filepath, overwrite, log_level = args
+    _make_in_process_logger(log_level)
+    if result_file.exists() and not overwrite:  
+        return
+    result_file = result_file.idem()
+    file_size = getsize(filepath)
+    _, urls = result_file._prep_multipart_upload(filepath, file_size, file_size + 1, {})
+    url = _keep_only_authentication_url_args(urls['1'])
+    return filepath, url
+
+
+def _find_target_urls(groups, module_name, lib, filepaths, overwrite, cores, state):
+    """Use GeoSeeq to get target urls for a set of files"""
+    with requests.Session() as session:
+        find_url_args = []
+        for group in groups:
+            sample = lib.sample(group['sample_name']).idem()
+            read_folder = sample.result_folder(module_name).idem()
+
+            for field_name, path in group['fields'].items():
+                result_file = read_folder.read_file(field_name)
+                filepath = filepaths[path]
+                find_url_args.append((
+                    result_file, filepath, overwrite, state.log_level
+                ))
+
+        with Pool(cores) as p:
+            for (file_name, target_url) in p.imap_unordered(_get_url_for_one_file, find_url_args):
+                yield file_name, target_url
+
+
+@click.command('read-links')
+@use_common_state
+@click.option('--cores', default=1, help='Number of uploads to run in parallel')
+@overwrite_option
+@yes_option
+@click.option('--regex', default=None, help='An optional regex to use to extract sample names from the file names')
+@private_option
+@module_option(FASTQ_MODULE_NAMES)
+@project_id_arg
+@click.argument('fastq_files', type=click.Path(exists=True), nargs=-1)
+def cli_find_urls_for_reads(state, cores, overwrite, yes, regex, private, module_name, project_id, fastq_files):
+    """Print a two column list with filenames and a target storage URL
+    """
+    knex = state.get_knex()
+    proj = handle_project_id(knex, project_id, yes, private)
+    filepaths = {basename(line): line for line in flatten_list_of_fastqs(fastq_files)}
+    click.echo(f'Found {len(filepaths)} files to upload.', err=True)
+    regex = _get_regex(knex, filepaths, module_name, proj, regex)
+    groups = _group_files(knex, filepaths, module_name, regex, yes)
+    for file_name, target_url in _find_target_urls(groups, module_name, proj, filepaths, overwrite, cores, state):
+        print(f'{file_name}\t{target_url}', file=state.outfile)

geoseeq/project.py

@@ -20,6 +20,7 @@ class Project(RemoteObject):
     ]
     optional_remote_fields = [
         "privacy_level",
+        "samples_count",
     ]
     parent_field = "org"
     url_prefix = "sample_groups"
@@ -258,7 +259,9 @@ class Project(RemoteObject):
     @property
     def n_samples(self):
         """Return the number of samples in this project."""
-        return self.samples_count
+        if self.hasattr('samples_count') and self.samples_count is not None:
+            return self.samples_count
+        return len(list(self.get_sample_uuids()))
     
     def bulk_find_files(self,
                         sample_uuids=[],

geoseeq/result/file_upload.py

@@ -41,14 +41,13 @@ class FileChunker:
 
 class ResultFileUpload:
     """Abstract class that handles upload methods for result files."""
-    
-    def _prep_multipart_upload(self, filepath, file_size, chunk_size, optional_fields):
-        n_parts = int(file_size / chunk_size) + 1
+
+    def _create_multipart_upload(self, filepath, file_size, optional_fields):
         optional_fields = optional_fields if optional_fields else {}
         optional_fields.update(
             {
                 "md5_checksum": md5_checksum(filepath),
-                "file_size_bytes": getsize(filepath),
+                "file_size_bytes": file_size,
             }
         )
         data = {
@@ -57,6 +56,11 @@ class ResultFileUpload:
             "result_type": "sample" if self.is_sample_result else "group",
         }
         response = self.knex.post(f"/ar_fields/{self.uuid}/create_upload", json=data)
+        return response
+    
+    def _prep_multipart_upload(self, filepath, file_size, chunk_size, optional_fields):
+        n_parts = int(file_size / chunk_size) + 1
+        response = self._create_multipart_upload(filepath, file_size, optional_fields)
         upload_id = response["upload_id"]
         parts = list(range(1, n_parts + 1))
         data = {
@@ -105,6 +109,7 @@ class ResultFileUpload:
 
     def _upload_parts(self, file_chunker, urls, max_retries, session, progress_tracker, threads):
         if threads == 1:
+            logger.info(f"Uploading parts in series for {file_chunker.filepath}")
             complete_parts = []
             for num, url in enumerate(list(urls.values())):
                 response_part = self._upload_one_part(file_chunker, url, num, max_retries, session)
@@ -114,6 +119,7 @@ class ResultFileUpload:
             return complete_parts
         
         with ThreadPoolExecutor(max_workers=threads) as executor:
+            logger.info(f"Uploading parts in parallel for {file_chunker.filepath} with {threads} threads.")
             futures = []
             for num, url in enumerate(list(urls.values())):
                 future = executor.submit(
@@ -128,6 +134,7 @@ class ResultFileUpload:
                 logger.info(
                     f'Uploaded part {response_part["PartNumber"]} of {len(urls)} for "{file_chunker.filepath}"'
                 )
+        complete_parts = sorted(complete_parts, key=lambda x: x["PartNumber"])
         return complete_parts
 
     def multipart_upload_file(

{geoseeq-0.3.0.dist-info → geoseeq-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.3.0
+Version: 0.3.1
 Summary: # Geoseeq API Client
 Author: David C. Danko
 Author-email: dcdanko@biotia.io

{geoseeq-0.3.0.dist-info → geoseeq-0.3.1.dist-info}/RECORD

@@ -7,7 +7,7 @@ geoseeq/file_system_cache.py,sha256=7pdM-1VncOvjkYv2pOocPHOXnEWz0jKhH6sm1LaXtoU,
 geoseeq/knex.py,sha256=WWYV3rwGuCdWL6zucgeExbaMmTg-I7IYqRHuvxhc6Q4,6007
 geoseeq/organization.py,sha256=a9xmGDE0tQsjPJfyFkYnWagxZ8xpdeckkwvkhH6LNIk,2462
 geoseeq/pipeline.py,sha256=RuAHyJM0wb0Z7WUPT7_8O6wB4UH5VUiBaPmWsLVeIKo,6819
-geoseeq/project.py,sha256=9U_fFFtoableLvsMI2_7zTwJLXJe5C_1oFY7Xuap1Rc,12924
+geoseeq/project.py,sha256=8HrK9JRhhXV03E9i7FQHPNj18vaEn7W81M7Ak0_S0-o,13080
 geoseeq/remote_object.py,sha256=6owZTIubA2wwbA4AiL0HdfWDMpZjOhUgqv_KxNG2XZg,6613
 geoseeq/sample.py,sha256=348NgY7wQBmFpoTq24T0ffVQ66iKB-hsCx2YdsBGTlc,4854
 geoseeq/search.py,sha256=gawad6Cx5FxJBPlYkXWb-UKAO-UC0_yhvyU9Ca1kaNI,3388
@@ -17,9 +17,10 @@ geoseeq/work_orders.py,sha256=5uLVVfdKE8qh4gGaHkdBpXJGRTujuSg59knWCqEET4A,8071
 geoseeq/cli/__init__.py,sha256=4WnK87K5seRK3SGJAxNWnQTqyg5uBhdhrOrzB1D4b3M,24
 geoseeq/cli/constants.py,sha256=Do5AUf9lMO9_P8KpFJ3XwwFBAWsxSjZ6sx9_QEGyC_c,176
 geoseeq/cli/copy.py,sha256=ocJ-T6vZBTPELUSBA65KndzAugrBWMq-3ux0dbffacE,1997
+geoseeq/cli/detail.py,sha256=uWxJ3v2GTpfGCxXkFHRqYilEUpZjJTkzES-Mpa2HEe4,1040
 geoseeq/cli/download.py,sha256=Hn2RhRgUpy1QDZuVkG6JgKlthGLnnfVYXIYrVsYcYL4,11859
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
-geoseeq/cli/main.py,sha256=03gd1t0wMSybDKtyubcnmhS6N-J0zZE-k21UCmKKoW4,1915
+geoseeq/cli/main.py,sha256=JgaUKKK_rm1CuGdUBzg2m_WJHV5fWSRj5DPTSRlf1xM,2050
 geoseeq/cli/manage.py,sha256=xy_iMfqSSMaSTQNYiTw2xEWpH4mpTAq4Cf_6QemEMnI,5409
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/search.py,sha256=KpqKiq4-wbgO6xrx58NxomYZcx451NqzA_MecgiND1g,881
@@ -32,8 +33,9 @@ geoseeq/cli/shared_params/config.py,sha256=06KrZhC3zljveo_21YvaZPcKIzyDBGHT_Yr79
 geoseeq/cli/shared_params/id_handlers.py,sha256=7s2g-UfKoTfwmDX_FI96-M_g_U1yYBUEj-8yy8NC_c8,6493
 geoseeq/cli/shared_params/obj_getters.py,sha256=ZSkt6LnDkVFlNVYKgLrjzg60-6BthZMr3eeD3HNqzac,2741
 geoseeq/cli/shared_params/opts_and_args.py,sha256=Jy2rVQydlvkgjoA_BYjSNuI-wUSPmE2S1C1rN6bycBI,1399
-geoseeq/cli/upload/__init__.py,sha256=QqBh0WXD9BaEDcKsKpb-WgWwQ0WzZdQUgYXYwyowW_c,340
+geoseeq/cli/upload/__init__.py,sha256=Wf30XGLllFFnnfeOnpYsO3SXyaq8yclHcYxGUNdSh6o,562
 geoseeq/cli/upload/upload.py,sha256=Ikly-vfLzDXpaMpyFTim1CfgqNjtrnRQHgOMi_7JubU,6367
+geoseeq/cli/upload/upload_advanced.py,sha256=Jq5eGe-wOdrzxGWVwaFPg0BAJcW0YSx_eHEmYjJeKuA,3434
 geoseeq/cli/upload/upload_reads.py,sha256=f5NPbnlZg93zjDUIX_OsjyOyvvEtkIn_ioYLFfClfjw,7583
 geoseeq/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 geoseeq/contrib/ncbi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -59,7 +61,7 @@ geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEY
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
 geoseeq/result/file_download.py,sha256=R087ypOAu2pMZ9vXvkO9babPBNtRXTF3ZhpPldcT_jM,4593
-geoseeq/result/file_upload.py,sha256=D5PpmraEnF1sNTJDitKnPiHwo1ak1s1UW_TBwYpd_ko,6707
+geoseeq/result/file_upload.py,sha256=g29WES3IjV8pmB8g3VRzxUk3wsSNgkCB6HIqZ_4ctuY,7158
 geoseeq/result/result_file.py,sha256=YMCAivUXgHUNEhH6yTGCHJh9gXHd8OLjjbnccdvW7iw,7138
 geoseeq/result/result_folder.py,sha256=qmxm-Z6DI1ohGnIJgQfh27AVJ1dbHM_p2XprepwUKEg,7782
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
@@ -75,9 +77,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.3.0.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.3.0.dist-info/METADATA,sha256=KpwaJ3heZAhI7ptGED_3yevWohc2BS0BKRIcxhwky8M,454
-geoseeq-0.3.0.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-geoseeq-0.3.0.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.3.0.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.3.0.dist-info/RECORD,,
+geoseeq-0.3.1.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.3.1.dist-info/METADATA,sha256=10uUQhfABpFPFN4KwAcA8NyABrFKgStOwecU65HlHvI,454
+geoseeq-0.3.1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+geoseeq-0.3.1.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.3.1.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.3.1.dist-info/RECORD,,