geoseeq 0.7.3.dev4__tar.gz → 0.7.5__tar.gz

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geoseeq
-Version: 0.7.3.dev4
+Version: 0.7.5
 Summary: GeoSeeq command line tools and python API
 Project-URL: Homepage, https://github.com/biotia/geoseeq_api_client
 Project-URL: Issues, https://github.com/biotia/geoseeq_api_client/issues
@@ -84,7 +84,7 @@ $ geoseeq download files --extension fastq.gz "GeoSeeq/Example CLI Project"
 
 GeoSeeq can automatically group fastq files into samples according to their
 sample name, read number, and lane number. It supports paired end, single end,
-and nanopore reads.
+nanopore, and pacbio reads.
 
 Assume you have data from a single ended sequencing run stored as fastq files:
  - Sample1_L1_R1.fastq.gz
@@ -115,6 +115,12 @@ GeoSeeq will automatically create a new sample named `Sample1` if it does not al
 
 This command would upload data [to this project.](https://portal.geoseeq.com/sample-groups/ed59b913-91ec-489b-a1b9-4ea137a6e5cf/samples). Since only organization members can upload data, you will need to replace `GeoSeeq` with your organization name.
 
+To rename samples on the fly, provide a CSV file with current and new names using the `--name-map` option:
+
+```
+$ geoseeq upload reads --name-map sample_map.csv current_name new_name "GeoSeeq/Example CLI Project" fastq_files.txt
+```
+
 Note: You will need to have an API token set to use this command (see above)
 
 ## Using the Python API in a program

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/README.md

@@ -60,7 +60,7 @@ $ geoseeq download files --extension fastq.gz "GeoSeeq/Example CLI Project"
 
 GeoSeeq can automatically group fastq files into samples according to their
 sample name, read number, and lane number. It supports paired end, single end,
-and nanopore reads.
+nanopore, and pacbio reads.
 
 Assume you have data from a single ended sequencing run stored as fastq files:
  - Sample1_L1_R1.fastq.gz
@@ -91,6 +91,12 @@ GeoSeeq will automatically create a new sample named `Sample1` if it does not al
 
 This command would upload data [to this project.](https://portal.geoseeq.com/sample-groups/ed59b913-91ec-489b-a1b9-4ea137a6e5cf/samples). Since only organization members can upload data, you will need to replace `GeoSeeq` with your organization name.
 
+To rename samples on the fly, provide a CSV file with current and new names using the `--name-map` option:
+
+```
+$ geoseeq upload reads --name-map sample_map.csv current_name new_name "GeoSeeq/Example CLI Project" fastq_files.txt
+```
+
 Note: You will need to have an API token set to use this command (see above)
 
 ## Using the Python API in a program

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/docs/uploading_data_examples.md

@@ -16,11 +16,11 @@ $ export GEOSEEQ_API_TOKEN=<your token from the geoseeq app>
 
 ## Uploading sequencing data
 
-GeoSeeq can automatically group fastq files into samples according to their 
+GeoSeeq can automatically group fastq files into samples according to their
 sample name, read number, and lane number. It supports paired end, single end,
-and nanopore reads.
+nanopore, and pacbio reads.
 
-Assume you have data from a single ended sequencing run stored as fastq files: 
+Assume you have data from a single ended sequencing run stored as fastq files:
  - Sample1_L1_R1.fastq.gz
  - Sample1_L1_R2.fastq.gz
  - Sample1_L2_R1.fastq.gz
@@ -47,13 +47,20 @@ Uploading Sample: Sample1
 
 GeoSeeq will automatically create a new sample named `Sample1` if it does not already exist.
 
+To rename samples during upload, provide a CSV with current and new names
+using `--name-map`:
+
+```
+$ geoseeq upload reads --name-map sample_map.csv current_name new_name "Example GeoSeeq Org/Example CLI Project" fastq_files.txt
+```
+
 Note: You will need to have an API token set to use this command (see above)
 
 ### Linking reads from S3, Wasabi, FTP, Azure, and other cloud storage services
 
 GeoSeeq allows you to link files stored on other cloud storage services without moving the files.
 
-Assume you have data from a single ended sequencing run stored as fastq files on an s3 bucket: 
+Assume you have data from a single ended sequencing run stored as fastq files on an s3 bucket:
  - `https://s3.wasabisys.com/mybucketname/Sample1_L1_R1.fastq.gz`
  - `https://s3.wasabisys.com/mybucketname/Sample1_L1_R2.fastq.gz`
  - `https://s3.wasabisys.com/mybucketname/Sample1_L2_R1.fastq.gz`

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/geoseeq/cli/constants.py

@@ -1,11 +1,11 @@
 SINGLE_END="short_read::single_end"
 PAIRED_END="short_read::paired_end"
 NANOPORE="long_read::nanopore"
+PACBIO="long_read::pacbio"
 
 READ_MODULE_NAMES = [
     SINGLE_END,
     PAIRED_END,
     NANOPORE,
+    PACBIO,
 ]
-
-

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/geoseeq/cli/download.py

@@ -3,6 +3,7 @@ import logging
 from os import makedirs
 from os.path import dirname, join
 
+import gzip
 import click
 import pandas as pd
 from multiprocessing import Pool
@@ -32,6 +33,7 @@ from .utils import convert_size
 from geoseeq.constants import FASTQ_MODULE_NAMES
 from geoseeq.result import ResultFile
 from geoseeq.upload_download_manager import GeoSeeqDownloadManager
+import os
 
 logger = logging.getLogger('geoseeq_api')
 
@@ -378,7 +380,27 @@ def cli_download_ids(state, cores, target_dir, file_name, yes, download, head, i
         download_manager.download_files()
 
 
-def _get_sample_result_files_with_names(sample, module_name=None, first=False):
+def _get_local_filename_for_fastq(sample, result_file, read_type, read_num, lane_num, file_name_mode):
+    """Return a local filename for a fastq file based on the specified naming mode."""
+    if file_name_mode == "original":
+        return result_file.get_stored_data_filename()
+    elif file_name_mode == "geoseeq":
+        sname = sample.name.replace(".", "-").replace(" ", "_").lower()
+        rtype = read_type.replace("::", "__").replace(".", "-").replace(" ", "_").lower()
+        filename = f"{sname}.{rtype}.R{read_num}.L{lane_num}.fastq.gz"
+        return filename
+    elif file_name_mode == "sample-uuid":
+        filename = f"{sample.uuid}.R{read_num}.L{lane_num}.fastq.gz"
+        return filename
+    elif file_name_mode == "file-uuid":
+        filename = f"{result_file.uuid}.fastq.gz"
+        return filename
+    else:
+        raise ValueError(f"Unknown file name mode: {file_name_mode}")
+
+
+def _get_sample_result_files_with_names(sample, module_name=None, which_fastqs_mode='all', file_name_mode='original'):
+    """Return list of (result_file, filename, key) tuples for all fastq files in a sample."""
     result_files_with_names = []
     for read_type, folder in sample.get_all_fastqs().items():
         if module_name and module_name != read_type:
@@ -388,19 +410,18 @@ def _get_sample_result_files_with_names(sample, module_name=None, first=False):
                 lane_num = lane_num + 1  # 1 indexed
                 if read_type in ["short_read::paired_end"]:
                     key = (sample, read_type, 1, lane_num)  # sample name, read type, read number, lane number
-                    result_files_with_names.append(
-                        (result_file[0], result_file[0].get_referenced_filename(), key)
-                    )
+                    fname = _get_local_filename_for_fastq(sample, result_file[0], read_type, 1, lane_num, file_name_mode)
+                    result_files_with_names.append((result_file[0], fname, key))
+                    if which_fastqs_mode == "first-r1":
+                        break
                     key = (sample, read_type, 2, lane_num)
-                    result_files_with_names.append(
-                        (result_file[1], result_file[1].get_referenced_filename(), key)
-                    )
+                    fname = _get_local_filename_for_fastq(sample, result_file[1], read_type, 2, lane_num, file_name_mode)
+                    result_files_with_names.append((result_file[1], fname, key))
                 else:
                     key = (sample, read_type, 1, lane_num)
-                    result_files_with_names.append(
-                        (result_file, result_file.get_referenced_filename(), key)
-                    )
-            if first:
+                    fname = _get_local_filename_for_fastq(sample, result_file, read_type, 1, lane_num, file_name_mode)
+                    result_files_with_names.append((result_file, fname, key))
+            if which_fastqs_mode in ["first-all", "first-r1"]:
                 break
 
     return result_files_with_names
@@ -442,14 +463,52 @@ def _make_read_configs(download_results, config_dir="."):
         with open(config_path, "w") as f:
             json.dump(config_blob, f, indent=4)
 
+def _open_maybe_gzip(local_path):
+    """Open a file that may be gzipped. Do not rely on file extension."""
+    with open(local_path, "rb") as f:
+        magic_number = f.read(2)
+    if magic_number == b'\x1f\x8b':
+        return gzip.open(local_path, "rt")
+    else:
+        return open(local_path, "r")
+
+
+def _trim_fastq_to_complete_reads(key, local_path):
+    """Trim a fastq file to the nearest complete read boundary under head_bytes.
+    
+    Write the output as a gzipped file regardless of input compression.
+    """
+    temp_path = local_path + ".tmp"
+    with _open_maybe_gzip(local_path) as infile, gzip.open(temp_path, "wt") as outfile:
+        lines_written = 0
+        while True:
+            read_lines = []
+            for _ in range(4):
+                line = infile.readline()
+                if not line:
+                    break
+                read_lines.append(line)
+            if len(read_lines) < 4:
+                break  # end of file
+            if infile.tell() > key[4]:  # key[4] is head_bytes
+                break  # reached head limit
+            for line in read_lines:
+                outfile.write(line)
+            lines_written += 4
+    # Replace original file with trimmed file
+    
+    os.replace(temp_path, local_path)
+
 
 @cli_download.command("fastqs")
 @use_common_state
 @cores_option
 @click.option("--target-dir", default=".")
 @yes_option
-@click.option("--first/--all", default=False, help="Download only the first folder of fastq files for each sample.")
+@click.option('--file-name-mode', type=click.Choice(['original', 'geoseeq', 'sample-uuid', 'file-uuid']), help="Choose how the downloaded fastq files are named.", default='original')
+@click.option("--which-fastqs-mode", type=click.Choice(["first-all", "first-r1", "all"]), default="all", help="Choose which fastq files to download per sample. ")
 @click.option("--download/--urls-only", default=True, help="Download files or just print urls")
+@head_option
 @click.option("--config-dir", default=None, help="Directory to write read config files. If unset do not write config files.")
 @module_option(FASTQ_MODULE_NAMES, use_default=False)
 @ignore_errors_option
@@ -460,8 +519,10 @@ def cli_download_fastqs(state,
                         cores,
                         target_dir,
                         yes,
-                        first,
+                        file_name_mode,
+                        which_fastqs_mode,
                         download,
+                        head,
                         config_dir,
                         module_name,
                         ignore_errors,
@@ -474,6 +535,20 @@ def cli_download_fastqs(state,
     This command will download fastq files from a GeoSeeq project. You can filter
     files by sample name and by specific fastq read types.
 
+    The filenames of the downloaded fastq files can be controlled using the --file-name-mode option:
+    - original: Use the original filename as uploaded to GeoSeeq (default)
+    - geoseeq: Use a normalized GeoSeeq generated filename that includes the sample name, read type, read number, and lane number.
+    - sample-uuid: Use the GeoSeeq UUID of the sample along with lane number and read number.
+    - file-uuid: Use the GeoSeeq UUID of the result file only.
+
+    If the --head option is used to only download the first N bytes of each fastq file, this command
+    will automatically clip the fastq files at the nearest complete read boundary to avoid incomplete reads.
+
+    The --which-fastqs-mode option controls which fastq files are downloaded per sample:
+    - first-all: Download all fastq files but from the first fastq folder only.
+    - first-r1: Download only the first read (R1) fastq file from the first fastq folder.
+    - all: Download all fastq files from all folders.
+
     ---
 
     Example Usage:
@@ -523,7 +598,7 @@ def cli_download_fastqs(state,
     result_files_with_names = []
     for sample in samples:
         try:
-            result_files_with_names += _get_sample_result_files_with_names(sample, module_name, first)
+            result_files_with_names += _get_sample_result_files_with_names(sample, module_name, which_fastqs_mode, file_name_mode)
         except Exception as e:
             logger.error(f"Error fetching fastq files for sample {sample.name}: {e}")
             if not ignore_errors:
@@ -538,9 +613,14 @@ def cli_download_fastqs(state,
         ignore_errors=ignore_errors,
         log_level=state.log_level,
         progress_tracker_factory=PBarManager().get_new_bar,
+        head=head,
     )
     for result_file, filename, key in result_files_with_names:
-        download_manager.add_download(result_file, join(target_dir, filename), key=key)
+        callback = None
+        if head:
+            callback = _trim_fastq_to_complete_reads
+            key = key + (head,)  # append head bytes to key
+        download_manager.add_download(result_file, join(target_dir, filename), key=key, callback=callback)
     if not download:
         print(download_manager.get_url_string(), file=state.outfile)
     else:

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/geoseeq/cli/main.py

@@ -55,7 +55,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo("0.7.3dev0")  # remember to update pyproject.toml
+    click.echo("0.7.5")  # remember to update pyproject.toml
 
 
 @main.group("advanced")

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/geoseeq/cli/upload/upload_reads.py

@@ -1,9 +1,10 @@
+# pylint: disable=line-too-long
 import logging
 import click
 import requests
 from os.path import basename
 import pandas as pd
-from multiprocessing import Pool, current_process
+from multiprocessing import current_process
 
 from geoseeq.cli.constants import *
 from geoseeq.cli.shared_params import (
@@ -46,10 +47,10 @@ def _upload_one_file(args):
 
 def _get_regex(knex, filepaths, module_name, lib, regex):
     """Return a regex that will group the files into samples
-    
+
     Tell the user how many files did could not be matched using the regex.
     """
-    seq_length, seq_type = module_name.split('::')[:2]
+    _, seq_type = module_name.split('::')[:2]
     args = {
         'filenames': list(filepaths.keys()),
         'sequence_type': seq_type,
@@ -181,7 +182,13 @@ def flatten_list_of_bams(filepaths):
 @private_option
 @link_option
 @no_new_versions_option
-@click.option('--name-map', default=None, nargs=3, help="A file to use for converting names. Takes three arguments: a file name, a column name for current names, and a column name for new names.")
+@click.option(
+    '--name-map',
+    default=None,
+    nargs=3,
+    required=False,
+    help='Optional CSV and column names used to map existing names to new ones. Provide: <file> <current_name_col> <new_name_col>.'
+)
 @module_option(FASTQ_MODULE_NAMES)
 @project_id_arg
 @click.argument('fastq_files', type=click.Path(exists=True), nargs=-1)
@@ -214,15 +221,23 @@ def cli_upload_reads_wizard(state, cores, overwrite, yes, regex, private, link_t
     $ ls -1 path/to/fastq/files/*.fastq.gz > file_list.txt
     $ geoseeq upload reads --yes --overwrite "GeoSeeq/Example CLI Project" file_list.txt
 
+    \b
+    # Remap sample names using a CSV file with current and new names
+    $ geoseeq upload reads --name-map sample_map.csv current_name new_name "GeoSeeq/Example CLI Project" fastq_files.txt
+
     ---
 
+    The optional ``--name-map`` flag takes three values: a CSV filename,
+    the column containing current names and the column containing new names.
+    When provided, sample names will be translated during upload.
+
     Command Arguments:
-    
+
     [PROJECT_ID] Can be a project UUID, GeoSeeq Resource Number (GRN), or an
     organization name and project name separated by a slash.
 
     \b
-    Examples: 
+    Examples:
      - Name pair: "GeoSeeq/Example CLI Project"
      - UUID: "ed59b913-91ec-489b-a1b9-4ea137a6e5cf"
      - GRN: "grn:gs1:project:ed59b913-91ec-489b-a1b9-4ea137a6e5cf"
@@ -284,12 +299,12 @@ def cli_upload_reads_wizard(state, cores, overwrite, yes, regex, private, link_t
     ---
 
     Command Arguments:
-    
+
     [PROJECT_ID] Can be a project UUID, GeoSeeq Resource Number (GRN), or an
     organization name and project name separated by a slash.
 
     \b
-    Examples: 
+    Examples:
      - Name pair: "GeoSeeq/Example CLI Project"
      - UUID: "ed59b913-91ec-489b-a1b9-4ea137a6e5cf"
      - GRN: "grn:gs1:project:ed59b913-91ec-489b-a1b9-4ea137a6e5cf"
@@ -303,4 +318,14 @@ def cli_upload_reads_wizard(state, cores, overwrite, yes, regex, private, link_t
     # filepaths = {basename(line): line for line in flatten_list_of_bams(files)}
     # click.echo(f'Found {len(filepaths)} files to upload.', err=True)
     # groups = _group_files(knex, filepaths, 'bam::bam', regex, yes)
-    # _do_upload(groups, 'bam::bam', link_type, proj, filepaths, overwrite, no_new_versions, cores, state)
+    # _do_upload(
+    #     groups,
+    #     'bam::bam',
+    #     link_type,
+    #     proj,
+    #     filepaths,
+    #     overwrite,
+    #     no_new_versions,
+    #     cores,
+    #     state,
+    # )

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/geoseeq/constants.py

@@ -7,6 +7,7 @@ FASTQ_MODULE_NAMES = [
     'short_read::paired_end',
     'short_read::single_end',
     'long_read::nanopore',
+    'long_read::pacbio',
     'raw::raw_reads',
     'genome::fasta',
 ]
@@ -16,4 +17,12 @@ CONFIG_FOLDER = environ.get("XDG_CONFIG_HOME", join(environ["HOME"], ".config"))
 CONFIG_DIR = environ.get("GEOSEEQ_CONFIG_DIR", join(CONFIG_FOLDER, "geoseeq"))
 PROFILES_PATH = join(CONFIG_DIR, "profiles.json")
 
-OBJECT_TYPE_STR = Literal['org', 'project', 'sample', 'sample_result_folder', 'project_result_folder', 'sample_result_file', 'project_result_file']
+OBJECT_TYPE_STR = Literal[
+    'org',
+    'project',
+    'sample',
+    'sample_result_folder',
+    'project_result_folder',
+    'sample_result_file',
+    'project_result_file',
+]

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/geoseeq/result/result_file.py

@@ -103,6 +103,18 @@ class ResultFile(RemoteObject, ResultFileUpload, ResultFileDownload, ResultFileS
         # except TypeError:
         #     return basename(self.get_blob_filename())
 
+    def get_stored_data_filename(self):
+        """Return the filename that is stored in the stored_data field.
+        
+        This is typically the filename that was originally uploaded to create this result file.
+        """
+        try:
+            key = [k for k in ["filename", "uri", "url"] if k in self.stored_data][0]
+        except IndexError:
+            raise TypeError("Cannot make a reference filename for a BLOB type result field.")
+        filepath = self.stored_data[key]
+        return basename(filepath)
+
     def _save(self):
         data = {field: getattr(self, field) for field in self.remote_fields if hasattr(self, field)}
         data["analysis_result"] = self.parent.uuid

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/geoseeq/sample.py

@@ -1,6 +1,6 @@
 import urllib
 from .remote_object import RemoteObject
-from .result import SampleResultFile, SampleResultFolder
+from .result import SampleResultFolder
 
 
 class Sample(RemoteObject):
@@ -166,12 +166,14 @@ class Sample(RemoteObject):
             "short_read::paired_end"
             "short_read::single_end"
             "long_read::nanopore"
+            "long_read::pacbio"
         """
         if preference_order is None:
             preference_order = [
                 "short_read::paired_end",
                 "short_read::single_end",
                 "long_read::nanopore",
+                "long_read::pacbio",
             ]
         all_fastqs = self.get_all_fastqs()
         for read_type in preference_order:

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "geoseeq"
-version = "0.7.3dev4"
+version = "0.7.5"
 authors = [
   { name="David C. Danko", email="dcdanko@biotia.io" },
 ]

{geoseeq-0.7.3.dev4 → geoseeq-0.7.5}/setup.py

@@ -5,7 +5,8 @@ import setuptools
 
 setuptools.setup(
     name='geoseeq',
-    version='0.5.6a7',  # DEPRECATED see pyproject.toml remember to update version string in CLI as well
+    version='0.7.4',  # DEPRECATED see pyproject.toml
+    # remember to update version string in CLI as well
     author="David C. Danko",
     author_email='dcdanko@biotia.io',
     description=open('README.md').read(),