fastQpick 0.1.0__tar.gz

fastqpick-0.1.0/LICENSE

@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2024, Pachter Lab
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

fastqpick-0.1.0/PKG-INFO

@@ -0,0 +1,197 @@
+Metadata-Version: 2.2
+Name: fastQpick
+Version: 0.1.0
+Summary: Fast and memory-efficient sampling of DNA-Seq or RNA-seq fastq data with or without replacement.
+Author-email: Joseph Rich <josephrich98@gmail.com>
+Maintainer-email: Joseph Rich <josephrich98@gmail.com>
+License: BSD 2-Clause License
+        
+        Copyright (c) 2024, Pachter Lab
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        1. Redistributions of source code must retain the above copyright notice, this
+           list of conditions and the following disclaimer.
+        
+        2. Redistributions in binary form must reproduce the above copyright notice,
+           this list of conditions and the following disclaimer in the documentation
+           and/or other materials provided with the distribution.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Project-URL: Homepage, https://github.com/pachterlab/fastQpick
+Keywords: fastQpick,bioinformatics,statistics,RNA-seq,DNA-seq
+Classifier: Environment :: Console
+Classifier: Framework :: Jupyter
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Classifier: Topic :: Utilities
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyfastx>=2.0.0
+Requires-Dist: tqdm>=4.66.0
+
+# fastQpick
+
+Fast and memory-efficient sampling of DNA-seq or RNA-seq FASTQ data with or without replacement.
+
+---
+
+## Installation
+
+### Install via PyPI
+```bash
+pip install fastQpick
+```
+
+### Install from Source Code
+
+Using pip:
+```bash
+pip install git+https://github.com/pachterlab/fastQpick.git
+```
+
+Or clone the repository and build manually:
+```bash
+git clone https://github.com/pachterlab/fastQpick.git
+cd fastQpick
+python -m build
+python -m pip install dist/fastQpick-x.x.x-py3-none-any.whl
+```
+
+---
+
+## Usage
+
+### Command-line Interface
+
+Run `fastQpick` with a specified fraction and options:
+```bash
+fastQpick --fraction FRACTION [OPTIONS] FASTQ_FILE1 FASTQ_FILE2 ...
+```
+
+### Python API
+
+Use `fastQpick` in your Python code:
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_file_list=['FASTQ_FILE1', 'FASTQ_FILE2', ...],
+    fraction=FRACTION,
+    ...
+)
+```
+
+---
+
+## Documentation
+
+- **Command-line Help**: Use the following command to see all available options:
+  ```bash
+  fastQpick --help
+  ```
+
+- **Python API Help**: Use the `help` function to explore the API:
+  ```python
+  help(fastQpick)
+  ```
+
+### Options
+- input_files (str, list, or tuple)        List of input FASTQ files or directories containing FASTQ files. Required. Positional argument on command line.
+-  fraction (int or float)                 The fraction of reads to sample, as a float greater than 0. Any value equal to or greater than 1 will turn on the -r flag automatically.
+-  seed (int or str)                       Random seed(s). Can provide multiple seeds separated by commas. Default: 42
+-  output_dir (str)                        Output directory. Default: ./fastQpick_output
+-  gzip_output (bool)                      Whether or not to gzip the output. Default: False (uncompressed)
+-  group_size (int)                        The size of grouped files. Provide each pair of files sequentially, separated by a space. E.g., I1, R1, R2 -  would have group_size=3. Default: 1 (unpaired)
+-  replacement (bool)                      Sample with replacement. Default: False (without replacement).
+-  overwrite (bool)                        Overwrite existing output files. Default: False
+-  low_memory (bool)                       Whether to use low memory mode (uses ~5.5x less memory than default, but adds marginal time to the data -  structure generation preprocessing). Default: False
+-  verbose (bool)                          Whether to print progress information. Default: True
+
+---
+
+## Features
+
+- Efficient sampling of large FASTQ files.
+- Works with both single and paired-end sequencing data.
+- Supports sampling with or without replacement.
+- Command-line interface and Python API for seamless integration.
+- Memory efficient - in low-memory mode, only uses as much memory as a list of (small) integers the length of the number of reads in the fastq file for each file.
+- Time efficient - only passes through the fastq once and writes to output in batches - can process 600M reads in 10-15 minutes
+
+## Low memory mode vs. standard
+Low memory mode vs. standard, when fraction=1 (i.e., number of reads to sample is the same as the number of reads in the fastq):
+- Adds an extra ~1-3 seconds per million reads per group_size (i.e., 500M reads would take 30 minutes instead of 20-25 minutes)
+- Saves an extra ~40MB RAM per million reads (i.e., 500M reads would take 3.75GB RAM vs 20.6GB RAM)
+
+---
+
+## Examples
+
+### 1. Sample 10% of reads with replacement from a FASTQ file:
+
+**Command-line**
+```bash
+fastQpick --fraction 0.1 -r input.fastq
+```
+
+**Python**
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_files='input.fastq',
+    fraction=0.1,
+    replacement=True
+)
+```
+
+### 2. Sample 100% of reads with replacement from multiple paired FASTQ files (R1, R2) across three seeds (i.e., bootstrapping):
+
+**Command-line**
+```bash
+fastQpick --fraction 1 -s 42,43,44 -r -g 2 input1_R1.fastq input1_R2.fastq
+```
+
+**Python**
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_files='input.fastq',
+    fraction=1,
+    seed="42,43,44",
+    replacement=True,
+    group_size=2,
+)
+```
+---
+
+## License
+
+fastQpick is licensed under the 2-clause BSD license. See the [LICENSE](LICENSE) file for details.
+
+---
+
+## Contributing
+
+We welcome contributions! Please see the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines on how to get involved.
+

fastqpick-0.1.0/README.md

@@ -0,0 +1,147 @@
+# fastQpick
+
+Fast and memory-efficient sampling of DNA-seq or RNA-seq FASTQ data with or without replacement.
+
+---
+
+## Installation
+
+### Install via PyPI
+```bash
+pip install fastQpick
+```
+
+### Install from Source Code
+
+Using pip:
+```bash
+pip install git+https://github.com/pachterlab/fastQpick.git
+```
+
+Or clone the repository and build manually:
+```bash
+git clone https://github.com/pachterlab/fastQpick.git
+cd fastQpick
+python -m build
+python -m pip install dist/fastQpick-x.x.x-py3-none-any.whl
+```
+
+---
+
+## Usage
+
+### Command-line Interface
+
+Run `fastQpick` with a specified fraction and options:
+```bash
+fastQpick --fraction FRACTION [OPTIONS] FASTQ_FILE1 FASTQ_FILE2 ...
+```
+
+### Python API
+
+Use `fastQpick` in your Python code:
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_file_list=['FASTQ_FILE1', 'FASTQ_FILE2', ...],
+    fraction=FRACTION,
+    ...
+)
+```
+
+---
+
+## Documentation
+
+- **Command-line Help**: Use the following command to see all available options:
+  ```bash
+  fastQpick --help
+  ```
+
+- **Python API Help**: Use the `help` function to explore the API:
+  ```python
+  help(fastQpick)
+  ```
+
+### Options
+- input_files (str, list, or tuple)        List of input FASTQ files or directories containing FASTQ files. Required. Positional argument on command line.
+-  fraction (int or float)                 The fraction of reads to sample, as a float greater than 0. Any value equal to or greater than 1 will turn on the -r flag automatically.
+-  seed (int or str)                       Random seed(s). Can provide multiple seeds separated by commas. Default: 42
+-  output_dir (str)                        Output directory. Default: ./fastQpick_output
+-  gzip_output (bool)                      Whether or not to gzip the output. Default: False (uncompressed)
+-  group_size (int)                        The size of grouped files. Provide each pair of files sequentially, separated by a space. E.g., I1, R1, R2 -  would have group_size=3. Default: 1 (unpaired)
+-  replacement (bool)                      Sample with replacement. Default: False (without replacement).
+-  overwrite (bool)                        Overwrite existing output files. Default: False
+-  low_memory (bool)                       Whether to use low memory mode (uses ~5.5x less memory than default, but adds marginal time to the data -  structure generation preprocessing). Default: False
+-  verbose (bool)                          Whether to print progress information. Default: True
+
+---
+
+## Features
+
+- Efficient sampling of large FASTQ files.
+- Works with both single and paired-end sequencing data.
+- Supports sampling with or without replacement.
+- Command-line interface and Python API for seamless integration.
+- Memory efficient - in low-memory mode, only uses as much memory as a list of (small) integers the length of the number of reads in the fastq file for each file.
+- Time efficient - only passes through the fastq once and writes to output in batches - can process 600M reads in 10-15 minutes
+
+## Low memory mode vs. standard
+Low memory mode vs. standard, when fraction=1 (i.e., number of reads to sample is the same as the number of reads in the fastq):
+- Adds an extra ~1-3 seconds per million reads per group_size (i.e., 500M reads would take 30 minutes instead of 20-25 minutes)
+- Saves an extra ~40MB RAM per million reads (i.e., 500M reads would take 3.75GB RAM vs 20.6GB RAM)
+
+---
+
+## Examples
+
+### 1. Sample 10% of reads with replacement from a FASTQ file:
+
+**Command-line**
+```bash
+fastQpick --fraction 0.1 -r input.fastq
+```
+
+**Python**
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_files='input.fastq',
+    fraction=0.1,
+    replacement=True
+)
+```
+
+### 2. Sample 100% of reads with replacement from multiple paired FASTQ files (R1, R2) across three seeds (i.e., bootstrapping):
+
+**Command-line**
+```bash
+fastQpick --fraction 1 -s 42,43,44 -r -g 2 input1_R1.fastq input1_R2.fastq
+```
+
+**Python**
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_files='input.fastq',
+    fraction=1,
+    seed="42,43,44",
+    replacement=True,
+    group_size=2,
+)
+```
+---
+
+## License
+
+fastQpick is licensed under the 2-clause BSD license. See the [LICENSE](LICENSE) file for details.
+
+---
+
+## Contributing
+
+We welcome contributions! Please see the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines on how to get involved.
+

fastqpick-0.1.0/fastQpick/__init__.py

@@ -0,0 +1,5 @@
+from .main import fastQpick
+
+from ._version import __version__
+__author__ = "Joseph Rich"
+__email__ = "josephrich98@gmail.com"

fastqpick-0.1.0/fastQpick/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

fastqpick-0.1.0/fastQpick/main.py

@@ -0,0 +1,241 @@
+import argparse
+import gzip
+import os
+import random
+from tqdm import tqdm
+import pyfastx  # to loop through fastq (faster than custom python code)
+
+from fastQpick._version import __version__
+from fastQpick.utils import save_params_to_config_file, is_directory_effectively_empty, group_items, count_reads
+
+# Global variables
+valid_fastq_extensions = (".fastq", ".fq", ".fastq.gz", ".fq.gz")
+batch_size = 200000  # for buffer
+fastq_to_length_dict = {}  # set to empty, and the user can provide otherwise it will be calculated
+
+def write_fastq(input_fastq, output_path, occurrence_list, total_reads, gzip_output, seed = None, verbose = True):
+    if gzip_output:
+        open_func = gzip.open
+        write_mode = "wt"
+    else:
+        open_func = open
+        write_mode = "w"
+    
+    buffer = []  # Temporary storage for the batch
+
+    input_fastq_read_only = pyfastx.Fastx(input_fastq)
+
+    # use tqdm if verbose else silently loop
+    iterator = (
+        tqdm(input_fastq_read_only, desc=f"Iterating through seed {seed}, file {input_fastq}", unit="read", total=total_reads)
+        if verbose else input_fastq_read_only
+    )
+
+    with open_func(output_path, write_mode) as f:
+        for i, (name, seq, qual) in enumerate(iterator):
+            # Add the FASTQ entry to the buffer
+            buffer.extend([f"@{name}\n{seq}\n+\n{qual}\n"] * occurrence_list[i])
+            
+            # If the buffer reaches the batch size, write all at once and clear the buffer
+            if (i + 1) % batch_size == 0:
+                f.writelines(buffer)
+                buffer.clear()  # Clear the buffer after writing
+        
+        # Write any remaining entries in the buffer
+        if buffer:
+            f.writelines(buffer)
+            buffer.clear()
+
+def make_occurrence_list(file, seed, total_reads, number_of_reads_to_sample, replacement, low_memory, verbose):
+    if verbose:
+        print(f"Calculating total reads and determining random indices for seed {seed}, file {file}")
+    if replacement:
+        if low_memory:
+            random_indices = (random.choice(range(total_reads)) for _ in range(number_of_reads_to_sample))
+        else:
+            random_indices = tuple(random.choices(range(total_reads), k=number_of_reads_to_sample))  # with replacement
+    else:
+        if low_memory:
+            random_indices = (index for index in random.sample(range(total_reads), k=number_of_reads_to_sample))
+        else:
+            random_indices = tuple(random.sample(range(total_reads), k=number_of_reads_to_sample))  # without replacement
+
+    # Initialize a list with zeros
+    occurrence_list = [0] * total_reads
+
+    # use tqdm if verbose, else just silently loop through
+    iterator = (
+        tqdm(random_indices, desc=f"Counting occurrences for seed {seed}, file {file}", unit="read", total=number_of_reads_to_sample)
+        if verbose else random_indices
+    )
+
+    # Count occurrences (I don't use a counter in order to save memory, as a counter is essentially a dictionary)
+    for index in iterator:
+        occurrence_list[index] += 1
+
+    del random_indices
+
+    return occurrence_list
+
+def bootstrap_single_file(files_total = None, gzip_output = None, output_directory = None, seed = None, fraction = None, replacement = None, low_memory = False, verbose=True):
+    if isinstance(files_total, str):
+        files_total = (files_total, )
+
+    total_reads = fastq_to_length_dict[files_total[0]]
+    number_of_reads_to_sample = int(fraction * total_reads)
+
+    occurrence_list = make_occurrence_list(file=files_total[0], seed=seed, total_reads=total_reads, number_of_reads_to_sample=number_of_reads_to_sample, replacement=replacement, low_memory=low_memory, verbose=verbose)
+    
+    for file in files_total:
+        # Create output directory if it doesn't exist
+        output_path = os.path.join(output_directory, os.path.basename(file))
+        if output_directory:
+            os.makedirs(output_directory, exist_ok=True)
+
+        if gzip_output and not output_path.endswith(".gz"):
+            output_path += ".gz"
+        elif not gzip_output and output_path.endswith(".gz"):
+            output_path = output_path[:-3]
+
+        # write fastq
+        write_fastq(input_fastq = file, output_path = output_path, occurrence_list = occurrence_list, total_reads = total_reads, gzip_output = gzip_output, seed = seed, verbose = verbose)
+
+def sample_multiple_files(file_list, fraction, seed_list, output, gzip_output, replacement, low_memory, verbose):
+    for seed in seed_list:
+        random.seed(seed)
+        for file in file_list:
+            bootstrap_single_file(files_total = file, gzip_output = gzip_output, output_directory = output, seed = seed, fraction = fraction, replacement = replacement, low_memory = low_memory, verbose = verbose)
+    
+def make_fastq_to_length_dict(file_list, verbose=True):
+    global fastq_to_length_dict
+    for file in file_list:
+        if isinstance(file, tuple):
+            if all(specific_file in fastq_to_length_dict for specific_file in file):
+                continue
+            if verbose:
+                print(f"Counting {file[0]}")
+            count = count_reads(file[0])
+            for i in range(len(file)):
+                fastq_to_length_dict[file[i]] = count
+        elif isinstance(file, str):
+            if file in fastq_to_length_dict:
+                continue
+            if verbose:
+                print(f"Counting {file}")
+            count = count_reads(file)
+            fastq_to_length_dict[file] = count
+    if verbose:
+        print("fastq_to_length_dict:", fastq_to_length_dict)
+
+def fastQpick(input_files, fraction, seed=42, output_dir="fastQpick_output", gzip_output=False, group_size=1, replacement=False, overwrite=False, low_memory=False, verbose=True, **kwargs):
+    """
+    Fast and memory-efficient sampling of DNA-Seq or RNA-seq fastq data with or without replacement.
+
+    Parameters
+    ----------
+    input_files (str, list, or tuple)       List of input FASTQ files or directories containing FASTQ files.
+    fraction (int or float)                 The fraction of reads to sample, as a float greater than 0. Any value equal to or greater than 1 will turn on the -r flag automatically.
+    seed (int or str)                       Random seed(s). Can provide multiple seeds separated by commas. Default: 42
+    output_dir (str)                        Output directory. Default: ./fastQpick_output
+    gzip_output (bool)                      Whether or not to gzip the output. Default: False (uncompressed)
+    group_size (int)                        The size of grouped files. Provide each pair of files sequentially, separated by a space. E.g., I1, R1, R2 would have group_size=3. Default: 1 (unpaired)
+    replacement (bool)                      Sample with replacement. Default: False (without replacement).
+    overwrite (bool)                        Overwrite existing output files. Default: False
+    low_memory (bool)                       Whether to use low memory mode (uses ~5.5x less memory than default, but adds marginal time to the data structure generation preprocessing). Default: False
+    verbose (bool)                          Whether to print progress information. Default: True
+
+    kwargs
+    ------
+    fastq_to_length_dict (dict)             Dictionary of FASTQ file paths to number of reads in each file. If not provided, will be calculated.
+    """
+    # check if fastq_to_length_dict is in kwargs
+    if "fastq_to_length_dict" in kwargs and isinstance(kwargs["fastq_to_length_dict"], dict):
+        global fastq_to_length_dict
+        fastq_to_length_dict = kwargs["fastq_to_length_dict"]
+
+    # Check overwrite
+    if not overwrite:
+        if os.path.exists(output_dir) and not is_directory_effectively_empty(output_dir):  # check if dir exists and is not empty
+            raise FileExistsError(f"Output directory '{output_dir}' already exists. Please specify a different output directory or set the overwrite flag to True.")
+
+    # Save arguments to a config file
+    os.makedirs(output_dir, exist_ok=True)
+    config_file = os.path.join(output_dir, "fastQpick_config.json")
+    save_params_to_config_file(config_file)
+
+    # type checking
+    # if fraction >= 1, set replacement to True
+    if float(fraction) >= 1.0:
+        replacement = True
+
+    # go through files, and only keep those that are valid fastq files or that are a folder containing valid fastq files in the direct subdirectory
+    input_files_parsed = []
+    if isinstance(input_files, str):
+        input_files_parsed = [input_files]
+    elif isinstance(input_files, tuple) or isinstance(input_files, list):
+        for path in input_files:
+            if not isinstance(path, str):
+                raise ValueError("Input file list must be a string, tuple of strings, or list of strings.")
+            if not os.path.exists(path):
+                raise FileNotFoundError(f"File or directory '{path}' not found.")
+            elif os.path.isfile(path) and not path.endswith(tuple(valid_fastq_extensions)):
+                raise ValueError(f"File '{path}' is not a valid FASTQ file.")
+            elif os.path.isdir(path):
+                input_files_before_path = input_files_parsed.copy()
+                for subpath in os.listdir(path):
+                    if os.path.isfile(subpath) and subpath.endswith(tuple(valid_fastq_extensions)):
+                        input_files_parsed.append(subpath)
+                if input_files_before_path == input_files_parsed:
+                    raise ValueError(f"No valid FASTQ files found in directory '{path}'.")
+            elif os.path.isfile(path) and path.endswith(tuple(valid_fastq_extensions)):
+                input_files_parsed.append(path)
+    else:
+        raise ValueError("Input file list must be a string, tuple of strings, or list of strings.")
+
+    if isinstance(seed, int):  # if a single int is passed as a seed
+        seed = [seed]
+    elif isinstance(seed, str):  # if a string of comma-separated ints is passed as a seed (like on the command line)
+        seed = [int(specific_seed) for specific_seed in seed.split(",")]
+
+    group_size = int(group_size)  # make sure group_size is an int (not a string)
+    fraction = float(fraction)  # make sure fraction is a float (not a string)
+
+    if group_size > 1:
+        input_files_parsed = group_items(input_files_parsed, group_size=group_size)
+    
+    # Count reads in each file and store in a dictionary
+    make_fastq_to_length_dict(input_files_parsed, verbose=verbose)
+
+    # Do the sampling
+    sample_multiple_files(file_list=input_files_parsed, fraction=fraction, seed_list=seed, output=output_dir, gzip_output=gzip_output, replacement=replacement, low_memory=low_memory, verbose=verbose)
+
+def main():
+    # Create argument parser
+    parser = argparse.ArgumentParser(description="Fast and memory-efficient sampling of DNA-Seq or RNA-seq fastq data with or without replacement.")
+    parser.add_argument("-f", "--fraction", required=True, default=False, help="The fraction of reads to sample, as a float greater than 0. Any value equal to or greater than 1 will turn on the -r flag automatically.")
+    parser.add_argument("-s", "--seed", required=False, default=42, help="Random seed(s). Can provide multiple seeds separated by commas. Default: 42")
+    parser.add_argument("-o", "--output_dir", required=False, type=str, default="fastQpick_output", help="Output file path. Default: ./fastQpick_output")
+    parser.add_argument("-z", "--gzip_output", required=False, default=False, help="Whether or not to gzip the output. Default: False (uncompressed)")
+    parser.add_argument("-g", "--group_size", required=False, default=1, help="The size of grouped files. Provide each pair of files sequentially, separated by a space. E.g., I1, R1, R2 would have group_size=3. Default: 1 (unpaired)")
+    parser.add_argument("-r", "--replacement", action="store_true", help="Sample with replacement. Default: False (without replacement).")
+    parser.add_argument("-w", "--overwrite", action="store_true", help="Overwrite existing output files. Default: False")
+    parser.add_argument("-l", "--low_memory", action="store_true", help="Whether to use low memory mode (uses ~5.5x less memory than default, but adds marginal time to the data structure generation preprocessing). Default: False")
+    parser.add_argument("-q", "--quiet", action="store_false", help="Turn off verbose output. Default: False")
+    parser.add_argument("-v", "--version", action="version", version=f"fastQpick {__version__}", help="Show program's version number and exit")
+
+    # Positional argument for input files (indefinite number)
+    parser.add_argument("input_files", nargs="+", help="Input FASTQ file(s) (one after the other, space-separated) or FASTQ folder(s)")
+
+    # Parse arguments
+    args = parser.parse_args()
+            
+    fastQpick(input_files=args.input_files,
+              fraction=args.fraction,
+              seed=args.seed,
+              output_dir=args.output_dir,
+              gzip_output=args.gzip_output,
+              group_size=args.group_size,
+              replacement=args.replacement,
+              overwrite=args.overwrite,
+              low_memory=args.low_memory,
+              verbose=args.quiet)

fastqpick-0.1.0/fastQpick/utils.py

@@ -0,0 +1,102 @@
+import gzip
+import inspect
+import json
+import os
+from collections import OrderedDict
+import pyfastx
+
+def count_reads(filepath):
+    fastq_file = pyfastx.Fastx(filepath)
+    num_reads = sum(1 for _ in fastq_file)
+    return num_reads
+
+def read_fastq(fastq_file, include_plus_line=False):
+    is_gzipped = fastq_file.endswith(".gz")
+    open_func = gzip.open if is_gzipped else open
+    open_mode = "rt" if is_gzipped else "r"
+
+    try:
+        if include_plus_line:
+            with open_func(fastq_file, open_mode) as file:
+                while True:
+                    header = file.readline().strip()
+                    sequence = file.readline().strip()
+                    plus_line = file.readline().strip()
+                    quality = file.readline().strip()
+
+                    if not header:
+                        break
+
+                    yield header, sequence, plus_line, quality
+        else:  # copy-paste the code so that it doesn't have to check the conditional every iteration
+            with open_func(fastq_file, open_mode) as file:
+                while True:
+                    header = file.readline().strip()
+                    sequence = file.readline().strip()
+                    plus_line = file.readline().strip()
+                    quality = file.readline().strip()
+
+                    if not header:
+                        break
+
+                    yield header, sequence, quality
+    except Exception as e:
+        raise RuntimeError(f"Error reading FASTQ file '{fastq_file}': {e}")
+
+def make_function_parameter_to_value_dict(levels_up = 1):
+    # Collect parameters in a dictionary
+    params = OrderedDict()
+
+    # Get the caller's frame (one level up in the stack)
+    frame = inspect.currentframe()
+
+    for _ in range(levels_up):
+        if frame is None:
+            break
+        frame = frame.f_back
+
+    function_args, varargs, varkw, values = inspect.getargvalues(frame)
+
+    # handle explicit function arguments
+    for arg in function_args:
+        params[arg] = values[arg]
+    
+    # handle *args
+    if varargs:
+        params["*args"] = values[varargs]
+    
+    # handle **kwargs
+    if varkw:
+        for key, value in values[varkw].items():
+            params[key] = value
+    
+    return params
+
+
+def save_params_to_config_file(out_file="run_config.json"):
+    out_file_directory = os.path.dirname(out_file)
+    if not out_file_directory:
+        out_file_directory = "."
+    else:
+        os.makedirs(out_file_directory, exist_ok=True)
+
+    # Collect parameters in a dictionary
+    params = make_function_parameter_to_value_dict(levels_up = 2)
+
+    # Write to JSON
+    with open(out_file, "w") as file:
+        json.dump(params, file, indent=4)
+
+
+def is_directory_effectively_empty(directory_path):
+    # Get all non-hidden entries, excluding system files like `.DS_Store`
+    entries = [
+        entry for entry in os.listdir(directory_path)
+        if entry not in {".DS_Store"} and not entry.startswith(".")
+    ]
+    return len(entries) == 0
+
+def group_items(file_list, group_size=2):
+    if len(file_list) % group_size != 0:
+        raise ValueError(f"The list length must be divisible by {group_size} to form groups.")
+    return [tuple(file_list[i:i + group_size]) for i in range(0, len(file_list), group_size)]

fastqpick-0.1.0/fastQpick.egg-info/PKG-INFO

@@ -0,0 +1,197 @@
+Metadata-Version: 2.2
+Name: fastQpick
+Version: 0.1.0
+Summary: Fast and memory-efficient sampling of DNA-Seq or RNA-seq fastq data with or without replacement.
+Author-email: Joseph Rich <josephrich98@gmail.com>
+Maintainer-email: Joseph Rich <josephrich98@gmail.com>
+License: BSD 2-Clause License
+        
+        Copyright (c) 2024, Pachter Lab
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        1. Redistributions of source code must retain the above copyright notice, this
+           list of conditions and the following disclaimer.
+        
+        2. Redistributions in binary form must reproduce the above copyright notice,
+           this list of conditions and the following disclaimer in the documentation
+           and/or other materials provided with the distribution.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Project-URL: Homepage, https://github.com/pachterlab/fastQpick
+Keywords: fastQpick,bioinformatics,statistics,RNA-seq,DNA-seq
+Classifier: Environment :: Console
+Classifier: Framework :: Jupyter
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Classifier: Topic :: Utilities
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyfastx>=2.0.0
+Requires-Dist: tqdm>=4.66.0
+
+# fastQpick
+
+Fast and memory-efficient sampling of DNA-seq or RNA-seq FASTQ data with or without replacement.
+
+---
+
+## Installation
+
+### Install via PyPI
+```bash
+pip install fastQpick
+```
+
+### Install from Source Code
+
+Using pip:
+```bash
+pip install git+https://github.com/pachterlab/fastQpick.git
+```
+
+Or clone the repository and build manually:
+```bash
+git clone https://github.com/pachterlab/fastQpick.git
+cd fastQpick
+python -m build
+python -m pip install dist/fastQpick-x.x.x-py3-none-any.whl
+```
+
+---
+
+## Usage
+
+### Command-line Interface
+
+Run `fastQpick` with a specified fraction and options:
+```bash
+fastQpick --fraction FRACTION [OPTIONS] FASTQ_FILE1 FASTQ_FILE2 ...
+```
+
+### Python API
+
+Use `fastQpick` in your Python code:
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_file_list=['FASTQ_FILE1', 'FASTQ_FILE2', ...],
+    fraction=FRACTION,
+    ...
+)
+```
+
+---
+
+## Documentation
+
+- **Command-line Help**: Use the following command to see all available options:
+  ```bash
+  fastQpick --help
+  ```
+
+- **Python API Help**: Use the `help` function to explore the API:
+  ```python
+  help(fastQpick)
+  ```
+
+### Options
+- input_files (str, list, or tuple)        List of input FASTQ files or directories containing FASTQ files. Required. Positional argument on command line.
+-  fraction (int or float)                 The fraction of reads to sample, as a float greater than 0. Any value equal to or greater than 1 will turn on the -r flag automatically.
+-  seed (int or str)                       Random seed(s). Can provide multiple seeds separated by commas. Default: 42
+-  output_dir (str)                        Output directory. Default: ./fastQpick_output
+-  gzip_output (bool)                      Whether or not to gzip the output. Default: False (uncompressed)
+-  group_size (int)                        The size of grouped files. Provide each pair of files sequentially, separated by a space. E.g., I1, R1, R2 -  would have group_size=3. Default: 1 (unpaired)
+-  replacement (bool)                      Sample with replacement. Default: False (without replacement).
+-  overwrite (bool)                        Overwrite existing output files. Default: False
+-  low_memory (bool)                       Whether to use low memory mode (uses ~5.5x less memory than default, but adds marginal time to the data -  structure generation preprocessing). Default: False
+-  verbose (bool)                          Whether to print progress information. Default: True
+
+---
+
+## Features
+
+- Efficient sampling of large FASTQ files.
+- Works with both single and paired-end sequencing data.
+- Supports sampling with or without replacement.
+- Command-line interface and Python API for seamless integration.
+- Memory efficient - in low-memory mode, only uses as much memory as a list of (small) integers the length of the number of reads in the fastq file for each file.
+- Time efficient - only passes through the fastq once and writes to output in batches - can process 600M reads in 10-15 minutes
+
+## Low memory mode vs. standard
+Low memory mode vs. standard, when fraction=1 (i.e., number of reads to sample is the same as the number of reads in the fastq):
+- Adds an extra ~1-3 seconds per million reads per group_size (i.e., 500M reads would take 30 minutes instead of 20-25 minutes)
+- Saves an extra ~40MB RAM per million reads (i.e., 500M reads would take 3.75GB RAM vs 20.6GB RAM)
+
+---
+
+## Examples
+
+### 1. Sample 10% of reads with replacement from a FASTQ file:
+
+**Command-line**
+```bash
+fastQpick --fraction 0.1 -r input.fastq
+```
+
+**Python**
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_files='input.fastq',
+    fraction=0.1,
+    replacement=True
+)
+```
+
+### 2. Sample 100% of reads with replacement from multiple paired FASTQ files (R1, R2) across three seeds (i.e., bootstrapping):
+
+**Command-line**
+```bash
+fastQpick --fraction 1 -s 42,43,44 -r -g 2 input1_R1.fastq input1_R2.fastq
+```
+
+**Python**
+```python
+from fastQpick import fastQpick
+
+fastQpick(
+    input_files='input.fastq',
+    fraction=1,
+    seed="42,43,44",
+    replacement=True,
+    group_size=2,
+)
+```
+---
+
+## License
+
+fastQpick is licensed under the 2-clause BSD license. See the [LICENSE](LICENSE) file for details.
+
+---
+
+## Contributing
+
+We welcome contributions! Please see the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines on how to get involved.
+

fastqpick-0.1.0/fastQpick.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+fastQpick/__init__.py
+fastQpick/_version.py
+fastQpick/main.py
+fastQpick/utils.py
+fastQpick.egg-info/PKG-INFO
+fastQpick.egg-info/SOURCES.txt
+fastQpick.egg-info/dependency_links.txt
+fastQpick.egg-info/entry_points.txt
+fastQpick.egg-info/requires.txt
+fastQpick.egg-info/top_level.txt
+tests/test_fastQpick.py

fastqpick-0.1.0/fastQpick.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

fastqpick-0.1.0/fastQpick.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+fastQpick = fastQpick.main:main

fastqpick-0.1.0/fastQpick.egg-info/requires.txt

@@ -0,0 +1,2 @@
+pyfastx>=2.0.0
+tqdm>=4.66.0

fastqpick-0.1.0/fastQpick.egg-info/top_level.txt

@@ -0,0 +1 @@
+fastQpick

fastqpick-0.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "fastQpick"
+version = "0.1.0"
+description = "Fast and memory-efficient sampling of DNA-Seq or RNA-seq fastq data with or without replacement."
+readme = {file = "README.md", content-type = "text/markdown"}
+license = {file = "LICENSE"}
+authors = [
+    {name = "Joseph Rich", email = "josephrich98@gmail.com"}
+]
+maintainers = [
+    {name = "Joseph Rich", email = "josephrich98@gmail.com"}
+]
+requires-python = ">=3.7"
+keywords = ["fastQpick", "bioinformatics", "statistics", "RNA-seq", "DNA-seq"]
+classifiers = [
+    "Environment :: Console",
+    "Framework :: Jupyter",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: BSD License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Bio-Informatics",
+    "Topic :: Utilities"
+]
+dependencies = [
+    "pyfastx>=2.0.0",
+    "tqdm>=4.66.0",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/pachterlab/fastQpick"
+
+[tool.setuptools]
+packages = ["fastQpick"]
+
+[tool.setuptools.package-data]
+fastQpick = ["*.txt", "*.md", "*.csv"]  # Ensure additional files like data or docs are included
+
+[project.scripts]
+fastQpick = "fastQpick.main:main"  # This replaces the console_scripts entry

fastqpick-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

fastqpick-0.1.0/tests/test_fastQpick.py

@@ -0,0 +1,320 @@
+import os
+import tempfile
+import pytest
+from fastQpick import fastQpick
+from fastQpick.utils import read_fastq, count_reads
+from pdb import set_trace as st
+
+@pytest.fixture
+def temp_fastq_file():
+    content = """@Header1
+AAAAAAAAAAAAAAAAAAAAA
++
+IIIIIIIIIIIIIIIIIIIII
+@Header2
+CCCCCCCCCCCCCCCCCCCCC
++
+IIIIIIIIIIIIIIIIIIIII
+@Header3
+GGGGGGGGGGGGGGGGGGGGG
++
+IIIIIIIIIIIIIIIIIIIII
+@Header4
+TTTTTTTTTTTTTTTTTTTTT
++
+IIIIIIIIIIIIIIIIIIIII
+@Header5
+AAAAAAAAAAAAACCCCCCCC
++
+IIIIIIIIIIIIIIIIIIIII
+"""
+    # Create a temporary file
+    with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".fastq") as temp_file:
+        temp_file.write(content)
+        temp_file.seek(0)  # Move to the start of the file
+        yield temp_file.name  # Provide the file path to the test
+
+    # Cleanup after the test
+    os.remove(temp_file.name)
+
+# Fixture to create two temporary FASTQ files
+@pytest.fixture
+def temp_paired_fastq_files():
+    content_1 = """@Header1_1
+AAAAAAAAAAAAAAAAAAAAA
++
+IIIIIIIIIIIIIIIIIIIII
+@Header2_1
+CCCCCCCCCCCCCCCCCCCCC
++
+IIIIIIIIIIIIIIIIIIIII
+@Header3_1
+GGGGGGGGGGGGGGGGGGGGG
++
+IIIIIIIIIIIIIIIIIIIII
+@Header4_1
+TTTTTTTTTTTTTTTTTTTTT
++
+IIIIIIIIIIIIIIIIIIIII
+"""
+    
+    content_2 = """@Header1_2
+AAAAAAAACCCCCCCCCCCCC
++
+IIIIIIIIIIIIIIIIIIIII
+@Header2_2
+AAAAAAAGGGGGGGGGGGGGG
++
+IIIIIIIIIIIIIIIIIIIII
+@Header3_2
+AAAAAAATTTTTTTTTTTTTT
++
+IIIIIIIIIIIIIIIIIIIII
+@Header4_2
+CCCCCCCCAAAAAAAAAAAAA
++
+IIIIIIIIIIIIIIIIIIIII
+"""
+
+    # Create two temporary files
+    with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".fastq") as temp_file1, \
+         tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".fastq") as temp_file2:
+        temp_file1.write(content_1)
+        temp_file2.write(content_2)
+        temp_file1.seek(0)
+        temp_file2.seek(0)
+        yield [temp_file1.name, temp_file2.name]  # Yield the paths of both files
+
+    # Cleanup after the test
+    os.remove(temp_file1.name)
+    os.remove(temp_file2.name)
+
+
+
+
+def is_gzipped(file_path):
+    with open(file_path, "rb") as f:
+        magic_number = f.read(2)
+        return magic_number == b"\x1f\x8b"
+
+def validate_fastq_format(file_path, ground_truth=None):
+    for header, seq, plus_line, qual in read_fastq(file_path, include_plus_line=True):
+        assert header.startswith("@"), f"Header does not start with '@': {header}"
+        assert len(seq) == len(qual), f"Sequence and quality lengths do not match: {seq} {qual}"
+        assert plus_line.startswith("+"), f"Plus line does not start with '+': {plus_line}"
+
+        if ground_truth:
+            assert header in ground_truth, f"Header not found in ground truth: {header}"
+            assert seq == ground_truth[header]["sequence"], f"Sequence mismatch - expected: {seq}; got: {ground_truth[header]['sequence']}"
+            assert plus_line == ground_truth[header]["plus_line"], f"Plus line mismatch - expected: {plus_line}; got: {ground_truth[header]['plus_line']}"
+            assert qual == ground_truth[header]["quality"], f"Quality mismatch - expected: {qual}; got: {ground_truth[header]['quality']}"
+
+def count_number_of_unique_headers(file_path):
+    headers = set()
+    for header, _, _, _ in read_fastq(file_path, include_plus_line=True):
+        headers.add(header)
+    return len(headers)
+
+def make_fastq_dict(file_path):
+    fastq_dict = {}
+    for header, seq, plus_line, qual in read_fastq(file_path, include_plus_line=True):
+        fastq_dict[header] = {}
+        fastq_dict[header]["sequence"] = seq
+        fastq_dict[header]["plus_line"] = plus_line
+        fastq_dict[header]["quality"] = qual
+    return fastq_dict
+
+        
+def check_pairwise_agreement(temp_paired_fastq_files, temp_output_dir, gzip_output):
+    file1_base_name = os.path.basename(temp_paired_fastq_files[0])
+    file2_base_name = os.path.basename(temp_paired_fastq_files[1])
+    
+    output_fastq_file1 = os.path.join(temp_output_dir, file1_base_name)
+    output_fastq_file2 = os.path.join(temp_output_dir, file2_base_name)
+
+    if gzip_output:
+        output_fastq_file1 += ".gz"
+        output_fastq_file2 += ".gz"
+
+    for (header1, seq1, plus_line1, qual1), (header2, seq2, plus_line2, qual2) in zip(
+        read_fastq(output_fastq_file1, include_plus_line=True), 
+        read_fastq(output_fastq_file2, include_plus_line=True)
+    ):
+        # Split headers up to the last underscore
+        split_header1 = header1.rsplit('_', 1)[0]
+        split_header2 = header2.rsplit('_', 1)[0]
+
+        # Assert that the two headers are equal
+        assert split_header1 == split_header2, f"Headers do not match: {split_header1} != {split_header2}"
+
+def run_all_single_file_tests(temp_output_dir, temp_fastq_file, gzip_output, fraction, replacement):
+    # Assert that the output directory exists
+        assert os.path.exists(temp_output_dir), "Output directory does not exist!"
+
+        # Optionally, verify the output files
+        output_files = os.listdir(temp_output_dir)
+        assert len(output_files) > 0, "No output files were created!"
+
+        file_base_name = os.path.basename(temp_fastq_file)
+        output_fastq_file = os.path.join(temp_output_dir, file_base_name)
+
+        if gzip_output:
+            output_fastq_file += ".gz"
+
+        input_fastq_dict = make_fastq_dict(temp_fastq_file)
+        validate_fastq_format(output_fastq_file, ground_truth=input_fastq_dict)
+
+        output_is_gzipped = is_gzipped(output_fastq_file)
+        assert output_is_gzipped == gzip_output, f"Gzipped output - expected: {gzip_output}; got: {output_is_gzipped}"
+
+        num_reads_truth = count_reads(temp_fastq_file)
+        num_reads_output = count_reads(output_fastq_file)
+
+        assert num_reads_output == num_reads_truth * fraction, f"Number of reads mismatch - expected: {num_reads_truth * fraction}; got: {num_reads_output}"
+
+        num_unique_reads = count_number_of_unique_headers(output_fastq_file)
+
+        if not replacement:
+            assert num_unique_reads == num_reads_output, f"Number of unique reads mismatch - expected: {num_reads_output}; got: {num_unique_reads}"
+
+        if replacement and fraction > 1:
+            assert num_unique_reads < num_reads_output, f"Number of unique reads mismatch - expected: less than {num_reads_output}; got: {num_unique_reads}"
+
+def test_single_file(temp_fastq_file):
+    fraction = 0.6
+    seed = 42
+    gzip_output = False
+    group_size = 1
+    replacement = False
+    
+    with tempfile.TemporaryDirectory() as temp_output_dir:
+        fastQpick(input_files=temp_fastq_file,
+                fraction=fraction,
+                seed=seed,
+                output_dir=temp_output_dir,
+                gzip_output=gzip_output,
+                group_size=group_size,
+                replacement=replacement,
+                overwrite=True
+                )
+        
+        run_all_single_file_tests(temp_output_dir=temp_output_dir, temp_fastq_file=temp_fastq_file, gzip_output=gzip_output, fraction=fraction, replacement=replacement)
+
+        # st()
+
+def test_single_file_bootstrapped(temp_fastq_file):
+    fraction = 1
+    seed = 42
+    gzip_output = False
+    group_size = 1
+    replacement = True
+    
+    with tempfile.TemporaryDirectory() as temp_output_dir:
+        fastQpick(input_files=temp_fastq_file,
+                fraction=fraction,
+                seed=seed,
+                output_dir=temp_output_dir,
+                gzip_output=gzip_output,
+                group_size=group_size,
+                replacement=replacement,
+                overwrite=True
+                )
+        
+        run_all_single_file_tests(temp_output_dir=temp_output_dir, temp_fastq_file=temp_fastq_file, gzip_output=gzip_output, fraction=fraction, replacement=replacement)
+
+        # st()
+
+def test_single_file_oversampled(temp_fastq_file):
+    fraction = 3
+    seed = 42
+    gzip_output = False
+    group_size = 1
+    replacement = True
+    
+    with tempfile.TemporaryDirectory() as temp_output_dir:
+        fastQpick(input_files=temp_fastq_file,
+                fraction=fraction,
+                seed=seed,
+                output_dir=temp_output_dir,
+                gzip_output=gzip_output,
+                group_size=group_size,
+                replacement=replacement,
+                overwrite=True
+                )
+        
+        run_all_single_file_tests(temp_output_dir=temp_output_dir, temp_fastq_file=temp_fastq_file, gzip_output=gzip_output, fraction=fraction, replacement=replacement)
+
+        # st()
+        
+def test_single_gzipped(temp_fastq_file):
+    fraction = 0.6
+    seed = 42
+    gzip_output = True
+    group_size = 1
+    replacement = False
+    
+    with tempfile.TemporaryDirectory() as temp_output_dir:
+        fastQpick(input_files=temp_fastq_file,
+                fraction=fraction,
+                seed=seed,
+                output_dir=temp_output_dir,
+                gzip_output=gzip_output,
+                group_size=group_size,
+                replacement=replacement,
+                overwrite=True
+                )
+        
+        run_all_single_file_tests(temp_output_dir=temp_output_dir, temp_fastq_file=temp_fastq_file, gzip_output=gzip_output, fraction=fraction, replacement=replacement)
+
+        # st()
+
+
+def test_paired_files(temp_paired_fastq_files):
+    fraction = 0.75
+    seed = 42
+    gzip_output = False
+    group_size = 2
+    replacement = False
+    
+    with tempfile.TemporaryDirectory() as temp_output_dir:
+        fastQpick(input_files=temp_paired_fastq_files,
+                fraction=fraction,
+                seed=seed,
+                output_dir=temp_output_dir,
+                gzip_output=gzip_output,
+                group_size=group_size,
+                replacement=replacement,
+                overwrite=True
+                )
+        
+        for fastq_file in temp_paired_fastq_files:
+            run_all_single_file_tests(temp_output_dir=temp_output_dir, temp_fastq_file=fastq_file, gzip_output=gzip_output, fraction=fraction, replacement=replacement)
+
+        check_pairwise_agreement(temp_paired_fastq_files=temp_paired_fastq_files, temp_output_dir=temp_output_dir, gzip_output=gzip_output)
+
+        # st()
+
+def test_paired_files_bootstrapped(temp_paired_fastq_files):
+    fraction = 1
+    seed = 42
+    gzip_output = False
+    group_size = 2
+    replacement = True
+    
+    with tempfile.TemporaryDirectory() as temp_output_dir:
+        fastQpick(input_files=temp_paired_fastq_files,
+                fraction=fraction,
+                seed=seed,
+                output_dir=temp_output_dir,
+                gzip_output=gzip_output,
+                group_size=group_size,
+                replacement=replacement,
+                overwrite=True
+                )
+        
+        for fastq_file in temp_paired_fastq_files:
+            run_all_single_file_tests(temp_output_dir=temp_output_dir, temp_fastq_file=fastq_file, gzip_output=gzip_output, fraction=fraction, replacement=replacement)
+
+        check_pairwise_agreement(temp_paired_fastq_files=temp_paired_fastq_files, temp_output_dir=temp_output_dir, gzip_output=gzip_output)
+
+        # st()