nanotimesort 1.0.0__tar.gz

nanotimesort-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Marc-Olivier Duceppe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nanotimesort-1.0.0/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: nanotimesort
+Version: 1.0.0
+Summary: Bin Oxford Nanopore reads by cumulative sequencing time intervals (Guppy and Dorado compatible)
+Author-email: Marc-Olivier Duceppe <duceppemo@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2019 Marc-Olivier Duceppe
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/duceppemo/nanoTimeSort
+Project-URL: Issues, https://github.com/duceppemo/nanoTimeSort/issues
+Project-URL: Wiki, https://github.com/duceppemo/nanoTimeSort/wiki
+Keywords: nanopore,fastq,bioinformatics,minion,dorado,guppy,binning
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <img src="docs/images/logo.svg" alt="nanoTimeSort" width="520">
+</p>
+
+<p align="center">
+  <a href="https://github.com/duceppemo/nanoTimeSort/actions/workflows/ci.yml"><img src="https://github.com/duceppemo/nanoTimeSort/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
+  <a href="https://codecov.io/gh/duceppemo/nanoTimeSort"><img src="https://codecov.io/gh/duceppemo/nanoTimeSort/branch/master/graph/badge.svg" alt="codecov"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
+  <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.8%2B-blue.svg" alt="Python 3.8+"></a>
+  <a href="pyproject.toml"><img src="https://img.shields.io/badge/dependencies-none-brightgreen.svg" alt="No dependencies"></a>
+</p>
+
+Bin Oxford Nanopore reads by **cumulative sequencing time intervals**, using the read start time
+that MinKNOW, Guppy and Dorado embed in every FASTQ header. Useful to answer *"how long did I
+actually need to sequence?"* — for time-to-detection studies, assembly saturation curves, or
+benchmarking real-time pipelines.
+
+```text
+fastq_pass/  ──►  sample_0-1h_125437reads_1103093674bp.fastq.gz
+                  sample_0-2h_225891reads_2087456221bp.fastq.gz   (bins are cumulative)
+                  sample_0-3h_301255reads_2812345678bp.fastq.gz
+```
+
+Compatible with **Guppy/MinKNOW** (`start_time=`) and **Dorado** (`st:Z:`) headers. Fast
+(each read compressed once, ~100× faster than v0.1), low-memory (streaming), parallel, and
+pure standard library.
+
+## Install
+
+```bash
+pip install git+https://github.com/duceppemo/nanoTimeSort.git
+```
+
+Requires Python ≥ 3.8. No other dependencies.
+
+## Usage
+
+```bash
+nanotimesort -f /path/to/fastq_pass/ -o /path/to/output/ -i 1h -p my_sample
+```
+
+📖 **Full documentation** — CLI reference, tutorial with example data, header compatibility,
+design notes and FAQ — is in the [**wiki**](https://github.com/duceppemo/nanoTimeSort/wiki).
+
+## License
+
+[MIT](LICENSE) © Marc-Olivier Duceppe

nanotimesort-1.0.0/README.md

@@ -0,0 +1,47 @@
+<p align="center">
+  <img src="docs/images/logo.svg" alt="nanoTimeSort" width="520">
+</p>
+
+<p align="center">
+  <a href="https://github.com/duceppemo/nanoTimeSort/actions/workflows/ci.yml"><img src="https://github.com/duceppemo/nanoTimeSort/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
+  <a href="https://codecov.io/gh/duceppemo/nanoTimeSort"><img src="https://codecov.io/gh/duceppemo/nanoTimeSort/branch/master/graph/badge.svg" alt="codecov"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
+  <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.8%2B-blue.svg" alt="Python 3.8+"></a>
+  <a href="pyproject.toml"><img src="https://img.shields.io/badge/dependencies-none-brightgreen.svg" alt="No dependencies"></a>
+</p>
+
+Bin Oxford Nanopore reads by **cumulative sequencing time intervals**, using the read start time
+that MinKNOW, Guppy and Dorado embed in every FASTQ header. Useful to answer *"how long did I
+actually need to sequence?"* — for time-to-detection studies, assembly saturation curves, or
+benchmarking real-time pipelines.
+
+```text
+fastq_pass/  ──►  sample_0-1h_125437reads_1103093674bp.fastq.gz
+                  sample_0-2h_225891reads_2087456221bp.fastq.gz   (bins are cumulative)
+                  sample_0-3h_301255reads_2812345678bp.fastq.gz
+```
+
+Compatible with **Guppy/MinKNOW** (`start_time=`) and **Dorado** (`st:Z:`) headers. Fast
+(each read compressed once, ~100× faster than v0.1), low-memory (streaming), parallel, and
+pure standard library.
+
+## Install
+
+```bash
+pip install git+https://github.com/duceppemo/nanoTimeSort.git
+```
+
+Requires Python ≥ 3.8. No other dependencies.
+
+## Usage
+
+```bash
+nanotimesort -f /path/to/fastq_pass/ -o /path/to/output/ -i 1h -p my_sample
+```
+
+📖 **Full documentation** — CLI reference, tutorial with example data, header compatibility,
+design notes and FAQ — is in the [**wiki**](https://github.com/duceppemo/nanoTimeSort/wiki).
+
+## License
+
+[MIT](LICENSE) © Marc-Olivier Duceppe

nanotimesort-1.0.0/nanotimesort/__init__.py

@@ -0,0 +1,4 @@
+"""nanoTimeSort: bin Oxford Nanopore reads by cumulative sequencing time intervals."""
+
+__version__ = "1.0.0"
+__author__ = "duceppemo"

nanotimesort-1.0.0/nanotimesort/binner.py

@@ -0,0 +1,315 @@
+"""Core binning engine.
+
+The pipeline runs in three stages, none of which holds reads in memory:
+
+1. **Scan** (parallel): stream every FASTQ once to find the earliest and
+   latest read start time and the total read count.
+2. **Chunk** (parallel): stream every FASTQ again; each read is gzip-
+   compressed exactly once, into the chunk file of the single interval it
+   belongs to.
+3. **Assemble** (sequential): cumulative output *k* is built by raw byte
+   concatenation of output *k-1* and the chunks of interval *k*. Gzip members
+   concatenate into a valid gzip stream, so no data is ever recompressed.
+
+Compared to the original implementation (which recompressed every read into
+every cumulative bin at gzip level 9), this is typically one to two orders of
+magnitude faster.
+"""
+
+from __future__ import annotations
+
+import gzip
+import os
+import shutil
+import sys
+import tempfile
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import dataclass
+from datetime import datetime
+from math import floor
+from time import time
+from typing import IO, Dict, Iterator, List, Optional, Tuple
+
+from .timestamps import extract_start_time
+
+UNIT_SECONDS = {"h": 3600.0, "m": 60.0, "s": 1.0}
+UNIT_NAMES = {"h": "hour", "m": "minute", "s": "second"}
+
+_COPY_BUFFER = 4 * 1024 * 1024
+
+
+@dataclass
+class ScanResult:
+    """Summary of one scan pass over a FASTQ file."""
+
+    reads: int = 0
+    missing: int = 0  # reads without a recognizable start time
+    t_min: Optional[datetime] = None
+    t_max: Optional[datetime] = None
+
+
+def open_fastq(path: str) -> IO[bytes]:
+    """Open a plain or gzipped FASTQ for binary reading."""
+    if path.endswith(".gz"):
+        return gzip.open(path, "rb")
+    return open(path, "rb", buffering=1024 * 1024)
+
+
+def fastq_records(handle: IO[bytes]) -> Iterator[Tuple[bytes, bytes, bytes]]:
+    """Yield (header, sequence, quality) byte lines, newline-stripped."""
+    while True:
+        header = handle.readline()
+        if not header:
+            return
+        seq = handle.readline()
+        handle.readline()  # '+' separator line
+        qual = handle.readline()
+        if not qual:
+            return  # truncated final record: skip it
+        yield header.rstrip(), seq.rstrip(), qual.rstrip()
+
+
+def find_fastq_files(input_path: str) -> List[str]:
+    """Collect FASTQ files from a folder (recursively) or a single file path."""
+    extensions = (".fastq", ".fastq.gz", ".fq", ".fq.gz")
+    if os.path.isfile(input_path):
+        return [input_path] if input_path.endswith(extensions) else []
+    fastq_list = []
+    for root, _dirs, filenames in os.walk(input_path):
+        for filename in filenames:
+            if filename.endswith(extensions):
+                fastq_list.append(os.path.join(root, filename))
+    return sorted(fastq_list)
+
+
+def scan_file(path: str) -> ScanResult:
+    """Pass 1: stream one file and record read count and time extremes."""
+    result = ScanResult()
+    if os.path.getsize(path) == 0:
+        return result
+    with open_fastq(path) as handle:
+        for header, _seq, _qual in fastq_records(handle):
+            start_time = extract_start_time(header)
+            if start_time is None:
+                result.missing += 1
+                continue
+            result.reads += 1
+            if result.t_min is None or start_time < result.t_min:
+                result.t_min = start_time
+            if result.t_max is None or start_time > result.t_max:
+                result.t_max = start_time
+    return result
+
+
+def chunk_file(
+    path: str,
+    file_index: int,
+    t_min: datetime,
+    bin_seconds: float,
+    num_bins: int,
+    chunk_dir: str,
+    compresslevel: int,
+) -> Tuple[List[int], List[int]]:
+    """Pass 2: write each read of one file into its interval chunk.
+
+    Chunk files are opened lazily, so intervals with no reads in this file
+    cost nothing. Returns per-interval read and base-pair counts.
+    """
+    reads_per_bin = [0] * num_bins
+    bp_per_bin = [0] * num_bins
+    handles: Dict[int, IO[bytes]] = {}
+
+    if os.path.getsize(path) == 0:
+        return reads_per_bin, bp_per_bin
+
+    try:
+        with open_fastq(path) as handle:
+            for header, seq, qual in fastq_records(handle):
+                start_time = extract_start_time(header)
+                if start_time is None:
+                    continue
+                elapsed = (start_time - t_min).total_seconds()
+                bin_index = min(floor(elapsed / bin_seconds), num_bins - 1)
+                out = handles.get(bin_index)
+                if out is None:
+                    chunk_path = os.path.join(
+                        chunk_dir, "chunk_b{:06d}_f{:06d}.fastq.gz".format(bin_index, file_index)
+                    )
+                    out = gzip.open(chunk_path, "wb", compresslevel=compresslevel)
+                    handles[bin_index] = out
+                out.write(header + b"\n" + seq + b"\n+\n" + qual + b"\n")
+                reads_per_bin[bin_index] += 1
+                bp_per_bin[bin_index] += len(seq)
+    finally:
+        for out in handles.values():
+            out.close()
+
+    return reads_per_bin, bp_per_bin
+
+
+def format_number(value: float) -> str:
+    """Render 2.0 as '2' and 0.5 as '0.5' for use in file names."""
+    return str(int(value)) if float(value).is_integer() else str(value)
+
+
+class NanoTimeSort:
+    """Bin Nanopore reads into cumulative sequencing-time interval files."""
+
+    def __init__(
+        self,
+        input_path: str,
+        output_folder: str,
+        interval: str,
+        prefix: str = "interval",
+        threads: int = 1,
+        compresslevel: int = 4,
+    ):
+        self.input_path = input_path
+        self.output_folder = output_folder
+        self.prefix = prefix
+        self.threads = max(1, threads)
+        self.compresslevel = compresslevel
+        self.bin_size, self.units = self._parse_interval(interval)
+        self.bin_seconds = self.bin_size * UNIT_SECONDS[self.units]
+
+    @staticmethod
+    def _parse_interval(interval: str) -> Tuple[float, str]:
+        units = interval[-1].lower()
+        if units not in UNIT_SECONDS:
+            raise ValueError(
+                "Invalid interval unit '{}'. Use one of: {}".format(
+                    units, ", ".join(sorted(UNIT_SECONDS))
+                )
+            )
+        try:
+            bin_size = float(interval[:-1])
+        except ValueError:
+            raise ValueError("Invalid interval value: '{}'".format(interval)) from None
+        if bin_size <= 0:
+            raise ValueError("Interval must be greater than zero.")
+        return bin_size, units
+
+    def run(self) -> List[str]:
+        """Execute the full pipeline. Returns the list of output file paths."""
+        overall_start = time()
+        os.makedirs(self.output_folder, exist_ok=True)
+
+        fastq_list = find_fastq_files(self.input_path)
+        if not fastq_list:
+            raise FileNotFoundError("No FASTQ files found in {}".format(self.input_path))
+        workers = min(self.threads, len(fastq_list))
+
+        # Pass 1: find the time range of the run.
+        print("Scanning {} FASTQ file(s)...".format(len(fastq_list)), end="", flush=True)
+        start = time()
+        scans = self._map(scan_file, fastq_list, workers)
+        total_reads = sum(s.reads for s in scans)
+        total_missing = sum(s.missing for s in scans)
+        t_mins = [s.t_min for s in scans if s.t_min is not None]
+        t_maxs = [s.t_max for s in scans if s.t_max is not None]
+        print(" {} reads in {}".format(total_reads, elapsed_time(time() - start)))
+        if total_missing:
+            print(
+                "Warning: {} read(s) without a 'start_time=' or 'st:Z:' header field "
+                "were skipped.".format(total_missing),
+                file=sys.stderr,
+            )
+        if not t_mins:
+            raise ValueError(
+                "No read start times found. Headers must contain a Guppy/MinKNOW "
+                "'start_time=' field or a Dorado 'st:Z:' tag."
+            )
+
+        t_min, t_max = min(t_mins), max(t_maxs)
+        run_seconds = (t_max - t_min).total_seconds()
+        num_bins = int(run_seconds // self.bin_seconds) + 1
+        print(
+            "Run spans {} -> {} intervals of {}{}".format(
+                elapsed_time(run_seconds), num_bins, format_number(self.bin_size), self.units
+            )
+        )
+
+        # Pass 2: compress each read once into its interval chunk.
+        print("Binning reads...", end="", flush=True)
+        start = time()
+        with tempfile.TemporaryDirectory(prefix="nanotimesort_", dir=self.output_folder) as chunk_dir:
+            jobs = [
+                (path, i, t_min, self.bin_seconds, num_bins, chunk_dir, self.compresslevel)
+                for i, path in enumerate(fastq_list)
+            ]
+            results = self._starmap(chunk_file, jobs, workers)
+            reads_per_bin = [0] * num_bins
+            bp_per_bin = [0] * num_bins
+            for file_reads, file_bp in results:
+                for b in range(num_bins):
+                    reads_per_bin[b] += file_reads[b]
+                    bp_per_bin[b] += file_bp[b]
+            print(" done in {}".format(elapsed_time(time() - start)))
+
+            # Pass 3: assemble cumulative outputs by gzip member concatenation.
+            print("Writing cumulative interval files...", end="", flush=True)
+            start = time()
+            outputs = self._assemble(chunk_dir, num_bins, reads_per_bin, bp_per_bin)
+            print(" done in {}".format(elapsed_time(time() - start)))
+
+        print("Total run time: {}".format(elapsed_time(time() - overall_start)))
+        return outputs
+
+    def _assemble(
+        self,
+        chunk_dir: str,
+        num_bins: int,
+        reads_per_bin: List[int],
+        bp_per_bin: List[int],
+    ) -> List[str]:
+        chunk_names = sorted(os.listdir(chunk_dir))
+        outputs: List[str] = []
+        previous_path: Optional[str] = None
+        cumulative_reads = 0
+        cumulative_bp = 0
+
+        for b in range(num_bins):
+            cumulative_reads += reads_per_bin[b]
+            cumulative_bp += bp_per_bin[b]
+            label = format_number((b + 1) * self.bin_size)
+            out_name = "{}_0-{}{}_{}reads_{}bp.fastq.gz".format(
+                self.prefix, label, self.units, cumulative_reads, cumulative_bp
+            )
+            out_path = os.path.join(self.output_folder, out_name)
+            prefix = "chunk_b{:06d}_".format(b)
+            with open(out_path, "wb") as out:
+                if previous_path is not None:
+                    with open(previous_path, "rb") as prev:
+                        shutil.copyfileobj(prev, out, _COPY_BUFFER)
+                for name in chunk_names:
+                    if name.startswith(prefix):
+                        with open(os.path.join(chunk_dir, name), "rb") as chunk:
+                            shutil.copyfileobj(chunk, out, _COPY_BUFFER)
+            outputs.append(out_path)
+            previous_path = out_path
+
+        return outputs
+
+    def _map(self, func, items, workers):
+        if workers <= 1 or len(items) <= 1:
+            return [func(item) for item in items]
+        with ProcessPoolExecutor(max_workers=workers) as pool:
+            return list(pool.map(func, items))
+
+    def _starmap(self, func, jobs, workers):
+        if workers <= 1 or len(jobs) <= 1:
+            return [func(*job) for job in jobs]
+        with ProcessPoolExecutor(max_workers=workers) as pool:
+            futures = [pool.submit(func, *job) for job in jobs]
+            return [f.result() for f in futures]
+
+
+def elapsed_time(seconds: float) -> str:
+    """Format a duration in seconds as a compact '1d2h3m4s' style string."""
+    if seconds < 1:
+        return "{:.2f}s".format(seconds)
+    minutes, secs = divmod(int(round(seconds)), 60)
+    hours, minutes = divmod(minutes, 60)
+    days, hours = divmod(hours, 24)
+    parts = [("d", days), ("h", hours), ("m", minutes), ("s", secs)]
+    return "".join("{}{}".format(value, name) for name, value in parts if value) or "0s"

nanotimesort-1.0.0/nanotimesort/cli.py

@@ -0,0 +1,78 @@
+"""Command-line interface for nanoTimeSort."""
+
+from __future__ import annotations
+
+import sys
+from argparse import ArgumentParser, RawTextHelpFormatter
+from multiprocessing import cpu_count
+
+from . import __version__
+from .binner import NanoTimeSort
+
+
+def build_parser() -> ArgumentParser:
+    cpu = cpu_count()
+    parser = ArgumentParser(
+        prog="nanotimesort",
+        description="Bin Oxford Nanopore reads by cumulative sequencing time intervals.\n"
+                    "Supports Guppy/MinKNOW ('start_time=') and Dorado ('st:Z:') FASTQ headers.",
+        formatter_class=RawTextHelpFormatter,
+    )
+    parser.add_argument(
+        "-f", "--fastq", metavar="/basecalled/folder/", required=True,
+        help="Input folder (searched recursively) or single FASTQ file.\n"
+             "Accepts .fastq, .fq, .fastq.gz and .fq.gz.",
+    )
+    parser.add_argument(
+        "-o", "--output", metavar="/output/folder/", required=True,
+        help="Output folder. Created if it does not exist.",
+    )
+    parser.add_argument(
+        "-i", "--interval", metavar="1h", required=True,
+        help="Time interval for the bins, e.g. '1h', '30m' or '90s'.\n"
+             "Bins are cumulative: with '-i 1h', the second file also\n"
+             "contains the reads of the first hour.",
+    )
+    parser.add_argument(
+        "-p", "--prefix", metavar="my_sample", default="interval",
+        help="Output file prefix. Files are named like\n"
+             "'my_sample_0-1h_123reads_456789bp.fastq.gz'.\n"
+             "Default: interval",
+    )
+    parser.add_argument(
+        "-t", "--threads", metavar=str(cpu), type=int, default=cpu,
+        help="Number of FASTQ files to process in parallel.\n"
+             "Default: {}".format(cpu),
+    )
+    parser.add_argument(
+        "-c", "--compression-level", metavar="4", type=int, default=4,
+        choices=range(1, 10),
+        help="Gzip compression level for output files (1=fastest, 9=smallest).\n"
+             "Default: 4",
+    )
+    parser.add_argument(
+        "-v", "--version", action="version", version="nanoTimeSort v{}".format(__version__),
+    )
+    return parser
+
+
+def main(argv=None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        binner = NanoTimeSort(
+            input_path=args.fastq,
+            output_folder=args.output,
+            interval=args.interval,
+            prefix=args.prefix,
+            threads=args.threads,
+            compresslevel=args.compression_level,
+        )
+        binner.run()
+    except (ValueError, FileNotFoundError) as err:
+        print("Error: {}".format(err), file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

nanotimesort-1.0.0/nanotimesort/timestamps.py

@@ -0,0 +1,69 @@
+"""Extraction and parsing of read start times from Nanopore FASTQ headers.
+
+Two header dialects are supported:
+
+* Guppy / MinKNOW (key=value fields)::
+
+    @<read_id> runid=... read=... ch=... start_time=2019-07-16T19:51:22Z
+    @<read_id> ... start_time=2025-01-13T10:45:28.681306+00:00 ...
+
+* Dorado (SAM-style tags)::
+
+    @<read_id> qs:f:21.3 du:f:12.44 ch:i:942 st:Z:2023-09-01T11:13:45.731+00:00 ...
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Optional, Union
+
+# Guppy/MinKNOW style field.
+_GUPPY_PREFIX = b"start_time="
+# Dorado SAM-tag style field (st:Z:<ISO 8601>).
+_DORADO_PREFIX = b"st:Z:"
+
+_FRACTION_RE = re.compile(r"\.(\d+)")
+
+
+def extract_start_time(header: bytes) -> Optional[datetime]:
+    """Return the read start time from a FASTQ header line, or None if absent.
+
+    :param header: raw FASTQ header line (bytes, with or without trailing newline)
+    :return: timezone-aware datetime (UTC assumed when the timestamp is naive)
+    """
+    for item in header.split():
+        if item.startswith(_GUPPY_PREFIX):
+            return parse_timestamp(item[len(_GUPPY_PREFIX):])
+        if item.startswith(_DORADO_PREFIX):
+            return parse_timestamp(item[len(_DORADO_PREFIX):])
+    return None
+
+
+def parse_timestamp(raw: Union[bytes, str]) -> datetime:
+    """Parse an ISO 8601 / RFC 3339 timestamp into a timezone-aware datetime.
+
+    Handles 'Z' suffixes and unusual fractional-second precision on Python
+    versions where datetime.fromisoformat() is strict (< 3.11).
+    """
+    if isinstance(raw, bytes):
+        raw = raw.decode("ascii")
+    try:
+        dt = datetime.fromisoformat(raw)
+    except ValueError:
+        dt = datetime.fromisoformat(_normalize(raw))
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
+def _normalize(s: str) -> str:
+    """Rewrite a timestamp so strict fromisoformat() implementations accept it."""
+    if s.endswith(("Z", "z")):
+        s = s[:-1] + "+00:00"
+
+    def _pad(match: "re.Match[str]") -> str:
+        digits = match.group(1)[:6]
+        return "." + digits.ljust(6, "0")
+
+    return _FRACTION_RE.sub(_pad, s, count=1)

nanotimesort-1.0.0/nanotimesort.egg-info/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: nanotimesort
+Version: 1.0.0
+Summary: Bin Oxford Nanopore reads by cumulative sequencing time intervals (Guppy and Dorado compatible)
+Author-email: Marc-Olivier Duceppe <duceppemo@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2019 Marc-Olivier Duceppe
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/duceppemo/nanoTimeSort
+Project-URL: Issues, https://github.com/duceppemo/nanoTimeSort/issues
+Project-URL: Wiki, https://github.com/duceppemo/nanoTimeSort/wiki
+Keywords: nanopore,fastq,bioinformatics,minion,dorado,guppy,binning
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <img src="docs/images/logo.svg" alt="nanoTimeSort" width="520">
+</p>
+
+<p align="center">
+  <a href="https://github.com/duceppemo/nanoTimeSort/actions/workflows/ci.yml"><img src="https://github.com/duceppemo/nanoTimeSort/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
+  <a href="https://codecov.io/gh/duceppemo/nanoTimeSort"><img src="https://codecov.io/gh/duceppemo/nanoTimeSort/branch/master/graph/badge.svg" alt="codecov"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
+  <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.8%2B-blue.svg" alt="Python 3.8+"></a>
+  <a href="pyproject.toml"><img src="https://img.shields.io/badge/dependencies-none-brightgreen.svg" alt="No dependencies"></a>
+</p>
+
+Bin Oxford Nanopore reads by **cumulative sequencing time intervals**, using the read start time
+that MinKNOW, Guppy and Dorado embed in every FASTQ header. Useful to answer *"how long did I
+actually need to sequence?"* — for time-to-detection studies, assembly saturation curves, or
+benchmarking real-time pipelines.
+
+```text
+fastq_pass/  ──►  sample_0-1h_125437reads_1103093674bp.fastq.gz
+                  sample_0-2h_225891reads_2087456221bp.fastq.gz   (bins are cumulative)
+                  sample_0-3h_301255reads_2812345678bp.fastq.gz
+```
+
+Compatible with **Guppy/MinKNOW** (`start_time=`) and **Dorado** (`st:Z:`) headers. Fast
+(each read compressed once, ~100× faster than v0.1), low-memory (streaming), parallel, and
+pure standard library.
+
+## Install
+
+```bash
+pip install git+https://github.com/duceppemo/nanoTimeSort.git
+```
+
+Requires Python ≥ 3.8. No other dependencies.
+
+## Usage
+
+```bash
+nanotimesort -f /path/to/fastq_pass/ -o /path/to/output/ -i 1h -p my_sample
+```
+
+📖 **Full documentation** — CLI reference, tutorial with example data, header compatibility,
+design notes and FAQ — is in the [**wiki**](https://github.com/duceppemo/nanoTimeSort/wiki).
+
+## License
+
+[MIT](LICENSE) © Marc-Olivier Duceppe

nanotimesort-1.0.0/nanotimesort.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+README.md
+pyproject.toml
+nanotimesort/__init__.py
+nanotimesort/binner.py
+nanotimesort/cli.py
+nanotimesort/timestamps.py
+nanotimesort.egg-info/PKG-INFO
+nanotimesort.egg-info/SOURCES.txt
+nanotimesort.egg-info/dependency_links.txt
+nanotimesort.egg-info/entry_points.txt
+nanotimesort.egg-info/requires.txt
+nanotimesort.egg-info/top_level.txt
+tests/test_binner.py
+tests/test_cli.py
+tests/test_timestamps.py

nanotimesort-1.0.0/nanotimesort.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

nanotimesort-1.0.0/nanotimesort.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nanotimesort = nanotimesort.cli:main

nanotimesort-1.0.0/nanotimesort.egg-info/requires.txt

@@ -0,0 +1,5 @@
+
+[dev]
+pytest
+pytest-cov
+ruff

nanotimesort-1.0.0/nanotimesort.egg-info/top_level.txt

@@ -0,0 +1 @@
+nanotimesort

nanotimesort-1.0.0/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["setuptools>=64"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "nanotimesort"
+version = "1.0.0"
+description = "Bin Oxford Nanopore reads by cumulative sequencing time intervals (Guppy and Dorado compatible)"
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [{ name = "Marc-Olivier Duceppe", email = "duceppemo@gmail.com" }]
+requires-python = ">=3.8"
+keywords = ["nanopore", "fastq", "bioinformatics", "minion", "dorado", "guppy", "binning"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Bio-Informatics",
+]
+dependencies = []
+
+[project.optional-dependencies]
+dev = ["pytest", "pytest-cov", "ruff"]
+
+[project.urls]
+Homepage = "https://github.com/duceppemo/nanoTimeSort"
+Issues = "https://github.com/duceppemo/nanoTimeSort/issues"
+Wiki = "https://github.com/duceppemo/nanoTimeSort/wiki"
+
+[project.scripts]
+nanotimesort = "nanotimesort.cli:main"
+
+[tool.setuptools]
+packages = ["nanotimesort"]
+
+[tool.ruff]
+line-length = 110
+target-version = "py38"
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "B"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

nanotimesort-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

nanotimesort-1.0.0/tests/test_binner.py

@@ -0,0 +1,125 @@
+import gzip
+import os
+
+import pytest
+
+from nanotimesort.binner import NanoTimeSort, find_fastq_files
+
+BASE = "2024-05-01T{:02d}:{:02d}:00Z"
+
+
+def make_read(read_id, minutes, style="guppy", seq="ACGT" * 10):
+    """Build one FASTQ record with a start time `minutes` after 10:00 UTC."""
+    hour, minute = divmod(600 + minutes, 60)
+    stamp = BASE.format(hour, minute)
+    if style == "guppy":
+        header = "@{} runid=abc ch=1 start_time={}".format(read_id, stamp)
+    else:  # dorado
+        stamp = stamp.replace("Z", ".000+00:00")
+        header = "@{} qs:f:20.0 ch:i:1 st:Z:{}".format(read_id, stamp)
+    return "{}\n{}\n+\n{}\n".format(header, seq, "I" * len(seq))
+
+
+@pytest.fixture
+def run_folder(tmp_path):
+    """Two FASTQ files (one gzipped, one plain) spanning ~2.5 hours."""
+    fastq_dir = tmp_path / "fastq"
+    fastq_dir.mkdir()
+    # File 1 (gzipped, Guppy headers): reads at 0, 30 and 70 minutes.
+    content1 = (
+        make_read("read1", 0)
+        + make_read("read2", 30)
+        + make_read("read3", 70)
+    )
+    with gzip.open(fastq_dir / "part1.fastq.gz", "wt") as handle:
+        handle.write(content1)
+    # File 2 (plain, Dorado headers): reads at 90 and 150 minutes.
+    content2 = make_read("read4", 90, style="dorado") + make_read("read5", 150, style="dorado")
+    (fastq_dir / "part2.fastq").write_text(content2)
+    return fastq_dir
+
+
+def read_ids(path):
+    with gzip.open(path, "rt") as handle:
+        return [line.split()[0][1:] for i, line in enumerate(handle) if i % 4 == 0]
+
+
+def test_find_fastq_files(run_folder):
+    files = find_fastq_files(str(run_folder))
+    assert [os.path.basename(f) for f in files] == ["part1.fastq.gz", "part2.fastq"]
+
+
+def test_cumulative_binning(run_folder, tmp_path):
+    out_dir = tmp_path / "out"
+    binner = NanoTimeSort(
+        input_path=str(run_folder),
+        output_folder=str(out_dir),
+        interval="1h",
+        prefix="test",
+        threads=1,
+    )
+    outputs = binner.run()
+
+    # Run spans 150 minutes -> three cumulative 1 h bins.
+    names = [os.path.basename(p) for p in outputs]
+    bp = 40  # each read is 40 bp
+    assert names == [
+        "test_0-1h_2reads_{}bp.fastq.gz".format(2 * bp),
+        "test_0-2h_4reads_{}bp.fastq.gz".format(4 * bp),
+        "test_0-3h_5reads_{}bp.fastq.gz".format(5 * bp),
+    ]
+
+    # Bin contents are cumulative and mix Guppy + Dorado reads.
+    assert sorted(read_ids(outputs[0])) == ["read1", "read2"]
+    assert sorted(read_ids(outputs[1])) == ["read1", "read2", "read3", "read4"]
+    assert sorted(read_ids(outputs[2])) == ["read1", "read2", "read3", "read4", "read5"]
+
+
+def test_parallel_matches_serial(run_folder, tmp_path):
+    results = {}
+    for threads in (1, 2):
+        out_dir = tmp_path / "out_t{}".format(threads)
+        binner = NanoTimeSort(
+            input_path=str(run_folder),
+            output_folder=str(out_dir),
+            interval="30m",
+            prefix="par",
+            threads=threads,
+        )
+        outputs = binner.run()
+        results[threads] = {
+            os.path.basename(p): sorted(read_ids(p)) for p in outputs
+        }
+    assert results[1] == results[2]
+
+
+def test_single_file_input(run_folder, tmp_path):
+    out_dir = tmp_path / "out_single"
+    binner = NanoTimeSort(
+        input_path=str(run_folder / "part2.fastq"),
+        output_folder=str(out_dir),
+        interval="2h",
+        prefix="single",
+        threads=1,
+    )
+    outputs = binner.run()
+    assert len(outputs) == 1
+    assert sorted(read_ids(outputs[0])) == ["read4", "read5"]
+
+
+def test_invalid_interval():
+    with pytest.raises(ValueError):
+        NanoTimeSort("in", "out", interval="1x")
+    with pytest.raises(ValueError):
+        NanoTimeSort("in", "out", interval="-5m")
+
+
+def test_reads_without_timestamp_are_skipped(tmp_path, capsys):
+    fastq_dir = tmp_path / "fastq"
+    fastq_dir.mkdir()
+    content = make_read("good1", 0) + "@orphan length=4\nACGT\n+\nIIII\n" + make_read("good2", 10)
+    (fastq_dir / "mixed.fastq").write_text(content)
+    out_dir = tmp_path / "out"
+    binner = NanoTimeSort(str(fastq_dir), str(out_dir), interval="1h", threads=1)
+    outputs = binner.run()
+    assert sorted(read_ids(outputs[-1])) == ["good1", "good2"]

nanotimesort-1.0.0/tests/test_cli.py

@@ -0,0 +1,55 @@
+import gzip
+
+import pytest
+
+from nanotimesort import __version__
+from nanotimesort.cli import main
+
+
+@pytest.fixture
+def fastq_folder(tmp_path):
+    folder = tmp_path / "fastq"
+    folder.mkdir()
+    reads = "".join(
+        "@read{} runid=abc start_time=2024-05-01T1{}:00:00Z\nACGT\n+\nIIII\n".format(i, i)
+        for i in range(3)
+    )
+    (folder / "run.fastq").write_text(reads)
+    return folder
+
+
+def test_main_success(fastq_folder, tmp_path):
+    out_dir = tmp_path / "out"
+    rc = main(["-f", str(fastq_folder), "-o", str(out_dir), "-i", "1h", "-p", "cli"])
+    assert rc == 0
+    outputs = sorted(out_dir.glob("cli_0-*.fastq.gz"))
+    assert len(outputs) == 3
+    with gzip.open(outputs[-1], "rt") as handle:
+        assert sum(1 for line in handle if line.startswith("@read")) >= 1
+
+
+def test_main_no_fastq(tmp_path):
+    empty = tmp_path / "empty"
+    empty.mkdir()
+    rc = main(["-f", str(empty), "-o", str(tmp_path / "out"), "-i", "1h"])
+    assert rc == 1
+
+
+def test_main_bad_interval(fastq_folder, tmp_path):
+    rc = main(["-f", str(fastq_folder), "-o", str(tmp_path / "out"), "-i", "1x"])
+    assert rc == 1
+
+
+def test_main_no_timestamps(tmp_path):
+    folder = tmp_path / "fastq"
+    folder.mkdir()
+    (folder / "run.fastq").write_text("@read1 length=4\nACGT\n+\nIIII\n")
+    rc = main(["-f", str(folder), "-o", str(tmp_path / "out"), "-i", "1h"])
+    assert rc == 1
+
+
+def test_version(capsys):
+    with pytest.raises(SystemExit) as exc:
+        main(["--version"])
+    assert exc.value.code == 0
+    assert __version__ in capsys.readouterr().out

nanotimesort-1.0.0/tests/test_timestamps.py

@@ -0,0 +1,48 @@
+from datetime import datetime, timezone
+
+from nanotimesort.timestamps import extract_start_time, parse_timestamp
+
+GUPPY_HEADER = (
+    b"@c041234f-1234-4a81-9a5a-1234567890ab runid=abc123 read=42 ch=133 "
+    b"start_time=2019-07-16T19:51:22Z flow_cell_id=FAK12345"
+)
+MINKNOW_HEADER = (
+    b"@bd8655fb-383c-45cc-bff3-eb1dc86533e0 runid=abc parent_read_id=bd8655fb "
+    b"start_time=2025-01-13T10:45:28.681306+00:00 protocol_group_id=test"
+)
+DORADO_HEADER = (
+    b"@0000813e-1111-4c35-8f57-222233334444 qs:f:21.5 du:f:12.44 ns:i:62205 "
+    b"ts:i:10 mx:i:3 ch:i:942 st:Z:2023-09-01T11:13:45.731+00:00 rn:i:9566 "
+    b"fn:Z:PAO12345_pass_0.pod5 sm:f:421.3 sd:f:93.0 sv:Z:quantile dx:i:0 "
+    b"RG:Z:abc_dna_r10.4.1_e8.2_400bps_hac@v4.2.0"
+)
+
+
+def test_guppy_header():
+    dt = extract_start_time(GUPPY_HEADER)
+    assert dt == datetime(2019, 7, 16, 19, 51, 22, tzinfo=timezone.utc)
+
+
+def test_minknow_header_with_microseconds():
+    dt = extract_start_time(MINKNOW_HEADER)
+    assert dt == datetime(2025, 1, 13, 10, 45, 28, 681306, tzinfo=timezone.utc)
+
+
+def test_dorado_header():
+    dt = extract_start_time(DORADO_HEADER)
+    assert dt == datetime(2023, 9, 1, 11, 13, 45, 731000, tzinfo=timezone.utc)
+
+
+def test_header_without_time_returns_none():
+    assert extract_start_time(b"@read1 length=100") is None
+
+
+def test_naive_timestamp_assumed_utc():
+    dt = parse_timestamp("2023-09-01T11:13:45")
+    assert dt.tzinfo is not None
+    assert dt.utcoffset().total_seconds() == 0
+
+
+def test_z_suffix_with_milliseconds():
+    dt = parse_timestamp("2023-09-01T11:13:45.7Z")
+    assert dt == datetime(2023, 9, 1, 11, 13, 45, 700000, tzinfo=timezone.utc)