directclean 0.1.0__py3-none-any.whl

directclean/__init__.py

@@ -0,0 +1,14 @@
+"""
+DirectClean — Remove RT artifacts from Oxford Nanopore Direct-cDNA sequencing.
+
+A comprehensive preprocessing pipeline that detects and removes:
+  1. Internal TSO/RTP adapter chimeras (Rescuer module)
+  2. Homopolymer-mediated RT template switching artifacts (Filter module)
+
+Typical CLI usage::
+
+    directclean -i reads.fastq -r genome.fa -o results/ -t 8
+"""
+
+__version__ = "0.1.0"
+__author__ = "Qingxiang Guo"

directclean/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running DirectClean as ``python -m directclean``."""
+
+from directclean.cli.app import app
+
+if __name__ == "__main__":
+    app()

directclean/cli/__init__.py

@@ -0,0 +1 @@
+"""DirectClean CLI package."""

directclean/cli/app.py

@@ -0,0 +1,289 @@
+"""
+DirectClean command-line interface.
+
+Single-command design — no subcommands::
+
+    directclean -i reads.fastq -r genome.fa -o results/ -t 8
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.logging import RichHandler
+
+from directclean import __version__
+from directclean.pipeline import DirectCleanPipeline, PipelineConfig
+from directclean.rescuer.adaptor_seq import AdapterConfig
+from directclean.filter.homopolymer import HomopolymerConfig
+
+console = Console()
+
+LOGO = """
+[bold cyan]╔══════════════════════════════════════════════════════════════╗
+║  ____  _               _    ____ _                          ║
+║ |  _ \\(_)_ __ ___  ___| |_ / ___| | ___  __ _ _ __         ║
+║ | | | | | '__/ _ \\/ __| __| |   | |/ _ \\/ _` | '_ \\        ║
+║ | |_| | | | |  __/ (__| |_| |___| |  __/ (_| | | | |       ║
+║ |____/|_|_|  \\___|\\___|\\__|\\____|_|\\___|\\__,_|_| |_|       ║
+╚══════════════════════════════════════════════════════════════╝[/]
+"""
+
+HELP_TEXT = (
+    "DirectClean — Strand orientation, artifact removal, and chimeric "
+    "read rescue for Oxford Nanopore direct-cDNA sequencing.\n\n"
+    "Removes foldback inversions and homopolymer-mediated RT template "
+    "switching artifacts that existing tools do not address. "
+    "Chimeric reads are chopped at artifact junctions and flanking "
+    "sub-reads are rescued.\n\n"
+    "Pipeline: Breakinator → Restrander → Unknowns Rescue → "
+    "Adapter Rescue → Homopolymer Rescue"
+)
+
+app = typer.Typer(
+    name="directclean",
+    help=HELP_TEXT,
+    add_completion=False,
+    no_args_is_help=True,
+    context_settings={"help_option_names": ["-h", "--help"]},
+)
+
+
+def _setup_logging(verbose: bool) -> None:
+    """Configure logging with Rich handler."""
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(message)s",
+        datefmt="[%X]",
+        handlers=[
+            RichHandler(
+                console=console,
+                show_path=False,
+                rich_tracebacks=True,
+            )
+        ],
+    )
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        console.print(LOGO)
+        console.print(f"  v{__version__}")
+        raise typer.Exit()
+
+
+# ---------------------------------------------------------------------------
+# Main command
+# ---------------------------------------------------------------------------
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    # ---- Required ----
+    input_fastq: Path = typer.Option(
+        ...,
+        "--input",
+        "-i",
+        help="Raw input FASTQ file from Oxford Nanopore Direct-cDNA sequencing.",
+        exists=True,
+        dir_okay=False,
+        readable=True,
+    ),
+    reference: Path = typer.Option(
+        ...,
+        "--reference",
+        "-r",
+        help="Reference genome FASTA file.",
+        exists=True,
+        dir_okay=False,
+        readable=True,
+    ),
+    output_dir: Path = typer.Option(
+        ...,
+        "--output",
+        "-o",
+        help="Output directory for results.",
+    ),
+    # ---- General ----
+    threads: int = typer.Option(
+        4,
+        "--threads",
+        "-t",
+        help="Number of threads for minimap2, samtools, and breakinator.",
+        min=1,
+    ),
+    prefix: str = typer.Option(
+        "directclean",
+        "--prefix",
+        "-p",
+        help="Filename prefix for output files.",
+    ),
+    # ---- Breakinator parameters ----
+    junc_bed: Path | None = typer.Option(
+        None,
+        "--junc-bed",
+        "-j",
+        help=(
+            "Junction BED12 file for guided minimap2 alignment "
+            "(used by Breakinator stage). Recommended: GENCODE annotation."
+        ),
+        exists=True,
+        dir_okay=False,
+        readable=True,
+    ),
+    # ---- Rescuer parameters ----
+    max_edit_distance: int = typer.Option(
+        3,
+        "--max-edit-dist",
+        help="Maximum edit distance for adapter fuzzy matching.",
+        min=0,
+        max=5,
+    ),
+    min_confidence: int = typer.Option(
+        2,
+        "--min-confidence",
+        help=(
+            "Minimum signals (1-3) to chop: "
+            "1=TSO only, 2=two of polyA/RTP/TSO, 3=all three."
+        ),
+        min=1,
+        max=3,
+    ),
+    min_segment_length: int = typer.Option(
+        50,
+        "--min-segment-len",
+        help="Minimum sub-read length to keep after chopping (bp).",
+        min=10,
+    ),
+    # ---- Homopolymer filter parameters ----
+    scan_window: int = typer.Option(
+        10,
+        "--scan-window",
+        help="Sliding window size for A/T density scan (bp).",
+        min=5,
+    ),
+    density_threshold: float = typer.Option(
+        0.85,
+        "--density-threshold",
+        help="Minimum A/T fraction in scanning window to flag.",
+        min=0.5,
+        max=1.0,
+    ),
+    min_run: int = typer.Option(
+        5,
+        "--min-run",
+        help="Minimum consecutive A or T to flag.",
+        min=2,
+    ),
+    context_window: int = typer.Option(
+        50,
+        "--context-window",
+        help="Bases to extract on each side of a junction.",
+        min=10,
+    ),
+    # ---- Flags ----
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Enable debug logging.",
+    ),
+    html_report: bool = typer.Option(
+        True,
+        "--html-report/--no-html-report",
+        help="Generate an interactive HTML summary report with charts.",
+    ),
+    version: bool | None = typer.Option(
+        None,
+        "--version",
+        "-V",
+        help="Show version and exit.",
+        callback=_version_callback,
+        is_eager=True,
+    ),
+) -> None:
+    """
+    Strand orientation, artifact removal, and chimeric read rescue
+    for Oxford Nanopore direct-cDNA sequencing.
+
+    \b
+    Pipeline stages:
+      1. BREAKINATOR        Remove foldback inversion artifacts
+      2. RESTRANDER         Orient reads 5'→3', remove primer artifacts
+      3. UNKNOWNS RESCUE    Recover orientable reads from Restrander unknowns
+      4. ADAPTER RESCUE     Detect internal TSO/RTP adapters, chop and rescue
+      5. HOMOPOLYMER RESCUE Detect RT template switching at A/T-rich junctions
+
+    \b
+    Stages 1-2 remove artifactual reads.
+    Stages 3-5 chop chimeric reads at artifact junctions and rescue
+    flanking sub-reads — no reads are discarded.
+    """
+    _setup_logging(verbose)
+
+    # Print logo at pipeline start
+    console.print(LOGO)
+
+    # Build configuration from CLI options
+    adapter_cfg = AdapterConfig(
+        max_edit_distance=max_edit_distance,
+        min_segment_length=min_segment_length,
+    )
+    homopolymer_cfg = HomopolymerConfig(
+        scan_window=scan_window,
+        density_threshold=density_threshold,
+        min_run=min_run,
+        context_window=context_window,
+    )
+    pipeline_cfg = PipelineConfig(
+        adapter_config=adapter_cfg,
+        homopolymer_config=homopolymer_cfg,
+        min_confidence=min_confidence,
+        context_window=context_window,
+        threads=threads,
+        junc_bed=junc_bed,
+    )
+
+    # Run pipeline
+    try:
+        pipeline = DirectCleanPipeline(
+            input_fastq=input_fastq,
+            reference=reference,
+            output_dir=output_dir,
+            config=pipeline_cfg,
+            prefix=prefix,
+        )
+        report = pipeline.run()
+
+        console.print()
+        console.print("[bold green]✓ DirectClean completed successfully.[/]")
+        console.print(f"  Cleaned reads: [cyan]{pipeline.cleaned_fastq}[/]")
+        console.print(f"  Rescued reads: [cyan]{pipeline.rescued_fastq}[/]")
+
+        if html_report:
+            from directclean.report import HtmlReportGenerator
+
+            report_path = output_dir / f"{prefix}.report.html"
+            generator = HtmlReportGenerator(
+                report=report,
+                config=pipeline_cfg,
+                input_fastq=input_fastq,
+                output_dir=output_dir,
+                prefix=prefix,
+            )
+            generator.write(report_path)
+            console.print(f"  HTML report: [cyan]{report_path}[/]")
+
+    except FileNotFoundError as e:
+        console.print(f"[bold red]Error:[/] {e}")
+        raise typer.Exit(code=1)
+    except Exception as e:
+        console.print(f"[bold red]Error:[/] {e}")
+        if verbose:
+            console.print_exception()
+        raise typer.Exit(code=1)

directclean/external/__init__.py

@@ -0,0 +1 @@
+"""External tool wrappers for DirectClean."""

directclean/external/breakinator.py

@@ -0,0 +1,386 @@
+"""
+Breakinator wrapper — remove foldback inversion artifacts.
+
+Breakinator detects two types of artifacts in Direct-cDNA reads:
+*foldback inversions* (hairpin structures from sequencing) and
+*chimeric reads* (ligation artifacts).  DirectClean only removes
+**foldback** reads because chimeric reads are handled downstream
+by our own Rescuer and Homopolymer Filter modules.
+
+The workflow mirrors the manual steps::
+
+    1. minimap2  → raw name-sorted SAM  (Breakinator needs SAM input)
+    2. breakinator → artifacts.txt       (tabular classification)
+    3. Parse artifacts.txt → collect Foldback read IDs
+    4. split_fastq_by_ids → keep non-foldback reads
+
+Note: This minimap2 run is independent from Stage 4 (the alignment
+used by the Homopolymer Filter).  Breakinator requires a SAM file,
+not a coordinate-sorted BAM.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from directclean.external.dependencies import check_binary
+from directclean.utils.io import split_fastq_by_ids
+
+logger = logging.getLogger(__name__)
+
+_MIN_BREAKINATOR_VERSION = (1, 1, 1)
+
+
+def _check_breakinator_version(binary: str) -> None:
+    """Verify that the installed breakinator is >= 1.1.1 (Rust version).
+
+    The old Python-based breakinator (<=1.0.x) expects PAF input and will
+    crash with an IndexError when given SAM.  DirectClean requires the
+    Rust rewrite (>=1.1.1) which natively accepts SAM/BAM/CRAM.
+    """
+    try:
+        proc = subprocess.run(
+            [binary, "--version"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        output = (proc.stdout + proc.stderr).strip()
+    except Exception:
+        return  # If we can't check, let the actual run surface errors
+
+    # Match version strings like "breakinator 1.1.1" or "1.1.1"
+    match = re.search(r"(\d+)\.(\d+)\.(\d+)", output)
+    if match is None:
+        logger.warning(f"Could not parse breakinator version from: {output!r}")
+        return
+
+    version: tuple[int, ...] = tuple(int(x) for x in match.groups())
+    if version < _MIN_BREAKINATOR_VERSION:
+        min_ver = ".".join(str(v) for v in _MIN_BREAKINATOR_VERSION)
+        cur_ver = ".".join(str(v) for v in version)
+        raise RuntimeError(
+            f"breakinator {cur_ver} is too old. DirectClean requires "
+            f">= {min_ver} (the Rust version) which accepts SAM input. "
+            f"The old Python version expects PAF and will fail. "
+            f"Please update: conda install -c bioconda 'breakinator>={min_ver}'"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Report
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class BreakReport:
+    """Summary statistics from the Breakinator stage.
+
+    Attributes:
+        total_breakpoints:  Total breakpoints classified by Breakinator.
+        foldback_count:     Breakpoints classified as Foldback.
+        chimeric_count:     Breakpoints classified as Chimeric.
+        pass_count:         Breakpoints classified as Pass.
+        foldback_read_ids:  Number of unique reads with Foldback.
+        input_reads:        Total reads in input FASTQ.
+        kept_reads:         Reads after removing foldback.
+        removed_reads:      Foldback reads removed.
+    """
+
+    total_breakpoints: int = 0
+    foldback_count: int = 0
+    chimeric_count: int = 0
+    pass_count: int = 0
+    foldback_read_ids: int = 0
+    input_reads: int = 0
+    kept_reads: int = 0
+    removed_reads: int = 0
+
+    def __str__(self) -> str:
+        pct = (
+            f"{self.removed_reads / self.input_reads * 100:.1f}%"
+            if self.input_reads > 0
+            else "N/A"
+        )
+        return (
+            "=== Breakinator Report ===\n"
+            f"  Breakpoints total       : {self.total_breakpoints:,}\n"
+            f"    Foldback              : {self.foldback_count:,}\n"
+            f"    Chimeric              : {self.chimeric_count:,}\n"
+            f"    Pass                  : {self.pass_count:,}\n"
+            f"  ---\n"
+            f"  Input reads             : {self.input_reads:,}\n"
+            f"  Foldback reads removed  : {self.removed_reads:,} ({pct})\n"
+            f"  Reads kept              : {self.kept_reads:,}\n"
+            "=========================="
+        )
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _run_minimap2_for_breakinator(
+    reference: Path,
+    input_fastq: Path,
+    output_sam: Path,
+    threads: int,
+    junc_bed: Path | None = None,
+) -> None:
+    """Run minimap2 to produce a SAM file for Breakinator.
+
+    This is NOT coordinate-sorted — Breakinator reads SAM directly.
+    Uses the same Direct-cDNA splice parameters as the main alignment
+    but outputs SAM instead of piping to samtools sort.
+
+    Args:
+        reference:    Reference genome FASTA.
+        input_fastq:  Raw input FASTQ.
+        output_sam:   Output SAM path.
+        threads:      Number of threads.
+        junc_bed:     Optional junction BED for guided alignment.
+    """
+    minimap2_bin = check_binary("minimap2")
+
+    cmd = [
+        minimap2_bin,
+        "-Y",  # soft-clip with original sequence
+        "-t",
+        str(threads),
+        "-ax",
+        "splice",  # splice-aware
+        "-uf",  # forward strand for Direct-cDNA
+        "-k14",  # k-mer size
+        "--secondary=no",  # no secondary alignments
+    ]
+
+    if junc_bed is not None:
+        cmd.extend(["--junc-bed", str(junc_bed)])
+
+    cmd.extend([str(reference), str(input_fastq)])
+
+    logger.info(f"Running minimap2 for Breakinator: {' '.join(cmd[:6])}...")
+
+    with open(output_sam, "w") as sam_fh:
+        proc = subprocess.run(
+            cmd,
+            stdout=sam_fh,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+
+    if proc.returncode != 0:
+        raise RuntimeError(f"minimap2 failed (exit {proc.returncode}):\n{proc.stderr}")
+
+    logger.info(f"SAM written: {output_sam}")
+
+
+def _run_breakinator(
+    input_sam: Path,
+    output_artifacts: Path,
+    threads: int,
+) -> None:
+    """Run Breakinator on a SAM file.
+
+    Args:
+        input_sam:         Input SAM from minimap2.
+        output_artifacts:  Output tabular artifacts file.
+        threads:           Number of threads.
+    """
+    breakinator_bin = check_binary("breakinator")
+    _check_breakinator_version(breakinator_bin)
+
+    cmd = [
+        breakinator_bin,
+        "-i",
+        str(input_sam),
+        "-t",
+        str(threads),
+        "-o",
+        str(output_artifacts),
+        "--tabular",
+    ]
+
+    logger.info(f"Running Breakinator: {' '.join(cmd[:4])}...")
+
+    proc = subprocess.run(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+
+    if proc.returncode != 0:
+        raise RuntimeError(
+            f"Breakinator failed (exit {proc.returncode}):\n{proc.stderr}"
+        )
+
+    logger.info(f"Artifacts written: {output_artifacts}")
+
+
+def _parse_foldback_ids(artifacts_file: Path) -> tuple[set[str], dict]:
+    """Parse Breakinator tabular output and extract Foldback read IDs.
+
+    Only Foldback reads are removed — Chimeric reads are left for
+    DirectClean's own Rescuer and Homopolymer Filter to handle.
+
+    Args:
+        artifacts_file: Path to Breakinator --tabular output.
+
+    Returns:
+        (foldback_ids, breakpoint_stats) where breakpoint_stats has
+        counts for each classification.
+    """
+    foldback_ids: set[str] = set()
+    stats = {"total": 0, "Foldback": 0, "Chimeric": 0, "Pass": 0}
+
+    with open(artifacts_file) as fh:
+        for line in fh:
+            if line.startswith("#"):
+                continue
+            fields = line.strip().split("\t")
+            if len(fields) < 8:
+                continue
+
+            read_id = fields[6]
+            classification = fields[7]
+
+            stats["total"] += 1
+            if classification in stats:
+                stats[classification] += 1
+
+            if classification == "Foldback":
+                foldback_ids.add(read_id)
+
+    logger.info(
+        f"Breakinator: {stats['total']:,} breakpoints, "
+        f"{len(foldback_ids):,} unique foldback reads"
+    )
+    return foldback_ids, stats
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+class BreakinatorRunner:
+    """End-to-end Breakinator wrapper for foldback removal.
+
+    Runs minimap2 → breakinator → foldback filtering as a single
+    stage in the DirectClean pipeline.
+
+    Usage::
+
+        runner = BreakinatorRunner(
+            reference=Path("genome.fa"),
+            threads=8,
+        )
+        report = runner.run(
+            input_fastq=Path("raw.fastq"),
+            output_fastq=Path("no_foldback.fastq"),
+            work_dir=Path("output/"),
+        )
+
+    Args:
+        reference:  Reference genome FASTA.
+        threads:    Number of threads for minimap2 and breakinator.
+        junc_bed:   Optional junction BED file for guided alignment.
+    """
+
+    def __init__(
+        self,
+        reference: Path,
+        threads: int = 4,
+        junc_bed: Path | None = None,
+    ) -> None:
+        self.reference = Path(reference)
+        self.threads = threads
+        self.junc_bed = Path(junc_bed) if junc_bed is not None else None
+
+    def run(
+        self,
+        input_fastq: Path,
+        output_fastq: Path,
+        work_dir: Path,
+        prefix: str = "directclean",
+    ) -> BreakReport:
+        """Execute the full Breakinator stage.
+
+        Steps:
+            1. minimap2 alignment → SAM
+            2. Breakinator → artifacts.txt
+            3. Parse foldback IDs
+            4. Split FASTQ → kept + removed
+
+        Args:
+            input_fastq:  Raw input FASTQ.
+            output_fastq: Output FASTQ with foldback reads removed.
+            work_dir:     Working directory for intermediate files.
+            prefix:       Filename prefix for intermediates.
+
+        Returns:
+            BreakReport with statistics.
+        """
+        work_dir = Path(work_dir)
+        work_dir.mkdir(parents=True, exist_ok=True)
+
+        report = BreakReport()
+
+        # Intermediate file paths
+        sam_path = work_dir / f"{prefix}.breakinator.sam"
+        artifacts_path = work_dir / f"{prefix}.breakinator_artifacts.txt"
+        removed_path = work_dir / f"{prefix}.foldback_removed.fastq"
+
+        # Step 1: minimap2 → SAM
+        logger.info("Step 1/3: Aligning reads for Breakinator...")
+        _run_minimap2_for_breakinator(
+            reference=self.reference,
+            input_fastq=input_fastq,
+            output_sam=sam_path,
+            threads=self.threads,
+            junc_bed=self.junc_bed,
+        )
+
+        # Step 2: Breakinator → artifacts.txt
+        logger.info("Step 2/3: Running Breakinator...")
+        _run_breakinator(
+            input_sam=sam_path,
+            output_artifacts=artifacts_path,
+            threads=self.threads,
+        )
+
+        # Step 3: Parse foldback IDs and filter FASTQ
+        logger.info("Step 3/3: Filtering foldback reads...")
+        foldback_ids, bp_stats = _parse_foldback_ids(artifacts_path)
+
+        report.total_breakpoints = bp_stats["total"]
+        report.foldback_count = bp_stats["Foldback"]
+        report.chimeric_count = bp_stats["Chimeric"]
+        report.pass_count = bp_stats["Pass"]
+        report.foldback_read_ids = len(foldback_ids)
+
+        # Use our own split_fastq_by_ids — no seqkit dependency
+        kept, removed = split_fastq_by_ids(
+            input_fastq=input_fastq,
+            remove_ids=foldback_ids,
+            output_kept=output_fastq,
+            output_removed=removed_path,
+        )
+
+        report.input_reads = kept + removed
+        report.kept_reads = kept
+        report.removed_reads = removed
+
+        # Clean up large SAM file (user still has artifacts.txt for reference)
+        if sam_path.exists():
+            sam_path.unlink()
+            logger.debug(f"Cleaned up intermediate SAM: {sam_path}")
+
+        logger.info(f"Breakinator stage complete.\n{report}")
+        return report