smftools 0.2.5__py3-none-any.whl → 0.3.0__py3-none-any.whl

smftools/informatics/basecalling.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import subprocess
 
 

smftools/informatics/bed_functions.py

@@ -1,23 +1,134 @@
+from __future__ import annotations
+
 import concurrent.futures
 import os
+import shutil
+import subprocess
 from concurrent.futures import ProcessPoolExecutor
 from pathlib import Path
+from typing import TYPE_CHECKING
 
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-import pybedtools
-import pyBigWig
-import pysam
 
 from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
 
 from ..readwrite import make_dirs
 
 logger = get_logger(__name__)
 
-
-def _bed_to_bigwig(fasta: str, bed: str) -> str:
+if TYPE_CHECKING:
+    import pybedtools as pybedtools_types
+    import pyBigWig as pybigwig_types
+    import pysam as pysam_types
+
+try:
+    import pybedtools
+except Exception:
+    pybedtools = None  # type: ignore
+
+try:
+    import pyBigWig
+except Exception:
+    pyBigWig = None  # type: ignore
+
+try:
+    import pysam
+except Exception:
+    pysam = None  # type: ignore
+
+
+def _require_pybedtools() -> "pybedtools_types":
+    if pybedtools is not None:
+        return pybedtools
+    return require("pybedtools", extra="pybedtools", purpose="bedtools Python backend")
+
+
+def _require_pybigwig() -> "pybigwig_types":
+    if pyBigWig is not None:
+        return pyBigWig
+    return require("pyBigWig", extra="pybigwig", purpose="BigWig Python backend")
+
+
+def _require_pysam() -> "pysam_types":
+    if pysam is not None:
+        return pysam
+    return require("pysam", extra="pysam", purpose="FASTA indexing")
+
+
+def _resolve_backend(
+    backend: str | None, *, tool: str, python_available: bool, cli_name: str
+) -> str:
+    choice = (backend or "auto").strip().lower()
+    if choice not in {"auto", "python", "cli"}:
+        raise ValueError(f"{tool}_backend must be one of: auto, python, cli")
+    if choice == "python":
+        if not python_available:
+            raise RuntimeError(
+                f"{tool}_backend=python requires the Python package to be installed."
+            )
+        return "python"
+    if choice == "cli":
+        if not shutil.which(cli_name):
+            raise RuntimeError(f"{tool}_backend=cli requires {cli_name} in PATH.")
+        return "cli"
+    if shutil.which(cli_name):
+        return "cli"
+    if python_available:
+        return "python"
+    raise RuntimeError(f"Neither Python nor CLI backend is available for {tool}.")
+
+
+def _read_chrom_sizes(chrom_sizes: Path) -> list[tuple[str, int]]:
+    sizes: list[tuple[str, int]] = []
+    with chrom_sizes.open() as f:
+        for line in f:
+            chrom, size = line.split()[:2]
+            sizes.append((chrom, int(size)))
+    return sizes
+
+
+def _ensure_fasta_index(fasta: Path) -> Path:
+    fai = fasta.with_suffix(fasta.suffix + ".fai")
+    if fai.exists():
+        return fai
+    if shutil.which("samtools"):
+        cp = subprocess.run(
+            ["samtools", "faidx", str(fasta)],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        if cp.returncode != 0:
+            raise RuntimeError(f"samtools faidx failed (exit {cp.returncode}):\n{cp.stderr}")
+        return fai
+    if pysam is not None:
+        pysam_mod = _require_pysam()
+        pysam_mod.faidx(str(fasta))
+        return fai
+    raise RuntimeError("FASTA indexing requires pysam or samtools in PATH.")
+
+
+def _ensure_chrom_sizes(fasta: Path) -> Path:
+    fai = _ensure_fasta_index(fasta)
+    chrom_sizes = fasta.with_suffix(".chrom.sizes")
+    if chrom_sizes.exists():
+        return chrom_sizes
+    with fai.open() as f_in, chrom_sizes.open("w") as out:
+        for line in f_in:
+            chrom, size = line.split()[:2]
+            out.write(f"{chrom}\t{size}\n")
+    return chrom_sizes
+
+
+def _bed_to_bigwig(
+    fasta: str,
+    bed: str,
+    *,
+    bedtools_backend: str | None = "auto",
+    bigwig_backend: str | None = "auto",
+) -> str:
     """
     BED → bedGraph → bigWig
     Requires:
@@ -28,40 +139,70 @@ def _bed_to_bigwig(fasta: str, bed: str) -> str:
     fa = Path(fasta)  # path to .fa
     parent = bed.parent
     stem = bed.stem
-    fa_stem = fa.stem
-    fai = parent / f"{fa_stem}.fai"
+    chrom_sizes = _ensure_chrom_sizes(fa)
 
     bedgraph = parent / f"{stem}.bedgraph"
     bigwig = parent / f"{stem}.bw"
 
     # 1) Compute coverage → bedGraph
-    logger.debug(f"[pybedtools] generating coverage bedgraph from {bed}")
-    bt = pybedtools.BedTool(str(bed))
-    # bedtools genomecov -bg
-    coverage = bt.genome_coverage(bg=True, genome=str(fai))
-    coverage.saveas(str(bedgraph))
+    bedtools_choice = _resolve_backend(
+        bedtools_backend,
+        tool="bedtools",
+        python_available=pybedtools is not None,
+        cli_name="bedtools",
+    )
+    if bedtools_choice == "python":
+        logger.debug(f"[pybedtools] generating coverage bedgraph from {bed}")
+        pybedtools_mod = _require_pybedtools()
+        bt = pybedtools_mod.BedTool(str(bed))
+        # bedtools genomecov -bg
+        coverage = bt.genome_coverage(bg=True, genome=str(chrom_sizes))
+        coverage.saveas(str(bedgraph))
+    else:
+        if not shutil.which("bedtools"):
+            raise RuntimeError("bedtools is required but not available in PATH.")
+        cmd = [
+            "bedtools",
+            "genomecov",
+            "-i",
+            str(bed),
+            "-g",
+            str(chrom_sizes),
+            "-bg",
+        ]
+        logger.debug("[bedtools] generating coverage bedgraph: %s", " ".join(cmd))
+        with bedgraph.open("w") as out:
+            cp = subprocess.run(cmd, stdout=out, stderr=subprocess.PIPE, text=True)
+        if cp.returncode != 0:
+            raise RuntimeError(f"bedtools genomecov failed (exit {cp.returncode}):\n{cp.stderr}")
 
     # 2) Convert bedGraph → BigWig via pyBigWig
-    logger.debug(f"[pyBigWig] converting bedgraph → bigwig: {bigwig}")
-
-    # read chrom sizes from the FASTA .fai index
-    chrom_sizes = {}
-    with open(fai) as f:
-        for line in f:
-            fields = line.strip().split("\t")
-            chrom = fields[0]
-            size = int(fields[1])
-            chrom_sizes[chrom] = size
-
-    bw = pyBigWig.open(str(bigwig), "w")
-    bw.addHeader(list(chrom_sizes.items()))
-
-    with open(bedgraph) as f:
-        for line in f:
-            chrom, start, end, coverage = line.strip().split()
-            bw.addEntries(chrom, int(start), ends=int(end), values=float(coverage))
-
-    bw.close()
+    bigwig_choice = _resolve_backend(
+        bigwig_backend,
+        tool="bigwig",
+        python_available=pyBigWig is not None,
+        cli_name="bedGraphToBigWig",
+    )
+    if bigwig_choice == "python":
+        logger.debug(f"[pyBigWig] converting bedgraph → bigwig: {bigwig}")
+        pybigwig_mod = _require_pybigwig()
+        bw = pybigwig_mod.open(str(bigwig), "w")
+        bw.addHeader(_read_chrom_sizes(chrom_sizes))
+
+        with bedgraph.open() as f:
+            for line in f:
+                chrom, start, end, coverage = line.strip().split()
+                bw.addEntries(chrom, int(start), ends=int(end), values=float(coverage))
+
+        bw.close()
+    else:
+        if not shutil.which("bedGraphToBigWig"):
+            raise RuntimeError("bedGraphToBigWig is required but not available in PATH.")
+        cmd = ["bedGraphToBigWig", str(bedgraph), str(chrom_sizes), str(bigwig)]
+        logger.debug("[bedGraphToBigWig] converting bedgraph → bigwig: %s", " ".join(cmd))
+        cp = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+        if cp.returncode != 0:
+            raise RuntimeError(f"bedGraphToBigWig failed (exit {cp.returncode}):\n{cp.stderr}")
 
     logger.debug(f"BigWig written: {bigwig}")
     return str(bigwig)
@@ -113,6 +254,8 @@ def _plot_bed_histograms(
     coordinate_mode : {"one_based","zero_based"}
         One-based, inclusive (your file) vs BED-standard zero-based, half-open.
     """
+    plt = require("matplotlib.pyplot", extra="plotting", purpose="plotting BED histograms")
+
     os.makedirs(plotting_directory, exist_ok=True)
 
     bed_basename = os.path.basename(bed_file).rsplit(".bed", 1)[0]
@@ -167,7 +310,8 @@ def _plot_bed_histograms(
         return np.clip(x, lo, hi)
 
     # Load chromosome order/lengths from FASTA
-    with pysam.FastaFile(fasta) as fa:
+    pysam_mod = _require_pysam()
+    with pysam_mod.FastaFile(fasta) as fa:
         ref_names = list(fa.references)
         ref_lengths = dict(zip(ref_names, fa.lengths))
 
@@ -292,7 +436,17 @@ def _plot_bed_histograms(
     logger.debug("[plot_bed_histograms] Done.")
 
 
-def aligned_BAM_to_bed(aligned_BAM, out_dir, fasta, make_bigwigs, threads=None):
+def aligned_BAM_to_bed(
+    aligned_BAM,
+    out_dir,
+    fasta,
+    make_bigwigs,
+    threads=None,
+    *,
+    samtools_backend: str | None = "auto",
+    bedtools_backend: str | None = "auto",
+    bigwig_backend: str | None = "auto",
+):
     """
     Takes an aligned BAM as input and writes a BED file of reads as output.
     Bed columns are: Record name, start position, end position, read length, read name, mapping quality, read quality.
@@ -318,31 +472,79 @@ def aligned_BAM_to_bed(aligned_BAM, out_dir, fasta, make_bigwigs, threads=None):
 
     logger.debug(f"Creating BED-like file from BAM (with MAPQ and avg base quality): {aligned_BAM}")
 
-    with pysam.AlignmentFile(aligned_BAM, "rb") as bam, open(bed_output, "w") as out:
-        for read in bam.fetch(until_eof=True):
-            if read.is_unmapped:
-                chrom = "*"
-                start1 = 1
-                rl = read.query_length or 0
-                mapq = 0
-            else:
-                chrom = bam.get_reference_name(read.reference_id)
-                # pysam reference_start is 0-based → +1 for 1-based SAM-like start
-                start1 = int(read.reference_start) + 1
-                rl = read.query_length or 0
-                mapq = int(read.mapping_quality)
-
-            # End position in 1-based inclusive coords
-            end1 = start1 + (rl or 0) - 1
-
-            qname = read.query_name
-            quals = read.query_qualities
-            if quals is None or rl == 0:
-                avg_q = float("nan")
-            else:
-                avg_q = float(np.mean(quals))
-
-            out.write(f"{chrom}\t{start1}\t{end1}\t{rl}\t{qname}\t{mapq}\t{avg_q:.3f}\n")
+    backend_choice = _resolve_backend(
+        samtools_backend,
+        tool="samtools",
+        python_available=pysam is not None,
+        cli_name="samtools",
+    )
+    with open(bed_output, "w") as out:
+        if backend_choice == "python":
+            pysam_mod = _require_pysam()
+            with pysam_mod.AlignmentFile(aligned_BAM, "rb") as bam:
+                for read in bam.fetch(until_eof=True):
+                    if read.is_unmapped:
+                        chrom = "*"
+                        start1 = 1
+                        rl = read.query_length or 0
+                        mapq = 0
+                    else:
+                        chrom = bam.get_reference_name(read.reference_id)
+                        # pysam reference_start is 0-based → +1 for 1-based SAM-like start
+                        start1 = int(read.reference_start) + 1
+                        rl = read.query_length or 0
+                        mapq = int(read.mapping_quality)
+
+                    # End position in 1-based inclusive coords
+                    end1 = start1 + (rl or 0) - 1
+
+                    qname = read.query_name
+                    quals = read.query_qualities
+                    if quals is None or rl == 0:
+                        avg_q = float("nan")
+                    else:
+                        avg_q = float(np.mean(quals))
+
+                    out.write(f"{chrom}\t{start1}\t{end1}\t{rl}\t{qname}\t{mapq}\t{avg_q:.3f}\n")
+        else:
+            samtools_view = subprocess.Popen(
+                ["samtools", "view", str(aligned_BAM)],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+            assert samtools_view.stdout is not None
+            for line in samtools_view.stdout:
+                if not line.strip():
+                    continue
+                fields = line.rstrip("\n").split("\t")
+                if len(fields) < 11:
+                    continue
+                qname = fields[0]
+                flag = int(fields[1])
+                chrom = fields[2]
+                pos = int(fields[3])
+                mapq = int(fields[4])
+                seq = fields[9]
+                qual = fields[10]
+                rl = 0 if seq == "*" else len(seq)
+                is_unmapped = bool(flag & 0x4) or chrom == "*"
+                if is_unmapped:
+                    chrom = "*"
+                    start1 = 1
+                    mapq = 0
+                else:
+                    start1 = pos
+                end1 = start1 + (rl or 0) - 1
+                if qual == "*" or rl == 0:
+                    avg_q = float("nan")
+                else:
+                    avg_q = float(np.mean([ord(ch) - 33 for ch in qual]))
+                out.write(f"{chrom}\t{start1}\t{end1}\t{rl}\t{qname}\t{mapq}\t{avg_q:.3f}\n")
+            rc = samtools_view.wait()
+            if rc != 0:
+                stderr = samtools_view.stderr.read() if samtools_view.stderr else ""
+                raise RuntimeError(f"samtools view failed (exit {rc}):\n{stderr}")
 
     logger.debug(f"BED-like file created: {bed_output}")
 
@@ -368,7 +570,15 @@ def aligned_BAM_to_bed(aligned_BAM, out_dir, fasta, make_bigwigs, threads=None):
         futures = []
         futures.append(executor.submit(_plot_bed_histograms, aligned_bed, plotting_dir, fasta))
         if make_bigwigs:
-            futures.append(executor.submit(_bed_to_bigwig, fasta, aligned_bed))
+            futures.append(
+                executor.submit(
+                    _bed_to_bigwig,
+                    fasta,
+                    aligned_bed,
+                    bedtools_backend=bedtools_backend,
+                    bigwig_backend=bigwig_backend,
+                )
+            )
         concurrent.futures.wait(futures)
 
     logger.debug("Processing completed successfully.")

smftools/informatics/binarize_converted_base_identities.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+
 def binarize_converted_base_identities(
     base_identities,
     strand,

smftools/informatics/complement_base_list.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # complement_base_list
 
 

smftools/informatics/converted_BAM_to_adata.py

@@ -1,18 +1,20 @@
+from __future__ import annotations
+
 import gc
-import multiprocessing
+import logging
 import shutil
 import time
 import traceback
 from multiprocessing import Manager, Pool, current_process
 from pathlib import Path
-from typing import Iterable, Optional, Union
+from typing import TYPE_CHECKING, Iterable, Optional, Union
 
 import anndata as ad
 import numpy as np
 import pandas as pd
-import torch
 
-from smftools.logging_utils import get_logger
+from smftools.logging_utils import get_logger, setup_logging
+from smftools.optional_imports import require
 
 from ..readwrite import make_dirs
 from .bam_functions import count_aligned_reads, extract_base_identities
@@ -22,8 +24,10 @@ from .ohe import ohe_batching
 
 logger = get_logger(__name__)
 
-if __name__ == "__main__":
-    multiprocessing.set_start_method("forkserver", force=True)
+if TYPE_CHECKING:
+    import torch
+
+torch = require("torch", extra="torch", purpose="converted BAM processing")
 
 
 def converted_BAM_to_adata(
@@ -40,6 +44,7 @@ def converted_BAM_to_adata(
     deaminase_footprinting: bool = False,
     delete_intermediates: bool = True,
     double_barcoded_path: Path | None = None,
+    samtools_backend: str | None = "auto",
 ) -> tuple[ad.AnnData | None, Path]:
     """Convert BAM files into an AnnData object by binarizing modified base identities.
 
@@ -89,7 +94,9 @@ def converted_BAM_to_adata(
     )
 
     bam_path_list = bam_files
-    logger.info(f"Found {len(bam_files)} BAM files: {bam_files}")
+
+    bam_names = [bam.name for bam in bam_files]
+    logger.info(f"Found {len(bam_files)} BAM files within {split_dir}: {bam_names}")
 
     ## Process Conversion Sites
     max_reference_length, record_FASTA_dict, chromosome_FASTA_dict = process_conversion_sites(
@@ -98,7 +105,7 @@ def converted_BAM_to_adata(
 
     ## Filter BAM Files by Mapping Threshold
     records_to_analyze = filter_bams_by_mapping_threshold(
-        bam_path_list, bam_files, mapping_threshold
+        bam_path_list, bam_files, mapping_threshold, samtools_backend
     )
 
     ## Process BAMs in Parallel
@@ -113,6 +120,7 @@ def converted_BAM_to_adata(
         max_reference_length,
         device,
         deaminase_footprinting,
+        samtools_backend,
     )
 
     final_adata.uns["References"] = {}
@@ -240,14 +248,14 @@ def process_conversion_sites(
     return max_reference_length, record_FASTA_dict, chromosome_FASTA_dict
 
 
-def filter_bams_by_mapping_threshold(bam_path_list, bam_files, mapping_threshold):
+def filter_bams_by_mapping_threshold(bam_path_list, bam_files, mapping_threshold, samtools_backend):
     """Filters BAM files based on mapping threshold."""
     records_to_analyze = set()
 
     for i, bam in enumerate(bam_path_list):
-        aligned_reads, unaligned_reads, record_counts = count_aligned_reads(bam)
+        aligned_reads, unaligned_reads, record_counts = count_aligned_reads(bam, samtools_backend)
         aligned_percent = aligned_reads * 100 / (aligned_reads + unaligned_reads)
-        print(f"{aligned_percent:.2f}% of reads in {bam_files[i]} aligned successfully.")
+        logger.info(f"{aligned_percent:.2f}% of reads in {bam_files[i].name} aligned successfully.")
 
         for record, (count, percent) in record_counts.items():
             if percent >= mapping_threshold:
@@ -267,6 +275,7 @@ def process_single_bam(
     max_reference_length,
     device,
     deaminase_footprinting,
+    samtools_backend,
 ):
     """Worker function to process a single BAM file (must be at top-level for multiprocessing)."""
     adata_list = []
@@ -281,7 +290,7 @@ def process_single_bam(
         # Extract Base Identities
         fwd_bases, rev_bases, mismatch_counts_per_read, mismatch_trend_per_read = (
             extract_base_identities(
-                bam, record, range(current_length), max_reference_length, sequence
+                bam, record, range(current_length), max_reference_length, sequence, samtools_backend
             )
         )
         mismatch_trend_series = pd.Series(mismatch_trend_per_read)
@@ -433,9 +442,13 @@ def worker_function(
     max_reference_length,
     device,
     deaminase_footprinting,
+    samtools_backend,
     progress_queue,
+    log_level,
+    log_file,
 ):
     """Worker function that processes a single BAM and writes the output to an H5AD file."""
+    _ensure_worker_logging(log_level, log_file)
     worker_id = current_process().pid  # Get worker process ID
     sample = bam.stem
 
@@ -471,6 +484,7 @@ def worker_function(
             max_reference_length,
             device,
             deaminase_footprinting,
+            samtools_backend,
         )
 
         if adata is not None:
@@ -501,19 +515,13 @@ def process_bams_parallel(
     max_reference_length,
     device,
     deaminase_footprinting,
+    samtools_backend,
 ):
     """Processes BAM files in parallel, writes each H5AD to disk, and concatenates them at the end."""
     make_dirs(h5_dir)  # Ensure h5_dir exists
 
     logger.info(f"Starting parallel BAM processing with {num_threads} threads...")
-
-    # Ensure macOS uses forkserver to avoid spawning issues
-    try:
-        import multiprocessing
-
-        multiprocessing.set_start_method("forkserver", force=True)
-    except RuntimeError:
-        logger.warning(f"Multiprocessing context already set. Skipping set_start_method.")
+    log_level, log_file = _get_logger_config()
 
     with Manager() as manager:
         progress_queue = manager.Queue()
@@ -534,13 +542,16 @@ def process_bams_parallel(
                         max_reference_length,
                         device,
                         deaminase_footprinting,
+                        samtools_backend,
                         progress_queue,
+                        log_level,
+                        log_file,
                     ),
                 )
                 for i, bam in enumerate(bam_path_list)
             ]
 
-            logger.info(f"Submitted {len(bam_path_list)} BAMs for processing.")
+            logger.info(f"Submitting {len(results)} BAMs for processing.")
 
             # Track completed BAMs
             completed_bams = set()
@@ -550,15 +561,18 @@ def process_bams_parallel(
                     completed_bams.add(processed_bam)
                 except Exception as e:
                     logger.error(f"Timeout waiting for worker process. Possible crash? {e}")
+                    _log_async_result_errors(results, bam_path_list)
 
             pool.close()
             pool.join()  # Ensure all workers finish
 
+    _log_async_result_errors(results, bam_path_list)
+
     # Final Concatenation Step
     h5ad_files = [f for f in h5_dir.iterdir() if f.suffix == ".h5ad"]
 
     if not h5ad_files:
-        logger.debug(f"No valid H5AD files generated. Exiting.")
+        logger.warning(f"No valid H5AD files generated. Exiting.")
         return None
 
     logger.info(f"Concatenating {len(h5ad_files)} H5AD files into final output...")
@@ -568,6 +582,36 @@ def process_bams_parallel(
     return final_adata
 
 
+def _log_async_result_errors(results, bam_path_list):
+    """Log worker failures captured by multiprocessing AsyncResult objects."""
+    for bam, result in zip(bam_path_list, results):
+        if not result.ready():
+            continue
+        try:
+            result.get()
+        except Exception as exc:
+            logger.error("Worker process failed for %s: %s", bam, exc)
+
+
+def _get_logger_config() -> tuple[int, Path | None]:
+    smftools_logger = logging.getLogger("smftools")
+    level = smftools_logger.level
+    if level == logging.NOTSET:
+        level = logging.INFO
+    log_file: Path | None = None
+    for handler in smftools_logger.handlers:
+        if isinstance(handler, logging.FileHandler):
+            log_file = Path(handler.baseFilename)
+            break
+    return level, log_file
+
+
+def _ensure_worker_logging(log_level: int, log_file: Path | None) -> None:
+    smftools_logger = logging.getLogger("smftools")
+    if not smftools_logger.handlers:
+        setup_logging(level=log_level, log_file=log_file)
+
+
 def delete_intermediate_h5ads_and_tmpdir(
     h5_dir: Union[str, Path, Iterable[str], None],
     tmp_dir: Optional[Union[str, Path]] = None,