smftools 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

smftools/informatics/bam_functions.py

@@ -1,24 +1,145 @@
 from __future__ import annotations
 
-from pathlib import Path
+import glob
 import os
+import re
+import shutil
 import subprocess
-import glob
 import time
-from typing import Dict, List, Any, Tuple, Union, Optional, Iterable
-import re
+from collections import Counter, defaultdict, deque
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from itertools import zip_longest
-import pysam
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union
 
 import numpy as np
-import concurrent.futures
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from concurrent.futures import ProcessPoolExecutor
-
 from tqdm import tqdm
-from collections import defaultdict, Counter
 
-from ..readwrite import make_dirs, time_string, date_string
+from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
+
+from ..readwrite import date_string, time_string
+
+if TYPE_CHECKING:
+    import pysam as pysam_types
+
+try:
+    import pysam
+except Exception:
+    pysam = None  # type: ignore
+
+logger = get_logger(__name__)
+
+_PROGRESS_RE = re.compile(r"Output records written:\s*(\d+)")
+_EMPTY_RE = re.compile(r"^\s*$")
+
+
+def _require_pysam() -> "pysam_types":
+    """Return the pysam module or raise if unavailable."""
+    if pysam is not None:
+        return pysam
+    return require("pysam", extra="pysam", purpose="samtools-compatible Python backend")
+
+
+def _resolve_samtools_backend(backend: str | None) -> str:
+    """Resolve backend choice for samtools-compatible operations.
+
+    Args:
+        backend: One of {"auto", "python", "cli"} (case-insensitive).
+
+    Returns:
+        Resolved backend string ("python" or "cli").
+    """
+    choice = (backend or "auto").strip().lower()
+    if choice not in {"auto", "python", "cli"}:
+        raise ValueError("samtools_backend must be one of: auto, python, cli")
+
+    have_pysam = pysam is not None
+    have_samtools = shutil.which("samtools") is not None
+
+    if choice == "python":
+        if not have_pysam:
+            raise RuntimeError("samtools_backend=python requires pysam to be installed.")
+        return "python"
+    if choice == "cli":
+        if not have_samtools:
+            raise RuntimeError("samtools_backend=cli requires samtools in PATH.")
+        return "cli"
+
+    if have_samtools:
+        return "cli"
+    if have_pysam:
+        return "python"
+    raise RuntimeError("Neither pysam nor samtools is available in PATH.")
+
+
+def _has_bam_index(bam_path: Path) -> bool:
+    """Return True if the BAM index exists alongside the BAM."""
+    return (
+        bam_path.with_suffix(bam_path.suffix + ".bai").exists()
+        or Path(str(bam_path) + ".bai").exists()
+    )
+
+
+def _ensure_bam_index(bam_path: Path, backend: str) -> None:
+    """Ensure a BAM index exists, creating one if needed."""
+    if _has_bam_index(bam_path):
+        return
+    if backend == "python":
+        _index_bam_with_pysam(bam_path)
+    else:
+        _index_bam_with_samtools(bam_path)
+
+
+def _parse_idxstats_output(output: str) -> Tuple[int, int, Dict[str, Tuple[int, float]]]:
+    """Parse samtools idxstats output into counts and proportions."""
+    aligned_reads_count = 0
+    unaligned_reads_count = 0
+    record_counts: Dict[str, int] = {}
+    for line in output.splitlines():
+        if not line.strip():
+            continue
+        ref, _length, mapped, unmapped = line.split("\t")[:4]
+        if ref == "*":
+            unaligned_reads_count += int(unmapped)
+            continue
+        mapped_count = int(mapped)
+        aligned_reads_count += mapped_count
+        record_counts[ref] = mapped_count
+
+    proportions: Dict[str, Tuple[int, float]] = {}
+    for ref, count in record_counts.items():
+        proportion = count / aligned_reads_count if aligned_reads_count else 0.0
+        proportions[ref] = (count, proportion)
+
+    return aligned_reads_count, unaligned_reads_count, proportions
+
+
+def _stream_dorado_logs(stderr_iter) -> None:
+    """Stream dorado stderr and emit structured log messages.
+
+    Args:
+        stderr_iter: Iterable of stderr lines.
+    """
+    last_n: int | None = None
+
+    for raw in stderr_iter:
+        line = raw.rstrip("\n")
+        if _EMPTY_RE.match(line):
+            continue
+
+        m = _PROGRESS_RE.search(line)
+        if m:
+            n = int(m.group(1))
+            logger.debug("[dorado] Output records written: %d", n)
+            last_n = n
+            continue
+
+        logger.info("[dorado] %s", line)
+
+    if last_n is not None:
+        logger.info("[dorado] Final output records written: %d", last_n)
+
 
 def _bam_to_fastq_with_pysam(bam_path: Union[str, Path], fastq_path: Union[str, Path]) -> None:
     """
@@ -26,7 +147,14 @@ def _bam_to_fastq_with_pysam(bam_path: Union[str, Path], fastq_path: Union[str,
     """
     bam_path = str(bam_path)
     fastq_path = str(fastq_path)
-    with pysam.AlignmentFile(bam_path, "rb", check_sq=False) as bam, open(fastq_path, "w", encoding="utf-8") as fq:
+
+    logger.debug(f"Converting BAM to FASTQ using _bam_to_fastq_with_pysam")
+
+    pysam_mod = _require_pysam()
+    with (
+        pysam_mod.AlignmentFile(bam_path, "rb", check_sq=False) as bam,
+        open(fastq_path, "w", encoding="utf-8") as fq,
+    ):
         for r in bam.fetch(until_eof=True):
             # Optionally skip secondary/supplementary:
             # if r.is_secondary or r.is_supplementary:
@@ -45,36 +173,98 @@ def _bam_to_fastq_with_pysam(bam_path: Union[str, Path], fastq_path: Union[str,
                 # q is an array/list of ints (Phred scores).
                 # Convert to FASTQ string with Phred+33 encoding,
                 # clamping to sane range [0, 93] to stay in printable ASCII.
-                qual_str = "".join(
-                    chr(min(max(int(qv), 0), 93) + 33)
-                    for qv in q
-                )
+                qual_str = "".join(chr(min(max(int(qv), 0), 93) + 33) for qv in q)
 
             fq.write(f"@{name}\n{seq}\n+\n{qual_str}\n")
 
-def _sort_bam_with_pysam(in_bam: Union[str, Path], out_bam: Union[str, Path], threads: Optional[int] = None) -> None:
+
+def _sort_bam_with_pysam(
+    in_bam: Union[str, Path], out_bam: Union[str, Path], threads: Optional[int] = None
+) -> None:
+    """Sort a BAM file using pysam.
+
+    Args:
+        in_bam: Input BAM path.
+        out_bam: Output BAM path.
+        threads: Optional thread count.
+    """
+    logger.debug(f"Sorting BAM using _sort_bam_with_pysam")
     in_bam, out_bam = str(in_bam), str(out_bam)
     args = []
     if threads:
         args += ["-@", str(threads)]
     args += ["-o", out_bam, in_bam]
-    pysam.sort(*args)
+    pysam_mod = _require_pysam()
+    pysam_mod.sort(*args)
+
 
 def _index_bam_with_pysam(bam_path: Union[str, Path], threads: Optional[int] = None) -> None:
+    """Index a BAM file using pysam.
+
+    Args:
+        bam_path: BAM path to index.
+        threads: Optional thread count.
+    """
     bam_path = str(bam_path)
+    logger.debug(f"Indexing BAM using _index_bam_with_pysam")
+    pysam_mod = _require_pysam()
     # pysam.index supports samtools-style args
     if threads:
-        pysam.index("-@", str(threads), bam_path)
+        pysam_mod.index("-@", str(threads), bam_path)
     else:
-        pysam.index(bam_path)
+        pysam_mod.index(bam_path)
+
+
+def _bam_to_fastq_with_samtools(bam_path: Union[str, Path], fastq_path: Union[str, Path]) -> None:
+    """Convert BAM to FASTQ using samtools."""
+    if not shutil.which("samtools"):
+        raise RuntimeError("samtools is required but not available in PATH.")
+    cmd = ["samtools", "fastq", str(bam_path)]
+    logger.debug("Converting BAM to FASTQ using samtools: %s", " ".join(cmd))
+    with open(fastq_path, "w", encoding="utf-8") as fq:
+        cp = subprocess.run(cmd, stdout=fq, stderr=subprocess.PIPE, text=True)
+    if cp.returncode != 0:
+        raise RuntimeError(f"samtools fastq failed (exit {cp.returncode}):\n{cp.stderr}")
 
-def align_and_sort_BAM(fasta, 
-                       input, 
-                       cfg,
+
+def _sort_bam_with_samtools(
+    in_bam: Union[str, Path], out_bam: Union[str, Path], threads: Optional[int] = None
+) -> None:
+    """Sort a BAM file using samtools."""
+    if not shutil.which("samtools"):
+        raise RuntimeError("samtools is required but not available in PATH.")
+    cmd = ["samtools", "sort", "-o", str(out_bam)]
+    if threads:
+        cmd += ["-@", str(threads)]
+    cmd.append(str(in_bam))
+    logger.debug("Sorting BAM using samtools: %s", " ".join(cmd))
+    cp = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+    if cp.returncode != 0:
+        raise RuntimeError(f"samtools sort failed (exit {cp.returncode}):\n{cp.stderr}")
+
+
+def _index_bam_with_samtools(bam_path: Union[str, Path], threads: Optional[int] = None) -> None:
+    """Index a BAM file using samtools."""
+    if not shutil.which("samtools"):
+        raise RuntimeError("samtools is required but not available in PATH.")
+    cmd = ["samtools", "index"]
+    if threads:
+        cmd += ["-@", str(threads)]
+    cmd.append(str(bam_path))
+    logger.debug("Indexing BAM using samtools: %s", " ".join(cmd))
+    cp = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+    if cp.returncode != 0:
+        raise RuntimeError(f"samtools index failed (exit {cp.returncode}):\n{cp.stderr}")
+
+
+def align_and_sort_BAM(
+    fasta,
+    input,
+    cfg,
 ):
     """
     A wrapper for running dorado aligner and samtools functions
-    
+
     Parameters:
         fasta (str): File path to the reference genome to align to.
         input (str): File path to the basecalled file to align. Works for .bam and .fastq files
@@ -84,60 +274,105 @@ def align_and_sort_BAM(fasta,
         None
             The function writes out files for: 1) An aligned BAM, 2) and aligned_sorted BAM, 3) an index file for the aligned_sorted BAM, 4) A bed file for the aligned_sorted BAM, 5) A text file containing read names in the aligned_sorted BAM
     """
+    logger.debug("Aligning and sorting BAM using align_and_sort_BAM")
     input_basename = input.name
     input_suffix = input.suffix
-    input_as_fastq = input.with_name(input.stem + '.fastq')
+    input_as_fastq = input.with_name(input.stem + ".fastq")
 
     output_path_minus_suffix = cfg.output_directory / input.stem
-    
+
     aligned_BAM = output_path_minus_suffix.with_name(output_path_minus_suffix.stem + "_aligned")
     aligned_output = aligned_BAM.with_suffix(cfg.bam_suffix)
-    aligned_sorted_BAM =aligned_BAM.with_name(aligned_BAM.stem + "_sorted")
+    aligned_sorted_BAM = aligned_BAM.with_name(aligned_BAM.stem + "_sorted")
     aligned_sorted_output = aligned_sorted_BAM.with_suffix(cfg.bam_suffix)
 
     if cfg.threads:
         threads = str(cfg.threads)
     else:
         threads = None
-    
-    if cfg.aligner == 'minimap2':
+
+    samtools_backend = _resolve_samtools_backend(getattr(cfg, "samtools_backend", "auto"))
+
+    if cfg.aligner == "minimap2":
         if not cfg.align_from_bam:
-            print(f"Converting BAM to FASTQ: {input}")
-            _bam_to_fastq_with_pysam(input, input_as_fastq)
-            print(f"Aligning FASTQ to Reference: {input_as_fastq}")
+            logger.debug(f"Converting BAM to FASTQ: {input}")
+            if samtools_backend == "python":
+                _bam_to_fastq_with_pysam(input, input_as_fastq)
+            else:
+                _bam_to_fastq_with_samtools(input, input_as_fastq)
+            logger.debug(f"Aligning FASTQ to Reference: {input_as_fastq}")
             mm_input = input_as_fastq
-        else: 
-            print(f"Aligning BAM to Reference: {input}")
+        else:
+            logger.debug(f"Aligning BAM to Reference: {input}")
             mm_input = input
 
         if threads:
-            minimap_command = ['minimap2'] + cfg.aligner_args + ['-t', threads, str(fasta), str(mm_input)]
+            minimap_command = (
+                ["minimap2"] + cfg.aligner_args + ["-t", threads, str(fasta), str(mm_input)]
+            )
         else:
-            minimap_command = ['minimap2'] + cfg.aligner_args + [str(fasta), str(mm_input)]
-        subprocess.run(minimap_command, stdout=open(aligned_output, "wb"))
+            minimap_command = ["minimap2"] + cfg.aligner_args + [str(fasta), str(mm_input)]
+
+        with open(aligned_output, "wb") as out:
+            proc = subprocess.Popen(
+                minimap_command,
+                stdout=out,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+
+            assert proc.stderr is not None
+            for line in proc.stderr:
+                logger.info("[minimap2] %s", line.rstrip())
+
+            ret = proc.wait()
+            if ret != 0:
+                raise RuntimeError(f"minimap2 failed with exit code {ret}")
 
         if not cfg.align_from_bam:
             os.remove(input_as_fastq)
 
-    elif cfg.aligner == 'dorado':
+    elif cfg.aligner == "dorado":
         # Run dorado aligner
         print(f"Aligning BAM to Reference: {input}")
         if threads:
-            alignment_command = ["dorado", "aligner", "-t", threads] + cfg.aligner_args + [str(fasta), str(input)]
+            alignment_command = (
+                ["dorado", "aligner", "-t", threads] + cfg.aligner_args + [str(fasta), str(input)]
+            )
         else:
             alignment_command = ["dorado", "aligner"] + cfg.aligner_args + [str(fasta), str(input)]
-        subprocess.run(alignment_command, stdout=open(aligned_output, "wb"))
 
+        with open(aligned_output, "wb") as out:
+            proc = subprocess.Popen(
+                alignment_command,
+                stdout=out,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+
+            assert proc.stderr is not None
+            _stream_dorado_logs(proc.stderr)
+            ret = proc.wait()
+
+            if ret != 0:
+                raise RuntimeError(f"dorado failed with exit code {ret}")
     else:
-        print(f'Aligner not recognized: {cfg.aligner}. Choose from minimap2 and dorado')
+        logger.error(f"Aligner not recognized: {cfg.aligner}. Choose from minimap2 and dorado")
         return
-    
-    # --- Sort & Index with pysam ---
-    print(f"[pysam] Sorting: {aligned_output} -> {aligned_sorted_output}")
-    _sort_bam_with_pysam(aligned_output, aligned_sorted_output, threads=threads)
 
-    print(f"[pysam] Indexing: {aligned_sorted_output}")
-    _index_bam_with_pysam(aligned_sorted_output, threads=threads)
+    # --- Sort & Index ---
+    logger.debug(f"Sorting: {aligned_output} -> {aligned_sorted_output}")
+    if samtools_backend == "python":
+        _sort_bam_with_pysam(aligned_output, aligned_sorted_output, threads=threads)
+    else:
+        _sort_bam_with_samtools(aligned_output, aligned_sorted_output, threads=threads)
+
+    logger.debug(f"Indexing: {aligned_sorted_output}")
+    if samtools_backend == "python":
+        _index_bam_with_pysam(aligned_sorted_output, threads=threads)
+    else:
+        _index_bam_with_samtools(aligned_sorted_output, threads=threads)
+
 
 def bam_qc(
     bam_files: Iterable[str | Path],
@@ -147,6 +382,7 @@ def bam_qc(
     stats: bool = True,
     flagstats: bool = True,
     idxstats: bool = True,
+    samtools_backend: str | None = "auto",
 ) -> None:
     """
     QC for BAM/CRAMs: stats, flagstat, idxstats.
@@ -154,132 +390,148 @@ def bam_qc(
     Runs BAMs in parallel (up to `threads`, default serial).
     """
     import subprocess
-    import shutil
 
-    # Try to import pysam once
-    try:
-        import pysam
-        HAVE_PYSAM = True
-    except Exception:
-        HAVE_PYSAM = False
+    logger.debug("Performing BAM QC using bam_qc")
+
+    backend_choice = _resolve_samtools_backend(samtools_backend)
+    have_pysam = backend_choice == "python"
+    pysam_mod = _require_pysam() if have_pysam else None
 
     bam_qc_dir = Path(bam_qc_dir)
     bam_qc_dir.mkdir(parents=True, exist_ok=True)
 
-    bam_files = [Path(b) for b in bam_files]
+    bam_paths = [Path(b) for b in bam_files]
 
     def _has_index(p: Path) -> bool:
-        if p.suffix.lower() == ".bam":
-            bai = p.with_suffix(p.suffix + ".bai")
-            bai_alt = Path(str(p) + ".bai")
-            return bai.exists() or bai_alt.exists()
-        if p.suffix.lower() == ".cram":
-            crai = Path(str(p) + ".crai")
-            return crai.exists()
+        """Return True if a BAM/CRAM index exists for the path."""
+        suf = p.suffix.lower()
+        if suf == ".bam":
+            return p.with_suffix(p.suffix + ".bai").exists() or Path(str(p) + ".bai").exists()
+        if suf == ".cram":
+            return Path(str(p) + ".crai").exists()
         return False
 
     def _ensure_index(p: Path) -> None:
+        """Ensure a BAM/CRAM index exists, creating one if needed."""
         if _has_index(p):
             return
-        if HAVE_PYSAM:
-            # pysam.index supports both BAM & CRAM
-            pysam.index(str(p))
+        if have_pysam:
+            assert pysam_mod is not None
+            pysam_mod.index(str(p))  # supports BAM & CRAM
         else:
             cmd = ["samtools", "index", str(p)]
-            subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            # capture text so errors are readable; raise on failure
+            cp = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+            if cp.returncode != 0:
+                raise RuntimeError(f"samtools index failed (exit {cp.returncode}):\n{cp.stderr}")
 
-    def _run_one(bam: Path) -> Tuple[Path, List[Tuple[str, int]]]:
-        # outputs + return (file, [(task_name, returncode)])
-        results: List[Tuple[str, int]] = []
-        base = bam.stem  # filename without .bam
+    def _run_samtools_to_file(cmd: list[str], out_path: Path, bam: Path, tag: str) -> int:
+        """
+        Stream stderr to logger; write stdout to out_path; return rc; raise with stderr tail on failure.
+        """
+        last_err = deque(maxlen=80)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(out_path, "w") as fh:
+            proc = subprocess.Popen(cmd, stdout=fh, stderr=subprocess.PIPE, text=True)
+            assert proc.stderr is not None
+            for line in proc.stderr:
+                line = line.rstrip()
+                if line:
+                    last_err.append(line)
+                    logger.debug("[%s][%s] %s", tag, bam.name, line)
+            rc = proc.wait()
+
+        if rc != 0:
+            tail = "\n".join(last_err)
+            raise RuntimeError(f"{tag} failed for {bam} (exit {rc}). Stderr tail:\n{tail}")
+        return rc
+
+    def _run_one(bam: Path) -> tuple[Path, list[tuple[str, int]]]:
+        """Run stats/flagstat/idxstats for a single BAM.
+
+        Args:
+            bam: Path to the BAM file.
+
+        Returns:
+            Tuple of (bam_path, list of (stage, return_code)).
+        """
+        import subprocess
+
+        results: list[tuple[str, int]] = []
+        base = bam.stem  # e.g. sample.bam -> sample
         out_stats = bam_qc_dir / f"{base}_stats.txt"
         out_flag = bam_qc_dir / f"{base}_flagstat.txt"
-        out_idx  = bam_qc_dir / f"{base}_idxstats.txt"
+        out_idx = bam_qc_dir / f"{base}_idxstats.txt"
 
-        # Make sure index exists (samtools stats/flagstat don’t require, idxstats does)
+        # Make sure index exists (idxstats requires; stats/flagstat usually don't, but indexing is cheap/useful)
         try:
             _ensure_index(bam)
         except Exception as e:
-            # Still attempt stats/flagstat if requested
-            print(f"[warn] Indexing failed for {bam}: {e}")
-
-        # Choose runner per task
-        def run_stats():
-            if not stats:
-                return
-            if HAVE_PYSAM and hasattr(pysam, "stats"):
-                txt = pysam.stats(str(bam))
+            # Still attempt stats/flagstat if requested; idxstats may fail later if index is required.
+            logger.warning("Indexing failed for %s: %s", bam, e)
+
+        # --- stats ---
+        if stats:
+            if have_pysam:
+                assert pysam_mod is not None
+                if not hasattr(pysam_mod, "stats"):
+                    raise RuntimeError("pysam.stats is unavailable in this pysam build.")
+                txt = pysam_mod.stats(str(bam))
                 out_stats.write_text(txt)
                 results.append(("stats(pysam)", 0))
             else:
                 cmd = ["samtools", "stats", str(bam)]
-                with open(out_stats, "w") as fh:
-                    cp = subprocess.run(cmd, stdout=fh, stderr=subprocess.PIPE)
-                results.append(("stats(samtools)", cp.returncode))
-                if cp.returncode != 0:
-                    raise RuntimeError(cp.stderr.decode(errors="replace"))
-
-        def run_flagstat():
-            if not flagstats:
-                return
-            if HAVE_PYSAM and hasattr(pysam, "flagstat"):
-                txt = pysam.flagstat(str(bam))
+                rc = _run_samtools_to_file(cmd, out_stats, bam, "samtools stats")
+                results.append(("stats(samtools)", rc))
+
+        # --- flagstat ---
+        if flagstats:
+            if have_pysam:
+                assert pysam_mod is not None
+                if not hasattr(pysam_mod, "flagstat"):
+                    raise RuntimeError("pysam.flagstat is unavailable in this pysam build.")
+                txt = pysam_mod.flagstat(str(bam))
                 out_flag.write_text(txt)
                 results.append(("flagstat(pysam)", 0))
             else:
                 cmd = ["samtools", "flagstat", str(bam)]
-                with open(out_flag, "w") as fh:
-                    cp = subprocess.run(cmd, stdout=fh, stderr=subprocess.PIPE)
-                results.append(("flagstat(samtools)", cp.returncode))
-                if cp.returncode != 0:
-                    raise RuntimeError(cp.stderr.decode(errors="replace"))
-
-        def run_idxstats():
-            if not idxstats:
-                return
-            if HAVE_PYSAM and hasattr(pysam, "idxstats"):
-                txt = pysam.idxstats(str(bam))
+                rc = _run_samtools_to_file(cmd, out_flag, bam, "samtools flagstat")
+                results.append(("flagstat(samtools)", rc))
+
+        # --- idxstats ---
+        if idxstats:
+            if have_pysam:
+                assert pysam_mod is not None
+                if not hasattr(pysam_mod, "idxstats"):
+                    raise RuntimeError("pysam.idxstats is unavailable in this pysam build.")
+                txt = pysam_mod.idxstats(str(bam))
                 out_idx.write_text(txt)
                 results.append(("idxstats(pysam)", 0))
             else:
                 cmd = ["samtools", "idxstats", str(bam)]
-                with open(out_idx, "w") as fh:
-                    cp = subprocess.run(cmd, stdout=fh, stderr=subprocess.PIPE)
-                results.append(("idxstats(samtools)", cp.returncode))
-                if cp.returncode != 0:
-                    raise RuntimeError(cp.stderr.decode(errors="replace"))
-
-        # Sanity: ensure samtools exists if pysam missing
-        if not HAVE_PYSAM:
-            if not shutil.which("samtools"):
-                raise RuntimeError("Neither pysam nor samtools is available in PATH.")
-
-        # Execute tasks (serial per file; parallelized across files)
-        run_stats()
-        run_flagstat()
-        run_idxstats()
+                rc = _run_samtools_to_file(cmd, out_idx, bam, "samtools idxstats")
+                results.append(("idxstats(samtools)", rc))
+
         return bam, results
 
-    # Parallel across BAMs
     max_workers = int(threads) if threads and int(threads) > 0 else 1
-    futures = []
-    with ThreadPoolExecutor(max_workers=max_workers) as ex:
-        for b in bam_files:
-            futures.append(ex.submit(_run_one, b))
 
-        for fut in as_completed(futures):
+    with ThreadPoolExecutor(max_workers=max_workers) as ex:
+        futs = [ex.submit(_run_one, b) for b in bam_paths]
+        for fut in as_completed(futs):
             try:
                 bam, res = fut.result()
                 summary = ", ".join(f"{name}:{rc}" for name, rc in res) or "no-op"
-                print(f"[qc] {bam.name}: {summary}")
+                logger.info("[qc] %s: %s", bam.name, summary)
             except Exception as e:
-                print(f"[error] QC failed: {e}")
+                logger.exception("QC failed: %s", e)
+
+    if modality not in {"conversion", "direct", "deaminase"}:
+        logger.warning("Unknown modality '%s', continuing.", modality)
 
-    # Placeholders to keep your signature stable
-    if modality not in {"conversion", "direct"}:
-        print(f"[warn] Unknown modality '{modality}', continuing.")
+    logger.info("QC processing completed.")
 
-    print("QC processing completed.")
 
 def concatenate_fastqs_to_bam(
     fastq_files: List[Union[str, Tuple[str, str], Path, Tuple[Path, Path]]],
@@ -290,6 +542,8 @@ def concatenate_fastqs_to_bam(
     rg_sample_field: Optional[str] = None,
     progress: bool = True,
     auto_pair: bool = True,
+    gzip_suffixes: Tuple[str, ...] = (".gz", ".gzip"),
+    samtools_backend: str | None = "auto",
 ) -> Dict[str, Any]:
     """
     Concatenate FASTQ(s) into an **unaligned** BAM. Supports single-end and paired-end.
@@ -312,6 +566,10 @@ def concatenate_fastqs_to_bam(
         Show tqdm progress bars.
     auto_pair : bool
         Auto-pair R1/R2 based on filename patterns if given a flat list.
+    gzip_suffixes : tuple[str, ...]
+        Suffixes treated as gzip-compressed FASTQ files.
+    samtools_backend : str | None
+        Backend selection for samtools-compatible operations (auto|python|cli).
 
     Returns
     -------
@@ -326,12 +584,30 @@ def concatenate_fastqs_to_bam(
         """
         name = p.name
         lowers = name.lower()
-        for ext in (".fastq.gz", ".fq.gz", ".fastq.bz2", ".fq.bz2", ".fastq.xz", ".fq.xz", ".fastq", ".fq"):
+        gzip_exts = tuple(s.lower() for s in gzip_suffixes)
+        for ext in (
+            *(f".fastq{suf}" for suf in gzip_exts),
+            *(f".fq{suf}" for suf in gzip_exts),
+            ".fastq.bz2",
+            ".fq.bz2",
+            ".fastq.xz",
+            ".fq.xz",
+            ".fastq",
+            ".fq",
+        ):
             if lowers.endswith(ext):
                 return name[: -len(ext)]
         return p.stem  # fallback: remove last suffix only
 
     def _extract_barcode_from_filename(p: Path) -> str:
+        """Extract a barcode token from a FASTQ filename.
+
+        Args:
+            p: FASTQ path.
+
+        Returns:
+            Barcode token string.
+        """
         stem = _strip_fastq_ext(p)
         if "_" in stem:
             token = stem.split("_")[-1]
@@ -340,10 +616,18 @@ def concatenate_fastqs_to_bam(
         return stem
 
     def _classify_read_token(stem: str) -> Tuple[Optional[str], Optional[int]]:
+        """Classify a FASTQ filename stem into (prefix, read_number).
+
+        Args:
+            stem: Filename stem.
+
+        Returns:
+            Tuple of (prefix, read_number) or (None, None) if not matched.
+        """
         # return (prefix, readnum) if matches; else (None, None)
         patterns = [
-            r"(?i)(.*?)[._-]r?([12])$",        # prefix_R1 / prefix.r2 / prefix-1
-            r"(?i)(.*?)[._-]read[_-]?([12])$", # prefix_read1
+            r"(?i)(.*?)[._-]r?([12])$",  # prefix_R1 / prefix.r2 / prefix-1
+            r"(?i)(.*?)[._-]read[_-]?([12])$",  # prefix_read1
         ]
         for pat in patterns:
             m = re.match(pat, stem)
@@ -352,6 +636,14 @@ def concatenate_fastqs_to_bam(
         return None, None
 
     def _pair_by_filename(paths: List[Path]) -> Tuple[List[Tuple[Path, Path]], List[Path]]:
+        """Pair FASTQ files based on filename conventions.
+
+        Args:
+            paths: FASTQ paths to pair.
+
+        Returns:
+            Tuple of (paired list, leftover list).
+        """
         pref_map: Dict[str, Dict[int, Path]] = {}
         unpaired: List[Path] = []
         for pth in paths:
@@ -373,11 +665,59 @@ def concatenate_fastqs_to_bam(
         return pairs, leftovers
 
     def _fastq_iter(p: Path):
+        """Yield FASTQ records using pysam.FastxFile.
+
+        Args:
+            p: FASTQ path.
+
+        Yields:
+            Pysam Fastx records.
+        """
         # pysam.FastxFile handles compressed extensions transparently
-        with pysam.FastxFile(str(p)) as fx:
+        pysam_mod = _require_pysam()
+        with pysam_mod.FastxFile(str(p)) as fx:
             for rec in fx:
                 yield rec  # rec.name, rec.sequence, rec.quality
 
+    def _fastq_iter_plain(p: Path) -> Iterable[Tuple[str, str, str]]:
+        """Yield FASTQ records from plain-text parsing.
+
+        Args:
+            p: FASTQ path.
+
+        Yields:
+            Tuple of (name, sequence, quality).
+        """
+        import bz2
+        import gzip
+        import lzma
+
+        lowers = p.name.lower()
+        if any(lowers.endswith(suf) for suf in (s.lower() for s in gzip_suffixes)):
+            handle = gzip.open(p, "rt", encoding="utf-8")
+        elif lowers.endswith(".bz2"):
+            handle = bz2.open(p, "rt", encoding="utf-8")
+        elif lowers.endswith(".xz"):
+            handle = lzma.open(p, "rt", encoding="utf-8")
+        else:
+            handle = p.open("r", encoding="utf-8")
+
+        with handle as fh:
+            while True:
+                header = fh.readline()
+                if not header:
+                    break
+                seq = fh.readline()
+                fh.readline()
+                qual = fh.readline()
+                if not qual:
+                    break
+                name = header.strip()
+                if name.startswith("@"):
+                    name = name[1:]
+                name = name.split()[0]
+                yield name, seq.strip(), qual.strip()
+
     def _make_unaligned_segment(
         name: str,
         seq: str,
@@ -386,11 +726,25 @@ def concatenate_fastqs_to_bam(
         read1: bool,
         read2: bool,
     ) -> pysam.AlignedSegment:
-        a = pysam.AlignedSegment()
+        """Construct an unaligned pysam.AlignedSegment.
+
+        Args:
+            name: Read name.
+            seq: Read sequence.
+            qual: FASTQ quality string.
+            bc: Barcode string.
+            read1: Whether this is read 1.
+            read2: Whether this is read 2.
+
+        Returns:
+            Unaligned pysam.AlignedSegment.
+        """
+        pysam_mod = _require_pysam()
+        a = pysam_mod.AlignedSegment()
         a.query_name = name
         a.query_sequence = seq
         if qual is not None:
-            a.query_qualities = pysam.qualitystring_to_array(qual)
+            a.query_qualities = pysam_mod.qualitystring_to_array(qual)
         a.is_unmapped = True
         a.is_paired = read1 or read2
         a.is_read1 = read1
@@ -406,8 +760,51 @@ def concatenate_fastqs_to_bam(
             a.set_tag("RG", str(bc), value_type="Z")
         return a
 
+    def _write_sam_line(
+        handle,
+        name: str,
+        seq: str,
+        qual: str,
+        bc: str,
+        *,
+        read1: bool,
+        read2: bool,
+        add_read_group: bool,
+    ) -> None:
+        """Write a single unaligned SAM record to a text stream."""
+        if read1:
+            flag = 77
+        elif read2:
+            flag = 141
+        else:
+            flag = 4
+        tags = [f"{barcode_tag}:Z:{bc}"]
+        if add_read_group:
+            tags.append(f"RG:Z:{bc}")
+        tag_str = "\t".join(tags)
+        if not qual:
+            qual = "*"
+        line = "\t".join(
+            [
+                name,
+                str(flag),
+                "*",
+                "0",
+                "0",
+                "*",
+                "*",
+                "0",
+                "0",
+                seq,
+                qual,
+                tag_str,
+            ]
+        )
+        handle.write(f"{line}\n")
+
     # ---------- normalize inputs to Path ----------
     def _to_path_pair(x) -> Tuple[Path, Path]:
+        """Convert a tuple of path-like objects to Path instances."""
         a, b = x
         return Path(a), Path(b)
 
@@ -450,7 +847,10 @@ def concatenate_fastqs_to_bam(
     # ---------- BAM header ----------
     header = {"HD": {"VN": "1.6", "SO": "unknown"}, "SQ": []}
     if add_read_group:
-        header["RG"] = [{"ID": bc, **({"SM": rg_sample_field} if rg_sample_field else {})} for bc in barcodes_in_order]
+        header["RG"] = [
+            {"ID": bc, **({"SM": rg_sample_field} if rg_sample_field else {})}
+            for bc in barcodes_in_order
+        ]
     header.setdefault("PG", []).append(
         {"ID": "concat-fastq", "PN": "concatenate_fastqs_to_bam", "VN": "1"}
     )
@@ -462,7 +862,29 @@ def concatenate_fastqs_to_bam(
     singletons_written = 0
 
     # ---------- write BAM ----------
-    with pysam.AlignmentFile(str(output_bam), "wb", header=header) as bam_out:
+    backend_choice = _resolve_samtools_backend(samtools_backend)
+    if backend_choice == "python":
+        pysam_mod = _require_pysam()
+        bam_out_ctx = pysam_mod.AlignmentFile(str(output_bam), "wb", header=header)
+    else:
+        cmd = ["samtools", "view", "-b", "-o", str(output_bam), "-"]
+        logger.debug("Writing BAM using samtools: %s", " ".join(cmd))
+        bam_out_ctx = subprocess.Popen(
+            cmd, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
+        )
+        assert bam_out_ctx.stdin is not None
+        header_lines = ["@HD\tVN:1.6\tSO:unknown"]
+        if add_read_group:
+            for bc in barcodes_in_order:
+                rg_fields = [f"ID:{bc}"]
+                if rg_sample_field:
+                    rg_fields.append(f"SM:{rg_sample_field}")
+                rg_body = "\t".join(rg_fields)
+                header_lines.append(f"@RG\t{rg_body}")
+        header_lines.append("@PG\tID:concat-fastq\tPN:concatenate_fastqs_to_bam\tVN:1")
+        bam_out_ctx.stdin.write("\n".join(header_lines) + "\n")
+
+    try:
         # Paired
         it_pairs = explicit_pairs
         if progress and it_pairs:
@@ -472,30 +894,83 @@ def concatenate_fastqs_to_bam(
                 raise FileNotFoundError(f"Paired file missing: {r1_path} or {r2_path}")
             bc = per_path_barcode.get(r1_path) or per_path_barcode.get(r2_path) or "barcode"
 
-            it1 = _fastq_iter(r1_path)
-            it2 = _fastq_iter(r2_path)
+            if backend_choice == "python":
+                it1 = _fastq_iter(r1_path)
+                it2 = _fastq_iter(r2_path)
+            else:
+                it1 = _fastq_iter_plain(r1_path)
+                it2 = _fastq_iter_plain(r2_path)
 
             for rec1, rec2 in zip_longest(it1, it2, fillvalue=None):
+
                 def _clean(n: Optional[str]) -> Optional[str]:
+                    """Normalize FASTQ read names by trimming read suffixes."""
                     if n is None:
                         return None
                     return re.sub(r"(?:/1$|/2$|\s[12]$)", "", n)
 
                 name = (
-                    _clean(getattr(rec1, "name", None))
-                    or _clean(getattr(rec2, "name", None))
-                    or getattr(rec1, "name", None)
-                    or getattr(rec2, "name", None)
+                    _clean(getattr(rec1, "name", None) if backend_choice == "python" else rec1[0])
+                    if rec1 is not None
+                    else None
                 )
+                if name is None:
+                    name = (
+                        _clean(
+                            getattr(rec2, "name", None) if backend_choice == "python" else rec2[0]
+                        )
+                        if rec2 is not None
+                        else None
+                    )
+                if name is None:
+                    name = (
+                        getattr(rec1, "name", None)
+                        if backend_choice == "python" and rec1 is not None
+                        else (rec1[0] if rec1 is not None else None)
+                    )
+                if name is None:
+                    name = (
+                        getattr(rec2, "name", None)
+                        if backend_choice == "python" and rec2 is not None
+                        else (rec2[0] if rec2 is not None else None)
+                    )
 
                 if rec1 is not None:
-                    a1 = _make_unaligned_segment(name, rec1.sequence, rec1.quality, bc, read1=True, read2=False)
-                    bam_out.write(a1)
+                    if backend_choice == "python":
+                        a1 = _make_unaligned_segment(
+                            name, rec1.sequence, rec1.quality, bc, read1=True, read2=False
+                        )
+                        bam_out_ctx.write(a1)
+                    else:
+                        _write_sam_line(
+                            bam_out_ctx.stdin,
+                            name,
+                            rec1[1],
+                            rec1[2],
+                            bc,
+                            read1=True,
+                            read2=False,
+                            add_read_group=add_read_group,
+                        )
                     per_file_counts[r1_path] = per_file_counts.get(r1_path, 0) + 1
                     total_written += 1
                 if rec2 is not None:
-                    a2 = _make_unaligned_segment(name, rec2.sequence, rec2.quality, bc, read1=False, read2=True)
-                    bam_out.write(a2)
+                    if backend_choice == "python":
+                        a2 = _make_unaligned_segment(
+                            name, rec2.sequence, rec2.quality, bc, read1=False, read2=True
+                        )
+                        bam_out_ctx.write(a2)
+                    else:
+                        _write_sam_line(
+                            bam_out_ctx.stdin,
+                            name,
+                            rec2[1],
+                            rec2[2],
+                            bc,
+                            read1=False,
+                            read2=True,
+                            add_read_group=add_read_group,
+                        )
                     per_file_counts[r2_path] = per_file_counts.get(r2_path, 0) + 1
                     total_written += 1
 
@@ -515,12 +990,40 @@ def concatenate_fastqs_to_bam(
             if not pth.exists():
                 raise FileNotFoundError(pth)
             bc = per_path_barcode.get(pth, "barcode")
-            for rec in _fastq_iter(pth):
-                a = _make_unaligned_segment(rec.name, rec.sequence, rec.quality, bc, read1=False, read2=False)
-                bam_out.write(a)
+            if backend_choice == "python":
+                iterator = _fastq_iter(pth)
+            else:
+                iterator = _fastq_iter_plain(pth)
+            for rec in iterator:
+                if backend_choice == "python":
+                    a = _make_unaligned_segment(
+                        rec.name, rec.sequence, rec.quality, bc, read1=False, read2=False
+                    )
+                    bam_out_ctx.write(a)
+                else:
+                    _write_sam_line(
+                        bam_out_ctx.stdin,
+                        rec[0],
+                        rec[1],
+                        rec[2],
+                        bc,
+                        read1=False,
+                        read2=False,
+                        add_read_group=add_read_group,
+                    )
                 per_file_counts[pth] = per_file_counts.get(pth, 0) + 1
                 total_written += 1
                 singletons_written += 1
+    finally:
+        if backend_choice == "python":
+            bam_out_ctx.close()
+        else:
+            if bam_out_ctx.stdin is not None:
+                bam_out_ctx.stdin.close()
+            rc = bam_out_ctx.wait()
+            if rc != 0:
+                stderr = bam_out_ctx.stderr.read() if bam_out_ctx.stderr else ""
+                raise RuntimeError(f"samtools view failed (exit {rc}):\n{stderr}")
 
     return {
         "total_reads": total_written,
@@ -530,43 +1033,61 @@ def concatenate_fastqs_to_bam(
         "barcodes": barcodes_in_order,
     }
 
-def count_aligned_reads(bam_file):
+
+def count_aligned_reads(bam_file, samtools_backend: str | None = "auto"):
     """
     Counts the number of aligned reads in a bam file that map to each reference record.
-    
+
     Parameters:
         bam_file (str): A string representing the path to an aligned BAM file.
-    
+
     Returns:
        aligned_reads_count (int): The total number or reads aligned in the BAM.
        unaligned_reads_count (int): The total number of reads not aligned in the BAM.
        record_counts (dict): A dictionary keyed by reference record instance that points toa tuple containing the total reads mapped to the record and the fraction of mapped reads which map to the record.
 
     """
-    print('{0}: Counting aligned reads in BAM > {1}'.format(time_string(), bam_file))
+    logger.info("Counting aligned reads in BAM > {}".format(bam_file.name))
+    backend_choice = _resolve_samtools_backend(samtools_backend)
     aligned_reads_count = 0
     unaligned_reads_count = 0
-    # Make a dictionary, keyed by the reference_name of reference chromosome that points to an integer number of read counts mapped to the chromosome, as well as the proportion of mapped reads in that chromosome
-    record_counts = defaultdict(int)
-
-    with pysam.AlignmentFile(str(bam_file), "rb") as bam:
-        total_reads = bam.mapped + bam.unmapped
-        # Iterate over reads to get the total mapped read counts and the reads that map to each reference
-        for read in tqdm(bam, desc='Counting aligned reads in BAM', total=total_reads):
-            if read.is_unmapped:
-                unaligned_reads_count += 1
-            else:
-                aligned_reads_count += 1
-                record_counts[read.reference_name] += 1  # Automatically increments if key exists, adds if not
 
-        # reformat the dictionary to contain read counts mapped to the reference, as well as the proportion of mapped reads in reference
-        for reference in record_counts:
-            proportion_mapped_reads_in_record = record_counts[reference] / aligned_reads_count
-            record_counts[reference] = (record_counts[reference], proportion_mapped_reads_in_record)
+    if backend_choice == "python":
+        pysam_mod = _require_pysam()
+        record_counts = defaultdict(int)
+        with pysam_mod.AlignmentFile(str(bam_file), "rb") as bam:
+            total_reads = bam.mapped + bam.unmapped
+            # Iterate over reads to get the total mapped read counts and the reads that map to each reference
+            for read in bam:
+                if read.is_unmapped:
+                    unaligned_reads_count += 1
+                else:
+                    aligned_reads_count += 1
+                    record_counts[read.reference_name] += (
+                        1  # Automatically increments if key exists, adds if not
+                    )
+
+            # reformat the dictionary to contain read counts mapped to the reference, as well as the proportion of mapped reads in reference
+            for reference in record_counts:
+                proportion_mapped_reads_in_record = record_counts[reference] / aligned_reads_count
+                record_counts[reference] = (
+                    record_counts[reference],
+                    proportion_mapped_reads_in_record,
+                )
+        return aligned_reads_count, unaligned_reads_count, dict(record_counts)
+
+    bam_path = Path(bam_file)
+    _ensure_bam_index(bam_path, backend_choice)
+    cmd = ["samtools", "idxstats", str(bam_path)]
+    cp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    if cp.returncode != 0:
+        raise RuntimeError(f"samtools idxstats failed (exit {cp.returncode}):\n{cp.stderr}")
+    return _parse_idxstats_output(cp.stdout)
 
-    return aligned_reads_count, unaligned_reads_count, dict(record_counts)
 
-def demux_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix, barcode_kit, barcode_both_ends, trim, threads):
+def demux_and_index_BAM(
+    aligned_sorted_BAM, split_dir, bam_suffix, barcode_kit, barcode_both_ends, trim, threads
+):
     """
     A wrapper function for splitting BAMS and indexing them.
     Parameters:
@@ -577,11 +1098,12 @@ def demux_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix, barcode_kit,
         barcode_both_ends (bool): Whether to require both ends to be barcoded.
         trim (bool): Whether to trim off barcodes after demultiplexing.
         threads (int): Number of threads to use.
-    
+
     Returns:
         bam_files (list): List of split BAM file path strings
             Splits an input BAM file on barcode value and makes a BAM index file.
     """
+
     input_bam = aligned_sorted_BAM.with_suffix(bam_suffix)
     command = ["dorado", "demux", "--kit-name", barcode_kit]
     if barcode_both_ends:
@@ -594,25 +1116,37 @@ def demux_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix, barcode_kit,
         pass
     command += ["--emit-summary", "--sort-bam", "--output-dir", str(split_dir)]
     command.append(str(input_bam))
-    command_string = ' '.join(command)
-    print(f"Running: {command_string}")
-    subprocess.run(command)
+    command_string = " ".join(command)
+    logger.info("Running dorado demux: %s", " ".join(command))
+
+    proc = subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+
+    assert proc.stderr is not None
+    _stream_dorado_logs(proc.stderr)
+    rc = proc.wait()
+
+    if rc != 0:
+        raise RuntimeError(f"dorado demux failed with exit code {rc}")
 
     bam_files = sorted(
-        p for p in split_dir.glob(f"*{bam_suffix}")
-        if p.is_file() and p.suffix == bam_suffix
+        p for p in split_dir.glob(f"*{bam_suffix}") if p.is_file() and p.suffix == bam_suffix
     )
 
     if not bam_files:
         raise FileNotFoundError(f"No BAM files found in {split_dir} with suffix {bam_suffix}")
-    
+
     # ---- Optional renaming with prefix ----
     renamed_bams = []
     prefix = "de" if barcode_both_ends else "se"
 
     for bam in bam_files:
         bam = Path(bam)
-        bai = bam.with_suffix(bam_suffix + ".bai")   # dorado’s sorting produces .bam.bai
+        bai = bam.with_suffix(bam_suffix + ".bai")  # dorado’s sorting produces .bam.bai
 
         if prefix:
             new_name = f"{prefix}_{bam.name}"
@@ -628,10 +1162,18 @@ def demux_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix, barcode_kit,
             bai.rename(new_bai)
 
         renamed_bams.append(new_bam)
-    
+
     return renamed_bams
 
-def extract_base_identities(bam_file, chromosome, positions, max_reference_length, sequence):
+
+def extract_base_identities(
+    bam_file,
+    chromosome,
+    positions,
+    max_reference_length,
+    sequence,
+    samtools_backend: str | None = "auto",
+):
     """
     Efficiently extracts base identities from mapped reads with reference coordinates.
 
@@ -646,38 +1188,95 @@ def extract_base_identities(bam_file, chromosome, positions, max_reference_lengt
         dict: Base identities from forward mapped reads.
         dict: Base identities from reverse mapped reads.
     """
+    logger.debug("Extracting nucleotide identities for each read using extract_base_identities")
     timestamp = time.strftime("[%Y-%m-%d %H:%M:%S]")
 
     positions = set(positions)
-    fwd_base_identities = defaultdict(lambda: np.full(max_reference_length, 'N', dtype='<U1'))
-    rev_base_identities = defaultdict(lambda: np.full(max_reference_length, 'N', dtype='<U1'))
+    fwd_base_identities = defaultdict(lambda: np.full(max_reference_length, "N", dtype="<U1"))
+    rev_base_identities = defaultdict(lambda: np.full(max_reference_length, "N", dtype="<U1"))
     mismatch_counts_per_read = defaultdict(lambda: defaultdict(Counter))
 
-    #print(f"{timestamp} Reading reads from {chromosome} BAM file: {bam_file}")
-    with pysam.AlignmentFile(str(bam_file), "rb") as bam:
-        total_reads = bam.mapped
-        ref_seq = sequence.upper()
-        for read in bam.fetch(chromosome):
-            if not read.is_mapped:
-                continue  # Skip unmapped reads
+    backend_choice = _resolve_samtools_backend(samtools_backend)
+    ref_seq = sequence.upper()
+
+    if backend_choice == "python":
+        logger.debug("Extracting base identities using python")
+        pysam_mod = _require_pysam()
+        # print(f"{timestamp} Reading reads from {chromosome} BAM file: {bam_file}")
+        with pysam_mod.AlignmentFile(str(bam_file), "rb") as bam:
+            total_reads = bam.mapped
+            for read in bam.fetch(chromosome):
+                if not read.is_mapped:
+                    continue  # Skip unmapped reads
 
-            read_name = read.query_name
-            query_sequence = read.query_sequence
-            base_dict = rev_base_identities if read.is_reverse else fwd_base_identities
+                read_name = read.query_name
+                query_sequence = read.query_sequence
+                base_dict = rev_base_identities if read.is_reverse else fwd_base_identities
 
-            # Use get_aligned_pairs directly with positions filtering
-            aligned_pairs = read.get_aligned_pairs(matches_only=True)
+                # Use get_aligned_pairs directly with positions filtering
+                aligned_pairs = read.get_aligned_pairs(matches_only=True)
 
-            for read_position, reference_position in aligned_pairs:
-                if reference_position in positions:
+                for read_position, reference_position in aligned_pairs:
                     read_base = query_sequence[read_position]
                     ref_base = ref_seq[reference_position]
+                    if reference_position in positions:
+                        base_dict[read_name][reference_position] = read_base
 
-                    base_dict[read_name][reference_position] = read_base
-
-                # Track mismatches (excluding Ns)
-                if read_base != ref_base and read_base != 'N' and ref_base != 'N':
+                    # Track mismatches (excluding Ns)
+                    if read_base != ref_base and read_base != "N" and ref_base != "N":
+                        mismatch_counts_per_read[read_name][ref_base][read_base] += 1
+    else:
+        bam_path = Path(bam_file)
+        logger.debug("Extracting base identities using samtools")
+        _ensure_bam_index(bam_path, backend_choice)
+
+        def _iter_aligned_pairs(cigar: str, start: int) -> Iterable[Tuple[int, int]]:
+            qpos = 0
+            rpos = start
+            for length_str, op in re.findall(r"(\d+)([MIDNSHP=XB])", cigar):
+                length = int(length_str)
+                if op in {"M", "=", "X"}:
+                    for _ in range(length):
+                        yield qpos, rpos
+                        qpos += 1
+                        rpos += 1
+                elif op in {"I", "S"}:
+                    qpos += length
+                elif op in {"D", "N"}:
+                    rpos += length
+                elif op in {"H", "P"}:
+                    continue
+
+        cmd = ["samtools", "view", "-F", "4", str(bam_path), chromosome]
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        assert proc.stdout is not None
+        for line in proc.stdout:
+            if not line.strip() or line.startswith("@"):
+                continue
+            fields = line.rstrip("\n").split("\t")
+            if len(fields) < 11:
+                continue
+            read_name = fields[0]
+            flag = int(fields[1])
+            pos = int(fields[3])
+            cigar = fields[5]
+            query_sequence = fields[9]
+            if cigar == "*" or query_sequence == "*":
+                continue
+            base_dict = rev_base_identities if (flag & 16) else fwd_base_identities
+            for read_pos, ref_pos in _iter_aligned_pairs(cigar, pos - 1):
+                if read_pos >= len(query_sequence) or ref_pos >= len(ref_seq):
+                    continue
+                read_base = query_sequence[read_pos]
+                ref_base = ref_seq[ref_pos]
+                if ref_pos in positions:
+                    base_dict[read_name][ref_pos] = read_base
+                if read_base != ref_base and read_base != "N" and ref_base != "N":
                     mismatch_counts_per_read[read_name][ref_base][read_base] += 1
+        rc = proc.wait()
+        if rc != 0:
+            stderr = proc.stderr.read() if proc.stderr else ""
+            raise RuntimeError(f"samtools view failed (exit {rc}):\n{stderr}")
 
     # Determine C→T vs G→A dominance per read
     mismatch_trend_per_read = {}
@@ -694,39 +1293,145 @@ def extract_base_identities(bam_file, chromosome, positions, max_reference_lengt
         else:
             mismatch_trend_per_read[read_name] = "none"
 
-    return dict(fwd_base_identities), dict(rev_base_identities), dict(mismatch_counts_per_read), mismatch_trend_per_read
+    return (
+        dict(fwd_base_identities),
+        dict(rev_base_identities),
+        dict(mismatch_counts_per_read),
+        mismatch_trend_per_read,
+    )
+
+
+def extract_read_features_from_bam(
+    bam_file_path: str | Path, samtools_backend: str | None = "auto"
+) -> Dict[str, List[float]]:
+    """Extract read metrics from a BAM file.
+
+    Args:
+        bam_file_path: Path to the BAM file.
+        samtools_backend: Backend selection for samtools-compatible operations (auto|python|cli).
 
-def extract_read_features_from_bam(bam_file_path):
-    """
-    Make a dict of reads from a bam that points to a list of read metrics: read length, read median Q-score, reference length, mapped length, mapping quality
-    Params:
-        bam_file_path (str):
     Returns:
-        read_metrics (dict)
+        Mapping of read name to [read_length, read_median_qscore, reference_length,
+        mapped_length, mapping_quality].
     """
-    # Open the BAM file
-    print(f'Extracting read features from BAM: {bam_file_path}')
-    with pysam.AlignmentFile(bam_file_path, "rb") as bam_file:
-        read_metrics = {}
-        reference_lengths = bam_file.lengths  # List of lengths for each reference (chromosome)
-        for read in bam_file:
-            # Skip unmapped reads
-            if read.is_unmapped:
+    logger.debug(
+        "Extracting read metrics from BAM using extract_read_features_from_bam: %s",
+        bam_file_path,
+    )
+    backend_choice = _resolve_samtools_backend(samtools_backend)
+    read_metrics: Dict[str, List[float]] = {}
+
+    if backend_choice == "python":
+        pysam_mod = _require_pysam()
+        with pysam_mod.AlignmentFile(str(bam_file_path), "rb") as bam_file:
+            reference_lengths = dict(zip(bam_file.references, bam_file.lengths))
+            for read in bam_file:
+                if read.is_unmapped:
+                    continue
+                read_quality = read.query_qualities
+                if read_quality is None:
+                    median_read_quality = float("nan")
+                else:
+                    median_read_quality = float(np.median(read_quality))
+                reference_length = reference_lengths.get(read.reference_name, float("nan"))
+                mapped_length = sum(end - start for start, end in read.get_blocks())
+                mapping_quality = float(read.mapping_quality)
+                read_metrics[read.query_name] = [
+                    float(read.query_length),
+                    median_read_quality,
+                    float(reference_length),
+                    float(mapped_length),
+                    mapping_quality,
+                ]
+        return read_metrics
+
+    bam_path = Path(bam_file_path)
+
+    def _parse_reference_lengths(header_text: str) -> Dict[str, int]:
+        ref_lengths: Dict[str, int] = {}
+        for line in header_text.splitlines():
+            if not line.startswith("@SQ"):
                 continue
-            # Extract the read metrics
-            read_quality = read.query_qualities
-            median_read_quality = np.median(read_quality)
-            # Extract the reference (chromosome) name and its length
-            reference_name = read.reference_name
-            reference_index = bam_file.references.index(reference_name)
-            reference_length = reference_lengths[reference_index]
-            mapped_length = sum(end - start for start, end in read.get_blocks())
-            mapping_quality = read.mapping_quality  # Phred-scaled MAPQ
-            read_metrics[read.query_name] = [read.query_length, median_read_quality, reference_length, mapped_length, mapping_quality]
+            fields = line.split("\t")
+            name = None
+            length = None
+            for field in fields[1:]:
+                if field.startswith("SN:"):
+                    name = field.split(":", 1)[1]
+                elif field.startswith("LN:"):
+                    length = int(field.split(":", 1)[1])
+            if name is not None and length is not None:
+                ref_lengths[name] = length
+        return ref_lengths
+
+    def _mapped_length_from_cigar(cigar: str) -> int:
+        mapped = 0
+        for length_str, op in re.findall(r"(\d+)([MIDNSHP=XB])", cigar):
+            length = int(length_str)
+            if op in {"M", "=", "X"}:
+                mapped += length
+        return mapped
+
+    header_cp = subprocess.run(
+        ["samtools", "view", "-H", str(bam_path)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        check=False,
+    )
+    if header_cp.returncode != 0:
+        raise RuntimeError(
+            f"samtools view -H failed (exit {header_cp.returncode}):\n{header_cp.stderr}"
+        )
+    reference_lengths = _parse_reference_lengths(header_cp.stdout)
+
+    proc = subprocess.Popen(
+        ["samtools", "view", "-F", "4", str(bam_path)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    assert proc.stdout is not None
+    for line in proc.stdout:
+        if not line.strip() or line.startswith("@"):
+            continue
+        fields = line.rstrip("\n").split("\t")
+        if len(fields) < 11:
+            continue
+        read_name = fields[0]
+        reference_name = fields[2]
+        mapping_quality = float(fields[4])
+        cigar = fields[5]
+        sequence = fields[9]
+        quality = fields[10]
+        if sequence == "*":
+            read_length = float("nan")
+        else:
+            read_length = float(len(sequence))
+        if quality == "*" or not quality:
+            median_read_quality = float("nan")
+        else:
+            phreds = [ord(char) - 33 for char in quality]
+            median_read_quality = float(np.median(phreds))
+        reference_length = float(reference_lengths.get(reference_name, float("nan")))
+        mapped_length = float(_mapped_length_from_cigar(cigar)) if cigar != "*" else 0.0
+        read_metrics[read_name] = [
+            read_length,
+            median_read_quality,
+            reference_length,
+            mapped_length,
+            mapping_quality,
+        ]
+
+    rc = proc.wait()
+    if rc != 0:
+        stderr = proc.stderr.read() if proc.stderr else ""
+        raise RuntimeError(f"samtools view failed (exit {rc}):\n{stderr}")
 
     return read_metrics
 
-def extract_readnames_from_bam(aligned_BAM):
+
+def extract_readnames_from_bam(aligned_BAM, samtools_backend: str | None = "auto"):
     """
     Takes a BAM and writes out a txt file containing read names from the BAM
 
@@ -737,17 +1442,39 @@ def extract_readnames_from_bam(aligned_BAM):
         None
 
     """
-    import subprocess
     # Make a text file of reads for the BAM
-    txt_output = aligned_BAM.split('.bam')[0] + '_read_names.txt'
-    samtools_view = subprocess.Popen(["samtools", "view", aligned_BAM], stdout=subprocess.PIPE)
-    with open(txt_output, "w") as output_file:
-        cut_process = subprocess.Popen(["cut", "-f1"], stdin=samtools_view.stdout, stdout=output_file)   
-    samtools_view.stdout.close()
-    cut_process.wait()
-    samtools_view.wait()
-
-def separate_bam_by_bc(input_bam, output_prefix, bam_suffix, split_dir):
+    backend_choice = _resolve_samtools_backend(samtools_backend)
+    txt_output = aligned_BAM.split(".bam")[0] + "_read_names.txt"
+
+    if backend_choice == "python":
+        pysam_mod = _require_pysam()
+        with (
+            pysam_mod.AlignmentFile(aligned_BAM, "rb") as bam,
+            open(txt_output, "w", encoding="utf-8") as output_file,
+        ):
+            for read in bam:
+                output_file.write(f"{read.query_name}\n")
+        return
+
+    samtools_view = subprocess.Popen(
+        ["samtools", "view", aligned_BAM], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+    )
+    assert samtools_view.stdout is not None
+    with open(txt_output, "w", encoding="utf-8") as output_file:
+        for line in samtools_view.stdout:
+            if not line.strip():
+                continue
+            qname = line.split("\t", 1)[0]
+            output_file.write(f"{qname}\n")
+    rc = samtools_view.wait()
+    if rc != 0:
+        stderr = samtools_view.stderr.read() if samtools_view.stderr else ""
+        raise RuntimeError(f"samtools view failed (exit {rc}):\n{stderr}")
+
+
+def separate_bam_by_bc(
+    input_bam, output_prefix, bam_suffix, split_dir, samtools_backend: str | None = "auto"
+):
     """
     Separates an input BAM file on the BC SAM tag values.
 
@@ -756,56 +1483,119 @@ def separate_bam_by_bc(input_bam, output_prefix, bam_suffix, split_dir):
         output_prefix (str): A prefix to append to the output BAM.
         bam_suffix (str): A suffix to add to the bam file.
         split_dir (str): String indicating path to directory to split BAMs into
-    
+
     Returns:
         None
             Writes out split BAM files.
     """
+    logger.debug("Demultiplexing BAM based on the BC tag")
     bam_base = input_bam.name
     bam_base_minus_suffix = input_bam.stem
 
-    # Open the input BAM file for reading
-    with pysam.AlignmentFile(str(input_bam), "rb") as bam:
-        # Create a dictionary to store output BAM files
-        output_files = {}
-        # Iterate over each read in the BAM file
-        for read in bam:
-            try:
-                # Get the barcode tag value
-                bc_tag = read.get_tag("BC", with_value_type=True)[0]
-                #bc_tag = read.get_tag("BC", with_value_type=True)[0].split('barcode')[1]
-                # Open the output BAM file corresponding to the barcode
-                if bc_tag not in output_files:
-                    output_path = split_dir / f"{output_prefix}_{bam_base_minus_suffix}_{bc_tag}{bam_suffix}"
-                    output_files[bc_tag] = pysam.AlignmentFile(str(output_path), "wb", header=bam.header)
-                # Write the read to the corresponding output BAM file
-                output_files[bc_tag].write(read)
-            except KeyError:
-                 print(f"BC tag not present for read: {read.query_name}")
-    # Close all output BAM files
-    for output_file in output_files.values():
-        output_file.close()
-
-def split_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix):
+    backend_choice = _resolve_samtools_backend(samtools_backend)
+
+    if backend_choice == "python":
+        pysam_mod = _require_pysam()
+        # Open the input BAM file for reading
+        with pysam_mod.AlignmentFile(str(input_bam), "rb") as bam:
+            # Create a dictionary to store output BAM files
+            output_files = {}
+            # Iterate over each read in the BAM file
+            for read in bam:
+                try:
+                    # Get the barcode tag value
+                    bc_tag = read.get_tag("BC", with_value_type=True)[0]
+                    # bc_tag = read.get_tag("BC", with_value_type=True)[0].split('barcode')[1]
+                    # Open the output BAM file corresponding to the barcode
+                    if bc_tag not in output_files:
+                        output_path = (
+                            split_dir
+                            / f"{output_prefix}_{bam_base_minus_suffix}_{bc_tag}{bam_suffix}"
+                        )
+                        output_files[bc_tag] = pysam_mod.AlignmentFile(
+                            str(output_path), "wb", header=bam.header
+                        )
+                    # Write the read to the corresponding output BAM file
+                    output_files[bc_tag].write(read)
+                except KeyError:
+                    logger.warning(f"BC tag not present for read: {read.query_name}")
+        # Close all output BAM files
+        for output_file in output_files.values():
+            output_file.close()
+        return
+
+    def _collect_bc_tags() -> set[str]:
+        bc_tags: set[str] = set()
+        proc = subprocess.Popen(
+            ["samtools", "view", str(input_bam)],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        assert proc.stdout is not None
+        for line in proc.stdout:
+            if not line.strip():
+                continue
+            fields = line.rstrip("\n").split("\t")
+            for tag in fields[11:]:
+                if tag.startswith("BC:"):
+                    bc_tags.add(tag.split(":", 2)[2])
+                    break
+        rc = proc.wait()
+        if rc != 0:
+            stderr = proc.stderr.read() if proc.stderr else ""
+            raise RuntimeError(f"samtools view failed (exit {rc}):\n{stderr}")
+        return bc_tags
+
+    bc_tags = _collect_bc_tags()
+    if not bc_tags:
+        logger.warning("No BC tags found in %s", input_bam)
+        return
+
+    for bc_tag in bc_tags:
+        output_path = split_dir / f"{output_prefix}_{bam_base_minus_suffix}_{bc_tag}{bam_suffix}"
+        cmd = ["samtools", "view", "-b", "-d", f"BC:{bc_tag}", "-o", str(output_path)]
+        cmd.append(str(input_bam))
+        cp = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+        if cp.returncode != 0:
+            raise RuntimeError(
+                f"samtools view failed for BC={bc_tag} (exit {cp.returncode}):\n{cp.stderr}"
+            )
+
+
+def split_and_index_BAM(
+    aligned_sorted_BAM, split_dir, bam_suffix, samtools_backend: str | None = "auto"
+):
     """
     A wrapper function for splitting BAMS and indexing them.
     Parameters:
         aligned_sorted_BAM (str): A string representing the file path of the aligned_sorted BAM file.
         split_dir (str): A string representing the file path to the directory to split the BAMs into.
         bam_suffix (str): A suffix to add to the bam file.
-    
+
     Returns:
         None
             Splits an input BAM file on barcode value and makes a BAM index file.
     """
+    logger.debug("Demultiplexing and indexing BAMS based on BC tag using split_and_index_BAM")
     aligned_sorted_output = aligned_sorted_BAM + bam_suffix
     file_prefix = date_string()
-    separate_bam_by_bc(aligned_sorted_output, file_prefix, bam_suffix, split_dir)
+    separate_bam_by_bc(
+        aligned_sorted_output,
+        file_prefix,
+        bam_suffix,
+        split_dir,
+        samtools_backend=samtools_backend,
+    )
     # Make a BAM index file for the BAMs in that directory
-    bam_pattern = '*' + bam_suffix
+    bam_pattern = "*" + bam_suffix
     bam_files = glob.glob(split_dir / bam_pattern)
-    bam_files = [str(bam) for bam in bam_files if '.bai' not in str(bam)]
+    bam_files = [str(bam) for bam in bam_files if ".bai" not in str(bam)]
+    backend_choice = _resolve_samtools_backend(samtools_backend)
     for input_file in bam_files:
-        pysam.index(input_file)
+        if backend_choice == "python":
+            _index_bam_with_pysam(input_file)
+        else:
+            _index_bam_with_samtools(input_file)
 
-    return bam_files
+    return bam_files