smftools 0.2.5__py3-none-any.whl → 0.3.0__py3-none-any.whl

smftools/informatics/fasta_functions.py

@@ -1,23 +1,93 @@
 from __future__ import annotations
 
 import gzip
+import shutil
+import subprocess
 from concurrent.futures import ProcessPoolExecutor
+from importlib.util import find_spec
 from pathlib import Path
-from typing import Dict, Iterable, Tuple
+from typing import TYPE_CHECKING, Dict, Iterable, Tuple
 
 import numpy as np
-import pysam
 from Bio import SeqIO
 from Bio.Seq import Seq
 from Bio.SeqRecord import SeqRecord
-from pyfaidx import Fasta
 
 from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
 
 from ..readwrite import time_string
 
 logger = get_logger(__name__)
 
+if TYPE_CHECKING:
+    import pysam as pysam_module
+
+
+def _require_pysam() -> "pysam_module":
+    if pysam_types is not None:
+        return pysam_types
+    return require("pysam", extra="pysam", purpose="FASTA access")
+
+
+pysam_types = None
+if find_spec("pysam") is not None:
+    pysam_types = require("pysam", extra="pysam", purpose="FASTA access")
+
+
+def _resolve_fasta_backend() -> str:
+    """Resolve the backend to use for FASTA access."""
+    if pysam_types is not None:
+        return "python"
+    if shutil is not None and shutil.which("samtools"):
+        return "cli"
+    raise RuntimeError("FASTA access requires pysam or samtools in PATH.")
+
+
+def _ensure_fasta_index(fasta: Path) -> None:
+    fai = fasta.with_suffix(fasta.suffix + ".fai")
+    if fai.exists():
+        return
+    if subprocess is None or shutil is None or not shutil.which("samtools"):
+        pysam_mod = _require_pysam()
+        pysam_mod.faidx(str(fasta))
+        return
+    cp = subprocess.run(
+        ["samtools", "faidx", str(fasta)],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    if cp.returncode != 0:
+        raise RuntimeError(f"samtools faidx failed (exit {cp.returncode}):\n{cp.stderr}")
+
+
+def _bed_to_faidx_region(chrom: str, start: int, end: int) -> str:
+    """Convert 0-based half-open BED coords to samtools faidx region."""
+    start1 = start + 1
+    end1 = end
+    if start1 > end1:
+        start1, end1 = end1, start1
+    return f"{chrom}:{start1}-{end1}"
+
+
+def _fetch_sequence_with_samtools(fasta: Path, chrom: str, start: int, end: int) -> str:
+    if subprocess is None or shutil is None:
+        raise RuntimeError("samtools backend is unavailable.")
+    if not shutil.which("samtools"):
+        raise RuntimeError("samtools is required but not available in PATH.")
+    region = _bed_to_faidx_region(chrom, start, end)
+    cp = subprocess.run(
+        ["samtools", "faidx", str(fasta), region],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    if cp.returncode != 0:
+        raise RuntimeError(f"samtools faidx failed (exit {cp.returncode}):\n{cp.stderr}")
+    lines = [line.strip() for line in cp.stdout.splitlines() if line and not line.startswith(">")]
+    return "".join(lines)
+
 
 def _convert_FASTA_record(
     record: SeqRecord,
@@ -160,7 +230,7 @@ def index_fasta(fasta: str | Path, write_chrom_sizes: bool = True) -> Path:
         Path: Path to the index file or chromosome sizes file.
     """
     fasta = Path(fasta)
-    pysam.faidx(str(fasta))  # creates <fasta>.fai
+    _require_pysam().faidx(str(fasta))  # creates <fasta>.fai
 
     fai = fasta.with_suffix(fasta.suffix + ".fai")
     if write_chrom_sizes:
@@ -307,8 +377,13 @@ def subsample_fasta_from_bed(
     # Ensure output directory exists
     output_directory.mkdir(parents=True, exist_ok=True)
 
-    # Load the FASTA file using pyfaidx
-    fasta = Fasta(str(input_FASTA))  # pyfaidx requires string paths
+    backend = _resolve_fasta_backend()
+    _ensure_fasta_index(input_FASTA)
+
+    fasta_handle = None
+    if backend == "python":
+        pysam_mod = _require_pysam()
+        fasta_handle = pysam_mod.FastaFile(str(input_FASTA))
 
     # Open BED + output FASTA
     with input_bed.open("r") as bed, output_FASTA.open("w") as out_fasta:
@@ -319,15 +394,24 @@ def subsample_fasta_from_bed(
             end = int(fields[2])  # BED is 0-based and end is exclusive
             desc = " ".join(fields[3:]) if len(fields) > 3 else ""
 
-            if chrom not in fasta:
+            if backend == "python":
+                assert fasta_handle is not None
+                if chrom not in fasta_handle.references:
+                    logger.warning(f"{chrom} not found in FASTA")
+                    continue
+                sequence = fasta_handle.fetch(chrom, start, end)
+            else:
+                sequence = _fetch_sequence_with_samtools(input_FASTA, chrom, start, end)
+
+            if not sequence:
                 logger.warning(f"{chrom} not found in FASTA")
                 continue
 
-            # pyfaidx is 1-based indexing internally, but [start:end] works with BED coords
-            sequence = fasta[chrom][start:end].seq
-
             header = f">{chrom}:{start}-{end}"
             if desc:
                 header += f"    {desc}"
 
             out_fasta.write(f"{header}\n{sequence}\n")
+
+    if fasta_handle is not None:
+        fasta_handle.close()

smftools/informatics/h5ad_functions.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import glob
 import os
 from concurrent.futures import ProcessPoolExecutor, as_completed
@@ -7,9 +9,9 @@ from typing import Dict, List, Optional, Union
 import numpy as np
 import pandas as pd
 import scipy.sparse as sp
-from pod5 import Reader
 
 from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
 
 logger = get_logger(__name__)
 
@@ -90,6 +92,7 @@ def add_read_length_and_mapping_qc(
     extract_read_features_from_bam_callable=None,
     bypass: bool = False,
     force_redo: bool = True,
+    samtools_backend: str | None = "auto",
 ):
     """
     Populate adata.obs with read/mapping QC columns.
@@ -133,7 +136,7 @@ def add_read_length_and_mapping_qc(
                     "No `read_metrics` provided and `extract_read_features_from_bam` not found."
                 )
             for bam in bam_files:
-                bam_read_metrics = extractor(bam)
+                bam_read_metrics = extractor(bam, samtools_backend)
                 if not isinstance(bam_read_metrics, dict):
                     raise ValueError(f"extract_read_features_from_bam returned non-dict for {bam}")
                 read_metrics.update(bam_read_metrics)
@@ -228,6 +231,9 @@ def _collect_read_origins_from_pod5(pod5_path: str, target_ids: set[str]) -> dic
     Worker function: scan one POD5 file and return a mapping
     {read_id: pod5_basename} only for read_ids in `target_ids`.
     """
+    p5 = require("pod5", extra="ont", purpose="POD5 metadata")
+    Reader = p5.Reader
+
     basename = os.path.basename(pod5_path)
     mapping: dict[str, str] = {}
 

smftools/informatics/modkit_extract_to_adata.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import concurrent.futures
 import gc
 import re
@@ -16,9 +18,11 @@ from .bam_functions import count_aligned_reads
 logger = get_logger(__name__)
 
 
-def filter_bam_records(bam, mapping_threshold):
+def filter_bam_records(bam, mapping_threshold, samtools_backend: str | None = "auto"):
     """Processes a single BAM file, counts reads, and determines records to analyze."""
-    aligned_reads_count, unaligned_reads_count, record_counts_dict = count_aligned_reads(bam)
+    aligned_reads_count, unaligned_reads_count, record_counts_dict = count_aligned_reads(
+        bam, samtools_backend
+    )
 
     total_reads = aligned_reads_count + unaligned_reads_count
     percent_aligned = (aligned_reads_count * 100 / total_reads) if total_reads > 0 else 0
@@ -35,13 +39,16 @@ def filter_bam_records(bam, mapping_threshold):
     return set(records)
 
 
-def parallel_filter_bams(bam_path_list, mapping_threshold):
+def parallel_filter_bams(bam_path_list, mapping_threshold, samtools_backend: str | None = "auto"):
     """Parallel processing for multiple BAM files."""
     records_to_analyze = set()
 
     with concurrent.futures.ProcessPoolExecutor() as executor:
         results = executor.map(
-            filter_bam_records, bam_path_list, [mapping_threshold] * len(bam_path_list)
+            filter_bam_records,
+            bam_path_list,
+            [mapping_threshold] * len(bam_path_list),
+            [samtools_backend] * len(bam_path_list),
         )
 
     # Aggregate results
@@ -484,6 +491,7 @@ def modkit_extract_to_adata(
     delete_batch_hdfs=False,
     threads=None,
     double_barcoded_path=None,
+    samtools_backend: str | None = "auto",
 ):
     """
     Takes modkit extract outputs and organizes it into an adata object
@@ -591,7 +599,7 @@ def modkit_extract_to_adata(
 
     ######### Get Record names that have over a passed threshold of mapped reads #############
     # get all records that are above a certain mapping threshold in at least one sample bam
-    records_to_analyze = parallel_filter_bams(bam_path_list, mapping_threshold)
+    records_to_analyze = parallel_filter_bams(bam_path_list, mapping_threshold, samtools_backend)
 
     ##########################################################################################
 
@@ -635,7 +643,9 @@ def modkit_extract_to_adata(
                     rev_base_identities,
                     mismatch_counts_per_read,
                     mismatch_trend_per_read,
-                ) = extract_base_identities(bam, record, positions, max_reference_length, ref_seq)
+                ) = extract_base_identities(
+                    bam, record, positions, max_reference_length, ref_seq, samtools_backend
+                )
                 # Store read names of fwd and rev mapped reads
                 fwd_mapped_reads.update(fwd_base_identities.keys())
                 rev_mapped_reads.update(rev_base_identities.keys())

smftools/informatics/modkit_functions.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import subprocess
 
 from smftools.logging_utils import get_logger

smftools/informatics/ohe.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import concurrent.futures
 import os
 

smftools/informatics/pod5_functions.py

@@ -5,9 +5,8 @@ import subprocess
 from pathlib import Path
 from typing import Iterable
 
-import pod5 as p5
-
 from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
 
 from ..config import LoadExperimentConfig
 from ..informatics.basecalling import canoncall, modcall
@@ -15,6 +14,8 @@ from ..readwrite import make_dirs
 
 logger = get_logger(__name__)
 
+p5 = require("pod5", extra="ont", purpose="POD5 IO")
+
 
 def basecall_pod5s(config_path: str | Path) -> None:
     """Basecall POD5 inputs using a configuration file.

smftools/machine_learning/__init__.py

@@ -1,7 +1,23 @@
-from . import data, evaluation, inference, models, training, utils
+from __future__ import annotations
 
-__all__ = [
-    "calculate_relative_risk_on_activity",
-    "evaluate_models_by_subgroup",
-    "prepare_melted_model_data",
-]
+from importlib import import_module
+
+_LAZY_MODULES = {
+    "data": "smftools.machine_learning.data",
+    "evaluation": "smftools.machine_learning.evaluation",
+    "inference": "smftools.machine_learning.inference",
+    "models": "smftools.machine_learning.models",
+    "training": "smftools.machine_learning.training",
+    "utils": "smftools.machine_learning.utils",
+}
+
+
+def __getattr__(name: str):
+    if name in _LAZY_MODULES:
+        module = import_module(_LAZY_MODULES[name])
+        globals()[name] = module
+        return module
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = list(_LAZY_MODULES.keys())

smftools/machine_learning/data/__init__.py

@@ -1,2 +1,4 @@
+from __future__ import annotations
+
 from .anndata_data_module import AnnDataModule, build_anndata_loader
 from .preprocessing import random_fill_nans

smftools/machine_learning/data/anndata_data_module.py

@@ -1,12 +1,26 @@
+from __future__ import annotations
+
 import numpy as np
 import pandas as pd
-import pytorch_lightning as pl
-import torch
-from sklearn.utils.class_weight import compute_class_weight
-from torch.utils.data import DataLoader, Dataset, Subset
+
+from smftools.optional_imports import require
 
 from .preprocessing import random_fill_nans
 
+pl = require("pytorch_lightning", extra="ml-extended", purpose="Lightning data modules")
+torch = require("torch", extra="ml-base", purpose="ML data loading")
+sklearn_class_weight = require(
+    "sklearn.utils.class_weight",
+    extra="ml-base",
+    purpose="class weighting",
+)
+torch_utils_data = require("torch.utils.data", extra="ml-base", purpose="ML data loading")
+
+compute_class_weight = sklearn_class_weight.compute_class_weight
+DataLoader = torch_utils_data.DataLoader
+Dataset = torch_utils_data.Dataset
+Subset = torch_utils_data.Subset
+
 
 class AnnDataDataset(Dataset):
     """

smftools/machine_learning/data/preprocessing.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import numpy as np
 
 

smftools/machine_learning/evaluation/__init__.py

@@ -1,2 +1,4 @@
+from __future__ import annotations
+
 from .eval_utils import flatten_sliding_window_results
 from .evaluators import ModelEvaluator, PostInferenceModelEvaluator

smftools/machine_learning/evaluation/eval_utils.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pandas as pd
 
 

smftools/machine_learning/evaluation/evaluators.py

@@ -1,14 +1,19 @@
-import matplotlib.pyplot as plt
+from __future__ import annotations
+
 import numpy as np
 import pandas as pd
-from sklearn.metrics import (
-    auc,
-    confusion_matrix,
-    f1_score,
-    precision_recall_curve,
-    roc_auc_score,
-    roc_curve,
-)
+
+from smftools.optional_imports import require
+
+plt = require("matplotlib.pyplot", extra="plotting", purpose="evaluation plots")
+sklearn_metrics = require("sklearn.metrics", extra="ml-base", purpose="model evaluation")
+
+auc = sklearn_metrics.auc
+confusion_matrix = sklearn_metrics.confusion_matrix
+f1_score = sklearn_metrics.f1_score
+precision_recall_curve = sklearn_metrics.precision_recall_curve
+roc_auc_score = sklearn_metrics.roc_auc_score
+roc_curve = sklearn_metrics.roc_curve
 
 
 class ModelEvaluator:

smftools/machine_learning/inference/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from .lightning_inference import run_lightning_inference
 from .sklearn_inference import run_sklearn_inference
 from .sliding_window_inference import sliding_window_inference

smftools/machine_learning/inference/inference_utils.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pandas as pd
 
 

smftools/machine_learning/inference/lightning_inference.py

@@ -1,9 +1,14 @@
+from __future__ import annotations
+
 import numpy as np
 import pandas as pd
-import torch
+
+from smftools.optional_imports import require
 
 from .inference_utils import annotate_split_column
 
+torch = require("torch", extra="ml-base", purpose="Lightning inference")
+
 
 def run_lightning_inference(adata, model, datamodule, trainer, prefix="model", devices=1):
     """

smftools/machine_learning/inference/sklearn_inference.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import numpy as np
 import pandas as pd
 

smftools/machine_learning/inference/sliding_window_inference.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from ..data import AnnDataModule
 from ..evaluation import PostInferenceModelEvaluator
 from .lightning_inference import run_lightning_inference

smftools/machine_learning/models/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from .base import BaseTorchModel
 from .cnn import CNNClassifier
 from .lightning_base import TorchClassifierWrapper

smftools/machine_learning/models/base.py

@@ -1,9 +1,14 @@
+from __future__ import annotations
+
 import numpy as np
-import torch
-import torch.nn as nn
+
+from smftools.optional_imports import require
 
 from ..utils.device import detect_device
 
+torch = require("torch", extra="ml-base", purpose="ML base models")
+nn = torch.nn
+
 
 class BaseTorchModel(nn.Module):
     """

smftools/machine_learning/models/cnn.py

@@ -1,9 +1,14 @@
+from __future__ import annotations
+
 import numpy as np
-import torch
-import torch.nn as nn
+
+from smftools.optional_imports import require
 
 from .base import BaseTorchModel
 
+torch = require("torch", extra="ml-base", purpose="CNN models")
+nn = torch.nn
+
 
 class CNNClassifier(BaseTorchModel):
     def __init__(

smftools/machine_learning/models/lightning_base.py

@@ -1,15 +1,20 @@
-import matplotlib.pyplot as plt
+from __future__ import annotations
+
 import numpy as np
-import pytorch_lightning as pl
-import torch
-from sklearn.metrics import (
-    auc,
-    confusion_matrix,
-    f1_score,
-    precision_recall_curve,
-    roc_auc_score,
-    roc_curve,
-)
+
+from smftools.optional_imports import require
+
+plt = require("matplotlib.pyplot", extra="plotting", purpose="model evaluation plots")
+pl = require("pytorch_lightning", extra="ml-extended", purpose="Lightning models")
+torch = require("torch", extra="ml-base", purpose="Lightning models")
+sklearn_metrics = require("sklearn.metrics", extra="ml-base", purpose="model evaluation")
+
+auc = sklearn_metrics.auc
+confusion_matrix = sklearn_metrics.confusion_matrix
+f1_score = sklearn_metrics.f1_score
+precision_recall_curve = sklearn_metrics.precision_recall_curve
+roc_auc_score = sklearn_metrics.roc_auc_score
+roc_curve = sklearn_metrics.roc_curve
 
 
 class TorchClassifierWrapper(pl.LightningModule):

smftools/machine_learning/models/mlp.py

@@ -1,7 +1,11 @@
-import torch.nn as nn
+from __future__ import annotations
+
+from smftools.optional_imports import require
 
 from .base import BaseTorchModel
 
+nn = require("torch.nn", extra="ml-base", purpose="MLP models")
+
 
 class MLPClassifier(BaseTorchModel):
     def __init__(

smftools/machine_learning/models/positional.py

@@ -1,6 +1,11 @@
+from __future__ import annotations
+
 import numpy as np
-import torch
-import torch.nn as nn
+
+from smftools.optional_imports import require
+
+torch = require("torch", extra="ml-base", purpose="positional encoding")
+nn = torch.nn
 
 
 class PositionalEncoding(nn.Module):

smftools/machine_learning/models/rnn.py

@@ -1,7 +1,11 @@
-import torch.nn as nn
+from __future__ import annotations
+
+from smftools.optional_imports import require
 
 from .base import BaseTorchModel
 
+nn = require("torch.nn", extra="ml-base", purpose="RNN models")
+
 
 class RNNClassifier(BaseTorchModel):
     def __init__(self, input_size, hidden_dim, num_classes, **kwargs):

smftools/machine_learning/models/sklearn_models.py

@@ -1,13 +1,18 @@
-import matplotlib.pyplot as plt
+from __future__ import annotations
+
 import numpy as np
-from sklearn.metrics import (
-    auc,
-    confusion_matrix,
-    f1_score,
-    precision_recall_curve,
-    roc_auc_score,
-    roc_curve,
-)
+
+from smftools.optional_imports import require
+
+plt = require("matplotlib.pyplot", extra="plotting", purpose="model evaluation plots")
+sklearn_metrics = require("sklearn.metrics", extra="ml-base", purpose="model evaluation")
+
+auc = sklearn_metrics.auc
+confusion_matrix = sklearn_metrics.confusion_matrix
+f1_score = sklearn_metrics.f1_score
+precision_recall_curve = sklearn_metrics.precision_recall_curve
+roc_auc_score = sklearn_metrics.roc_auc_score
+roc_curve = sklearn_metrics.roc_curve
 
 
 class SklearnModelWrapper:

smftools/machine_learning/models/transformer.py

@@ -1,11 +1,16 @@
+from __future__ import annotations
+
 import numpy as np
-import torch
-import torch.nn as nn
+
+from smftools.optional_imports import require
 
 from ..utils.grl import grad_reverse
 from .base import BaseTorchModel
 from .positional import PositionalEncoding
 
+torch = require("torch", extra="ml-base", purpose="Transformer models")
+nn = torch.nn
+
 
 class TransformerEncoderLayerWithAttn(nn.TransformerEncoderLayer):
     def __init__(self, *args, **kwargs):

smftools/machine_learning/models/wrappers.py

@@ -1,5 +1,9 @@
-import torch
-import torch.nn as nn
+from __future__ import annotations
+
+from smftools.optional_imports import require
+
+torch = require("torch", extra="ml-base", purpose="model wrappers")
+nn = torch.nn
 
 
 class ScaledModel(nn.Module):

smftools/machine_learning/training/__init__.py

@@ -1,2 +1,4 @@
+from __future__ import annotations
+
 from .train_lightning_model import run_sliding_window_lightning_training, train_lightning_model
 from .train_sklearn_model import run_sliding_window_sklearn_training, train_sklearn_model

smftools/machine_learning/training/train_lightning_model.py

@@ -1,10 +1,20 @@
-import torch
-from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
+from __future__ import annotations
+
+from smftools.optional_imports import require
 
 from ..data import AnnDataModule
 from ..models import TorchClassifierWrapper
 
+torch = require("torch", extra="ml-base", purpose="Lightning training")
+pytorch_lightning = require("pytorch_lightning", extra="ml-extended", purpose="Lightning training")
+pl_callbacks = require(
+    "pytorch_lightning.callbacks", extra="ml-extended", purpose="Lightning training"
+)
+
+Trainer = pytorch_lightning.Trainer
+EarlyStopping = pl_callbacks.EarlyStopping
+ModelCheckpoint = pl_callbacks.ModelCheckpoint
+
 
 def train_lightning_model(
     model,