smftools 0.1.6__py3-none-any.whl → 0.2.1__py3-none-any.whl

smftools/__init__.py

@@ -0,0 +1,34 @@
+"""smftools"""
+
+import logging
+import warnings
+
+from . import informatics as inform
+from . import machine_learning as ml
+from . import plotting as pl
+from . import preprocessing as pp
+from . import tools as tl
+
+from . import config, datasets, hmm, readwrite
+from .readwrite import adata_to_df, safe_write_h5ad, safe_read_h5ad, merge_barcoded_anndatas_core
+
+from .load_adata import load_adata
+
+from importlib.metadata import version
+
+package_name = "smftools"
+__version__ = version(package_name)
+
+__all__ = [
+    "load_adata"
+    "adata_to_df",
+    "inform",
+    "ml",
+    "pp",
+    "tl",
+    "pl",
+    "readwrite",
+    "datasets",
+    "safe_write_h5ad",
+    "safe_read_h5ad"
+]

smftools/_settings.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+from typing import Union
+
+class SMFConfig:
+    """\
+    Config for smftools.
+    """
+
+    def __init__(
+        self,
+        *,
+        datasetdir: Union[Path, str] = "./datasets/"
+    ):
+        self._datasetdir = Path(datasetdir) if isinstance(datasetdir, str) else datasetdir
+
+    @property
+    def datasetdir(self) -> Path:
+        return self._datasetdir
+
+settings = SMFConfig()

smftools/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

smftools/cli.py

@@ -0,0 +1,184 @@
+import click
+import pandas as pd
+from pathlib import Path
+from typing import Dict, Optional
+
+from . import load_adata
+from .readwrite import merge_barcoded_anndatas_core, safe_read_h5ad, safe_write_h5ad
+
+@click.group()
+def cli():
+    """Command-line interface for smftools."""
+    pass
+
+####### Main processing workflow ###########
+@cli.command()
+@click.argument("config_path", type=click.Path(exists=True))
+def load(config_path):
+    """Load and process data from CONFIG_PATH."""
+    load_adata(config_path)
+##########################################
+
+
+####### Merging existing anndatas from an experiment that used two different demultiplexing rules #######
+REQUIRED_KEYS = ("adata_single_path", "adata_double_path")
+OPTIONAL_KEYS = (
+    "adata_single_backups_path",
+    "adata_double_backups_path",
+    "output_path",
+    "merged_filename",
+)
+
+def _read_config_csv(csv_path: Path) -> Dict[str, str]:
+    """
+    Read a multi-row, two-column CSV of key,value pairs into a dict.
+
+    Supported features:
+      - Optional header ("key,value") or none.
+      - Comments starting with '#' and blank lines are ignored.
+      - If duplicate keys occur, the last one wins.
+      - Keys are matched literally against REQUIRED_KEYS/OPTIONAL_KEYS.
+    """
+    try:
+        # Read as two columns regardless of header; comments ignored.
+        df = pd.read_csv(
+            csv_path,
+            dtype=str,
+            comment="#",
+            header=None,          # treat everything as rows; we'll normalize below
+            usecols=[0, 1],
+            names=["key", "value"]
+        )
+    except Exception as e:
+        raise click.ClickException(f"Failed to read CSV: {e}") from e
+
+    # Drop completely empty rows
+    df = df.fillna("").astype(str)
+    df["key"] = df["key"].str.strip()
+    df["value"] = df["value"].str.strip()
+    df = df[(df["key"] != "") & (df["key"].notna())]
+
+    if df.empty:
+        raise click.ClickException("Config CSV is empty after removing comments/blank lines.")
+
+    # Remove an optional header row if present
+    if df.iloc[0]["key"].lower() in {"key", "keys"}:
+        df = df.iloc[1:]
+        df = df[(df["key"] != "") & (df["key"].notna())]
+        if df.empty:
+            raise click.ClickException("Config CSV contains only a header row.")
+
+    # Build dict; last occurrence of a key wins
+    cfg = {}
+    for k, v in zip(df["key"], df["value"]):
+        cfg[k] = v
+
+    # Validate required keys
+    missing = [k for k in REQUIRED_KEYS if not cfg.get(k)]
+    if missing:
+        raise click.ClickException(
+            "Missing required keys in CSV: "
+            + ", ".join(missing)
+            + "\nExpected keys:\n  - "
+            + "\n  - ".join(REQUIRED_KEYS)
+            + "\nOptional keys:\n  - "
+            + "\n  - ".join(OPTIONAL_KEYS)
+        )
+
+    return cfg
+
+def _resolve_output_path(cfg: Dict[str, str], single_path: Path, double_path: Path) -> Path:
+    """Decide on the output .h5ad path based on CSV; create directories if needed."""
+    merged_filename = cfg.get("merged_filename") or f"merged_{single_path.stem}__{double_path.stem}.h5ad"
+    if not merged_filename.endswith(".h5ad"):
+        merged_filename += ".h5ad"
+
+    output_path_raw = cfg.get("output_path", "").strip()
+
+    if not output_path_raw:
+        out_dir = Path.cwd() / "merged_output"
+        out_dir.mkdir(parents=True, exist_ok=True)
+        return out_dir / merged_filename
+
+    output_path = Path(output_path_raw)
+
+    if output_path.suffix.lower() == ".h5ad":
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        return output_path
+
+    # Treat as directory
+    output_path.mkdir(parents=True, exist_ok=True)
+    return output_path / merged_filename
+
+def _maybe_read_adata(label: str, primary: Path, backups: Optional[Path]):
+
+    if backups:
+        click.echo(f"Loading {label} from {primary} with backups at {backups} ...")
+        return safe_read_h5ad(primary, backups_path=backups, restore_backups=True)
+    else:
+        click.echo(f"Loading {label} from {primary} with backups disabled ...")
+        return safe_read_h5ad(primary, restore_backups=False)
+
+
+@cli.command()
+@click.argument("config_path", type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path))
+def merge_barcoded_anndatas(config_path: Path):
+    """
+    Merge two AnnData objects from the same experiment that were demultiplexed
+    under different end-barcoding requirements, using a 1-row CSV for config.
+
+    CSV must include:
+      - adata_single_path
+      - adata_double_path
+
+    Optional columns:
+      - adata_single_backups_path
+      - adata_double_backups_path
+      - output_path            (file or directory; default: ./merged_output/)
+      - merged_filename        (default: merged_<single>__<double>.h5ad)
+
+    Example CSV:
+
+        adata_single_path,adata_double_path,adata_single_backups_path,adata_double_backups_path,output_path,merged_filename
+        /path/single.h5ad,/path/double.h5ad,,,,merged_output,merged_run.h5ad
+    """
+    try:
+        cfg = _read_config_csv(config_path)
+
+        single_path = Path(cfg["adata_single_path"]).expanduser().resolve()
+        double_path = Path(cfg["adata_double_path"]).expanduser().resolve()
+
+        for p, label in [(single_path, "adata_single_path"), (double_path, "adata_double_path")]:
+            if not p.exists():
+                raise click.ClickException(f"{label} does not exist: {p}")
+
+        single_backups = Path(cfg["adata_single_backups_path"]).expanduser().resolve() if cfg.get("adata_single_backups_path") else None
+        double_backups = Path(cfg["adata_double_backups_path"]).expanduser().resolve() if cfg.get("adata_double_backups_path") else None
+
+        if single_backups and not single_backups.exists():
+            raise click.ClickException(f"adata_single_backups_path does not exist: {single_backups}")
+        if double_backups and not double_backups.exists():
+            raise click.ClickException(f"adata_double_backups_path does not exist: {double_backups}")
+
+        output_path = _resolve_output_path(cfg, single_path, double_path)
+
+        # Load
+        adata_single, read_report_single = _maybe_read_adata("single-barcoded AnnData", single_path, single_backups)
+        adata_double, read_report_double = _maybe_read_adata("double-barcoded AnnData", double_path, double_backups)
+
+        click.echo("Merging AnnDatas ...")
+        merged = merge_barcoded_anndatas_core(adata_single, adata_double)
+
+        click.echo(f"Writing merged AnnData to: {output_path}")
+        backup_dir = output_path.cwd() / "merged_backups"
+        safe_write_h5ad(merged, output_path, backup=True, backup_dir=backup_dir)
+
+        click.secho(f"Done. Merged AnnData saved to {output_path}", fg="green")
+
+    except click.ClickException:
+        raise
+    except Exception as e:
+        # Surface unexpected errors cleanly
+        raise click.ClickException(f"Unexpected error: {e}") from e
+
+################################################################################################################

smftools/config/__init__.py

@@ -0,0 +1 @@
+from .experiment_config import LoadExperimentConfig, ExperimentConfig

smftools/config/conversion.yaml

@@ -0,0 +1,33 @@
+# Conversion (Bisulfite/APOBEC)footprinting defaults
+extends: default
+conversion_types: 
+  - '5mC' # 5mC
+
+# Read QC Params
+read_mod_filtering_use_other_c_as_background: True
+
+# HMM
+cpg: True # whether to use the default HMM endogenous CpG patch params
+hmm_methbases:
+  - "GpC"
+hmm_feature_sets:
+  footprint:
+    state: "Non-Modified"
+    features:
+      small_bound_stretch: [0, 20]
+      medium_bound_stretch: [20, 50]
+      putative_nucleosome: [50, 200]
+      large_bound_stretch: [200, inf]
+  accessible:
+    state: "Modified"
+    features:
+      small_accessible_patch: [0, 20]
+      mid_accessible_patch: [20, 80]
+      large_accessible_patch: [80, inf]
+  cpg:
+    state: "Modified"
+    features:
+      cpg_patch: [0, inf]
+
+hmm_merge_layer_features:
+  - ["GpC_all_accessible_features", 80]

smftools/config/deaminase.yaml

@@ -0,0 +1,56 @@
+# Deaminase footprinting defaults
+extends: default
+conversion_types: 
+  - '5mC' # 5mC
+
+mod_target_bases:
+- "C"
+
+read_mod_filtering_gpc_thresholds:
+  - null
+  - null
+read_mod_filtering_cpg_thresholds:
+  - null
+  - null
+read_mod_filtering_any_c_thresholds:
+  - 0.01
+  - 0.99
+read_mod_filtering_a_thresholds:
+  - null
+  - null
+
+read_mod_filtering_use_other_c_as_background: False
+
+# Duplicate Detection Params
+duplicate_detection_site_types:
+  - "any_C"
+
+# Autocorrelation params
+autocorr_site_types:
+  - "any_C"
+
+# Correlation matrix params
+correlation_matrix_site_types:
+  - "any_C_site"
+
+# HMM
+cpg: False # whether to use the default HMM endogenous CpG patch params
+hmm_methbases:
+  - "C"
+hmm_feature_sets:
+  footprint:
+    state: "Non-Modified"
+    features:
+      small_bound_stretch: [0, 25]
+      medium_bound_stretch: [25, 80]
+      putative_nucleosome: [80, 200]
+      large_bound_stretch: [200, inf]
+  accessible:
+    state: "Modified"
+    features:
+      small_accessible_patch: [0, 20]
+      mid_accessible_patch: [20, 100]
+      large_accessible_patch: [100, inf]
+
+hmm_merge_layer_features:
+  - ["C_all_accessible_features", 80]

smftools/config/default.yaml

@@ -0,0 +1,253 @@
+# Generic i/o
+bam_suffix: ".bam"
+recursive_input_search: True
+split_dir: "demultiplexed_BAMs"
+strands:
+  - bottom
+  - top
+conversions:
+  - unconverted
+sample_sheet_path: null # path to sample_sheet to load metadata into anndata.
+sample_sheet_mapping_column: 'Barcode' # The column in the sample sheet and current anndata to use for mapping metadata.
+fastq_barcode_map: null # For FASTQ files, an optional map of file paths to barcodes can be provided. Default is autodetecting barcodes.
+fastq_auto_pairing: True # For FASTQ files, attempt to find read pair files automatically.
+input_already_demuxed: False # If the input files are already demultiplexed.
+delete_intermediate_hdfs: True # Whether to delete the intermediate hdfs from the conversion/deamination workflows.
+
+# Compute params
+threads: 4
+device: "auto"
+
+# Sequencing modality and general experiment params
+smf_modality: 'conversion' # conversion, deaminase, direct
+sequencer: 'ont' # ont, pacbio, illumina
+barcode_kit: 'SQK-RBK114-96' # SQK-RBK114-96, SQK-NBD114-24, etc
+mod_target_bases:
+  - "GpC"
+  - "CpG"
+enzyme_target_bases:
+  - "GpC"
+
+# Nanopore specific basecalling params
+model_dir: null # Directory where dorado basecalling models are stored.
+model: "hac" # needed for dorado basecaller
+filter_threshold: 0.8 # Dorado probability filter threshold for base calling.
+
+# Alignment params
+aligner: "minimap2" # Aligner to use: dorado, minimap2
+aligner_args:
+  minimap2:
+    ont:
+      - '-a'
+      - '-x'
+      - 'map-ont'
+      - '--MD'
+      - '-Y' 
+      - '-y'
+      - '-N'
+      - '5'
+      - '--secondary=no' 
+    pacbio:
+      - '-a'
+      - '-x'
+      - 'map-hifi'
+      - '--MD'
+      - '-Y' 
+      - '-y'
+      - '-N'
+      - '5'
+      - '--secondary=no' 
+    illumina:
+      - '-a'
+      - '-x'
+      - 'sr'
+      - '--MD'
+      - '-Y' 
+      - '-y'
+      - '-N'
+      - '5'
+      - '--secondary=no'     
+  dorado:
+    ont:
+      - "--mm2-opts"
+      - "-N"
+      - "5"
+
+# Sorted BAM and BED specific handling
+make_bigwigs: False # Whether to make coverage bigwigs
+
+# Nanopore specific demultiplexing
+barcode_both_ends: False # dorado demultiplexing
+trim: False # dorado adapter and barcode removal during demultiplexing
+
+# Anndata structure
+mapping_threshold: 0.01 # Minimum proportion of mapped reads that need to fall within a region to include in the final AnnData.
+reference_column: 'Reference_strand'
+sample_column: 'Barcode'
+
+# Preprocessing -  Read length, quality, and mapping filtering params
+read_coord_filter:
+ - null
+ - null
+read_len_filter_thresholds:
+  - 200
+  - null
+read_len_to_ref_ratio_filter_thresholds:
+  - 0.8
+  - null
+read_quality_filter_thresholds:
+  - 20
+  - null
+read_mapping_quality_filter_thresholds:
+  - null
+  - null
+
+# Preprocessing -  Read modification filtering params
+read_mod_filtering_gpc_thresholds:
+  - 0.025
+  - 0.975
+read_mod_filtering_cpg_thresholds:
+  - 0.0
+  - 1.0
+read_mod_filtering_any_c_thresholds:
+  - 0.025
+  - 0.975
+read_mod_filtering_a_thresholds:
+  - 0.025
+  - 0.975
+read_mod_filtering_use_other_c_as_background: False
+min_valid_fraction_positions_in_read_vs_ref: 0.8
+
+# Preprocessing - Duplicate detection params
+duplicate_detection_site_types: # Site types to consider for duplicate detection workflow
+  - "GpC"
+  - "CpG"
+  - "ambiguous_GpC_CpG"
+duplicate_detection_distance_threshold: 0.07 # Hamming distance based similarity threshold to use for marking duplicate reads.
+hamming_vs_metric_keys: # Metrics to plot the hamming distance against.
+  - Fraction_any_C_site_modified
+duplicate_detection_keep_best_metric: "read_quality" # Obs metric to use to keep a representative read from a read duplicate cluster
+duplicate_detection_window_size_for_hamming_neighbors: 50 # How many neighboring reads to look at for calculating hamming distance pairs
+duplicate_detection_min_overlapping_positions: 20 # The minimum amount of valid overlapping positions that will allow duplicate detection to work
+duplicate_detection_do_hierarchical: True # Whether to follow up fwd/rev lexicographic duplicate detection with hieratchical clustering based method
+duplicate_detection_hierarchical_linkage: "average" # Method for hierarchical clustering distance calculation
+duplicate_detection_do_pca: False # Whether to do PCA before hierarchical linkage based duplicate detection.
+
+# Preprocessing - Complexity analysis params
+
+# General Plotting params
+sample_name_col_for_plotting: 'Barcode'
+
+# Basic Analysis - QC Plotting params
+rows_per_qc_histogram_grid: 12
+
+# Basic Analysis - Clustermap params
+layer_for_clustermap_plotting: 'nan0_0minus1'
+
+# Basic Analysis - UMAP/Leiden params
+layer_for_umap_plotting: 'nan_half'
+umap_layers_to_plot:
+  - "mapped_length"
+  - "Raw_modification_signal"
+
+# Basic Analysis - Spatial Autocorrelation params
+rows_per_qc_autocorr_grid: 6
+autocorr_rolling_window_size: 25
+autocorr_max_lag: 800
+autocorr_site_types:
+  - "GpC"
+  - "CpG"
+  - "any_C"
+
+# Basic Analysis - Correlation Matrix params
+correlation_matrix_types:
+  - "pearson"
+  - "binary_covariance"
+correlation_matrix_cmaps:
+  - "seismic"
+  - "viridis"
+correlation_matrix_site_types:
+  - "GpC_site"
+
+# HMM params
+hmm_n_states: 2 # Number of HMM states
+hmm_init_emission_probs:
+  - [0.8, 0.2]
+  - [0.2, 0.8] 
+hmm_init_transition_probs: 
+  - [0.9, 0.1]
+  - [0.1, 0.9]
+hmm_init_start_probs: 
+  - 0.5
+  - 0.5
+hmm_eps: 1e-8
+hmm_dtype: "float64"
+hmm_annotation_threshold: 0.5
+hmm_batch_size: 1024
+hmm_use_viterbi: False
+footprints: True # whether to use the default HMM footprint params
+accessible_patches: True # whether to use the default HMM accessible patch params
+cpg: False # whether to use the default HMM endogenous CpG patch params
+hmm_methbases:
+  - "GpC"
+  - "CpG"
+  - "C"
+  - "A"
+hmm_feature_sets:
+  footprint:
+    state: "Non-Modified"
+    features:
+      small_bound_stretch: [0, 25]
+      medium_bound_stretch: [25, 80]
+      putative_nucleosome: [80, 200]
+      large_bound_stretch: [200, inf]
+  accessible:
+    state: "Modified"
+    features:
+      small_accessible_patch: [0, 20]
+      mid_accessible_patch: [20, 100]
+      large_accessible_patch: [100, inf]
+hmm_merge_layer_features:
+  - [null, 80]
+
+# Pipeline control flow - Preprocessing and QC
+force_redo_preprocessing: False # Whether to force redo the entire preprocessing workflow from the initial raw anndata.
+force_reload_sample_sheet: True # Whether to force redo sample sheet loading
+bypass_add_read_length_and_mapping_qc: False # Whether to skip read length, quality, and mapping qc.
+force_redo_add_read_length_and_mapping_qc: False # Whether to force redo read length, quality, and mapping qc.
+bypass_clean_nan: False # Whether to skip NaN cleaning
+force_redo_clean_nan: False # Whether to redo NaN cleaning
+bypass_append_base_context: False # Whether to skip adding per reference base context additions.
+force_redo_append_base_context: False # Whether to redo per reference base context additions.
+invert_adata: False # Whether to invert the AnnData along the positions axis.
+bypass_append_binary_layer_by_base_context: False # Whether to skip adding new binary layers for each specific base context.
+force_redo_append_binary_layer_by_base_context: False # Whether to redo adding new binary layers for each specific base context.
+bypass_calculate_read_modification_stats: False # Whether to skip adding read level modification statistics.
+force_redo_calculate_read_modification_stats: False # Whether to force redo adding read level modification statistics.
+bypass_filter_reads_on_modification_thresholds: False # Whether to skip filtering reads based on read level modification statistics.
+force_redo_filter_reads_on_modification_thresholds: False # Whether to redo filtering reads based on read level modification statistics.
+bypass_flag_duplicate_reads: False # Whether to skip flagging duplicate reads based on modification similarity.
+force_redo_flag_duplicate_reads: False # Whether to redo flagging duplicate reads based on modification similarity.
+bypass_complexity_analysis: False # Whether to skip complexity analysis
+force_redo_complexity_analysis: False # Whether to redo complexity analysis
+
+# Pipeline control flow - Basic Analyses
+force_redo_basic_analyses: False # Whether to force redo the entire basic analysis pipeline from the AnnData
+bypass_basic_clustermaps: False # Whether to skip basic clustermap plotting
+force_redo_basic_clustermaps: False # Whether to redo basic clustermap plotting
+bypass_basic_umap: False # Whether to skip basic UMAP calculation/plotting
+force_redo_basic_umap: False # Whether to redo basic UMAP calculation/plotting
+bypass_spatial_autocorr_calculations: False # Whether to skip basic spatial autocorrelation calculation
+force_redo_spatial_autocorr_calculations: False # Whether to redo basic spatial autocorrelation calculation
+bypass_spatial_autocorr_plotting: False # Whether to skip basic spatial autocorrelation plotting
+force_redo_spatial_autocorr_plotting: False # Whether to redo basic spatial autocorrelation plotting
+bypass_matrix_corr_calculations: False # Whether to skip basic correlation matrix calculation
+force_redo_matrix_corr_calculations: False # Whether to force redo basic correlation matrix calculation
+bypass_matrix_corr_plotting: False # Whether to skip basic correlation matrix plotting
+force_redo_matrix_corr_plotting: False # Whether to force redo basic correlation matrix calculation
+
+# Pipeline control flow - HMMs
+bypass_hmm_fit: False # Whether to skip HMM fitting for each sample/reference
+force_redo_hmm_fit: False # Whether to redo HMM fitting for each sample/reference
+bypass_hmm_apply: False # Whether to skip HMM application for each sample/reference
+force_redo_hmm_apply: False # Whether to redo HMM application for each sample/reference

smftools/config/direct.yaml

@@ -0,0 +1,17 @@
+# Direct (Nanopore modified base calling)footprinting defaults
+extends: default
+filter_threshold: 0.8 # min threshold to call a canononical base
+m6A_threshold: 0.7 # min threshold to call a modified m6a base
+m5C_threshold: 0.7 # min threshold to call a modified 5mC base
+hm5C_threshold: 0.7 # min threshold to call a modified 5hmC base
+thresholds:
+  - filter_threshold
+  - m6A_threshold
+  - m5C_threshold
+  - hm5C_threshold
+mod_list: 
+  - '5mC_5hmC'
+  - '6mA' # mods to detect
+batch_size: 4 # How many mod TSVs to load into memory at a time when making anndata batches
+skip_unclassified: True # Whether to skip unclassified barcodes
+delete_batch_hdfs: True # Whether to delete intermediate barcode level hdfs after making final anndata