smftools 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

smftools/cli_entry.py

@@ -0,0 +1,435 @@
+import click
+import pandas as pd
+from pathlib import Path
+from typing import Dict, Optional, Sequence
+
+from .cli.load_adata import load_adata
+from .cli.cli_flows import flow_I
+from .cli.preprocess_adata import preprocess_adata
+from .cli.spatial_adata import spatial_adata
+from .cli.hmm_adata import hmm_adata
+
+from .readwrite import merge_barcoded_anndatas_core, safe_read_h5ad, safe_write_h5ad, concatenate_h5ads
+
+@click.group()
+def cli():
+    """Command-line interface for smftools."""
+    pass
+
+####### Load anndata from raw data ###########
+@cli.command()
+@click.argument("config_path", type=click.Path(exists=True))
+def load(config_path):
+    """Load and process data from CONFIG_PATH."""
+    load_adata(config_path)
+##########################################
+
+####### Preprocessing ###########
+@cli.command()
+@click.argument("config_path", type=click.Path(exists=True))
+def preprocess(config_path):
+    """Preprocess data from CONFIG_PATH."""
+    preprocess_adata(config_path)
+##########################################
+
+####### Spatial ###########
+@cli.command()
+@click.argument("config_path", type=click.Path(exists=True))
+def spatial(config_path):
+    """Process data from CONFIG_PATH."""
+    spatial_adata(config_path)
+##########################################
+
+####### HMM ###########
+@cli.command()
+@click.argument("config_path", type=click.Path(exists=True))
+def hmm(config_path):
+    """Process data from CONFIG_PATH."""
+    hmm_adata(config_path)
+##########################################
+
+####### batch command ###########
+@cli.command()
+@click.argument(
+    "task",
+    type=click.Choice(["load", "preprocess", "spatial", "hmm"], case_sensitive=False),
+)
+@click.argument(
+    "config_table",
+    type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path),
+)
+@click.option(
+    "--column",
+    "-c",
+    default="config_path",
+    show_default=True,
+    help="Column name containing config paths (ignored for plain TXT).",
+)
+@click.option(
+    "--sep",
+    default=None,
+    help="Field separator: default auto-detect (.tsv -> '\\t', .csv -> ',', others treated as TXT).",
+)
+def batch(task, config_table: Path, column: str, sep: str | None):
+    """
+    Run a TASK (load, preprocess, spatial, hmm) on multiple CONFIG_PATHs
+    listed in a CSV/TSV or plain TXT file.
+
+    Plain text format: one config path per line, no header.
+    """
+
+    # ----------------------------
+    # Decide file type
+    # ----------------------------
+    suffix = config_table.suffix.lower()
+
+    # TXT mode → each line is a config path
+    if suffix in {".txt", ".list"}:
+        paths = []
+        with config_table.open() as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    paths.append(Path(line).expanduser())
+
+        if not paths:
+            raise click.ClickException(f"No config paths found in text file: {config_table}")
+
+        config_paths = paths
+
+    else:
+        # CSV / TSV mode
+        # auto-detect separator if not provided
+        if sep is None:
+            if suffix in {".tsv", ".tab"}:
+                sep = "\t"
+            else:
+                sep = ","
+
+        try:
+            df = pd.read_csv(config_table, sep=sep, dtype=str)
+        except Exception as e:
+            raise click.ClickException(f"Failed to read table {config_table}: {e}") from e
+
+        if df.empty:
+            raise click.ClickException(f"Config table is empty: {config_table}")
+
+        # If table has no header or only one column, treat it as raw paths
+        if df.shape[1] == 1 and column not in df.columns:
+            # re-read as headerless single-column list, so we don't drop the first path
+            try:
+                df = pd.read_csv(
+                    config_table,
+                    sep=sep,
+                    header=None,
+                    names=[column],
+                    dtype=str,
+                )
+            except Exception as e:
+                raise click.ClickException(f"Failed to read {config_table} as headerless list: {e}") from e
+
+            config_series = df[column]
+        else:
+            if column not in df.columns:
+                raise click.ClickException(
+                    f"Column '{column}' not found in {config_table}. "
+                    f"Available columns: {', '.join(df.columns)}"
+                )
+            config_series = df[column]
+
+        config_paths = (
+            config_series.dropna()
+            .map(str)
+            .map(lambda p: Path(p).expanduser())
+            .tolist()
+        )
+
+    # ----------------------------
+    # Validate config paths
+    # ----------------------------
+    if not config_paths:
+        raise click.ClickException("No config paths found.")
+
+    # ----------------------------
+    # Map task to function
+    # ----------------------------
+    task = task.lower()
+    task_funcs = {
+        "load": load_adata,
+        "preprocess": preprocess_adata,
+        "spatial": spatial_adata,
+        "hmm": hmm_adata,
+    }
+
+    func = task_funcs[task]
+
+    click.echo(
+        f"Running task '{task}' on {len(config_paths)} config paths from {config_table}"
+    )
+
+    # ----------------------------
+    # Loop over paths
+    # ----------------------------
+    for i, cfg in enumerate(config_paths, start=1):
+        if not cfg.exists():
+            click.echo(f"[{i}/{len(config_paths)}] SKIP (missing): {cfg}")
+            continue
+
+        click.echo(f"[{i}/{len(config_paths)}] {task} → {cfg}")
+
+        try:
+            func(str(cfg))   # underlying functions take a string path
+        except Exception as e:
+            click.echo(f"  ERROR on {cfg}: {e}")
+
+    click.echo("Batch processing complete.")
+##########################################
+
+####### concatenate command ###########
+@cli.command("concatenate")
+@click.argument(
+    "output_path",
+    type=click.Path(path_type=Path, dir_okay=False),
+)
+@click.option(
+    "--input-dir",
+    "-d",
+    type=click.Path(path_type=Path, file_okay=False),
+    default=None,
+    help="Directory containing .h5ad/.h5ad.gz files to concatenate.",
+)
+@click.option(
+    "--csv-path",
+    "-c",
+    type=click.Path(path_type=Path, dir_okay=False),
+    default=None,
+    help="CSV/TSV/TXT containing file paths of h5ad files.",
+)
+@click.option(
+    "--csv-column",
+    "-C",
+    default="h5ad_path",
+    help="Column in the CSV containing file paths (ignored for TXT).",
+    show_default=True,
+)
+@click.option(
+    "--suffix",
+    "-s",
+    multiple=True,
+    default=[".h5ad", ".h5ad.gz"],
+    help="Allowed file suffixes (repeatable).",
+    show_default=True,
+)
+@click.option(
+    "--delete",
+    is_flag=False,
+    help="Delete input .h5ad files after concatenation.",
+)
+@click.option(
+    "--restore",
+    is_flag=True,
+    help="Restore .h5ad backups during reading.",
+)
+def concatenate_cmd(
+    output_path: Path,
+    input_dir: Path | None,
+    csv_path: Path | None,
+    csv_column: str,
+    suffix: Sequence[str],
+    delete: bool,
+    restore: bool,
+):
+    """
+    Concatenate multiple .h5ad files into a single output file.
+
+    Two modes:
+
+        smftools concatenate out.h5ad --input-dir ./dir
+
+        smftools concatenate out.h5ad --csv-path paths.csv --csv-column h5ad_path
+
+    TXT input also works (one file path per line).
+
+    Uses safe_read_h5ad() and safe_write_h5ad().
+    """
+
+    if input_dir and csv_path:
+        raise click.ClickException("Provide only ONE of --input-dir or --csv-path.")
+
+    try:
+        out = concatenate_h5ads(
+            output_path=output_path,
+            input_dir=input_dir,
+            csv_path=csv_path,
+            csv_column=csv_column,
+            file_suffixes=tuple(suffix),
+            delete_inputs=delete,
+            restore_backups=restore,
+        )
+        click.echo(f"✓ Concatenated file written to: {out}")
+
+    except Exception as e:
+        raise click.ClickException(str(e)) from e
+##########################################
+
+####### Merging existing anndatas from an experiment that used two different demultiplexing rules #######
+# REQUIRED_KEYS = ("adata_single_path", "adata_double_path")
+# OPTIONAL_KEYS = (
+#     "adata_single_backups_path",
+#     "adata_double_backups_path",
+#     "output_path",
+#     "merged_filename",
+# )
+
+# def _read_config_csv(csv_path: Path) -> Dict[str, str]:
+#     """
+#     Read a multi-row, two-column CSV of key,value pairs into a dict.
+
+#     Supported features:
+#       - Optional header ("key,value") or none.
+#       - Comments starting with '#' and blank lines are ignored.
+#       - If duplicate keys occur, the last one wins.
+#       - Keys are matched literally against REQUIRED_KEYS/OPTIONAL_KEYS.
+#     """
+#     try:
+#         # Read as two columns regardless of header; comments ignored.
+#         df = pd.read_csv(
+#             csv_path,
+#             dtype=str,
+#             comment="#",
+#             header=None,          # treat everything as rows; we'll normalize below
+#             usecols=[0, 1],
+#             names=["key", "value"]
+#         )
+#     except Exception as e:
+#         raise click.ClickException(f"Failed to read CSV: {e}") from e
+
+#     # Drop completely empty rows
+#     df = df.fillna("").astype(str)
+#     df["key"] = df["key"].str.strip()
+#     df["value"] = df["value"].str.strip()
+#     df = df[(df["key"] != "") & (df["key"].notna())]
+
+#     if df.empty:
+#         raise click.ClickException("Config CSV is empty after removing comments/blank lines.")
+
+#     # Remove an optional header row if present
+#     if df.iloc[0]["key"].lower() in {"key", "keys"}:
+#         df = df.iloc[1:]
+#         df = df[(df["key"] != "") & (df["key"].notna())]
+#         if df.empty:
+#             raise click.ClickException("Config CSV contains only a header row.")
+
+#     # Build dict; last occurrence of a key wins
+#     cfg = {}
+#     for k, v in zip(df["key"], df["value"]):
+#         cfg[k] = v
+
+#     # Validate required keys
+#     missing = [k for k in REQUIRED_KEYS if not cfg.get(k)]
+#     if missing:
+#         raise click.ClickException(
+#             "Missing required keys in CSV: "
+#             + ", ".join(missing)
+#             + "\nExpected keys:\n  - "
+#             + "\n  - ".join(REQUIRED_KEYS)
+#             + "\nOptional keys:\n  - "
+#             + "\n  - ".join(OPTIONAL_KEYS)
+#         )
+
+#     return cfg
+
+# def _resolve_output_path(cfg: Dict[str, str], single_path: Path, double_path: Path) -> Path:
+#     """Decide on the output .h5ad path based on CSV; create directories if needed."""
+#     merged_filename = cfg.get("merged_filename") or f"merged_{single_path.stem}__{double_path.stem}.h5ad"
+#     if not merged_filename.endswith(".h5ad"):
+#         merged_filename += ".h5ad"
+
+#     output_path_raw = cfg.get("output_path", "").strip()
+
+#     if not output_path_raw:
+#         out_dir = Path.cwd() / "merged_output"
+#         out_dir.mkdir(parents=True, exist_ok=True)
+#         return out_dir / merged_filename
+
+#     output_path = Path(output_path_raw)
+
+#     if output_path.suffix.lower() == ".h5ad":
+#         output_path.parent.mkdir(parents=True, exist_ok=True)
+#         return output_path
+
+#     # Treat as directory
+#     output_path.mkdir(parents=True, exist_ok=True)
+#     return output_path / merged_filename
+
+# def _maybe_read_adata(label: str, primary: Path, backups: Optional[Path]):
+
+#     if backups:
+#         click.echo(f"Loading {label} from {primary} with backups at {backups} ...")
+#         return safe_read_h5ad(primary, backups_path=backups, restore_backups=True)
+#     else:
+#         click.echo(f"Loading {label} from {primary} with backups disabled ...")
+#         return safe_read_h5ad(primary, restore_backups=False)
+
+
+# @cli.command()
+# @click.argument("config_path", type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path))
+# def merge_barcoded_anndatas(config_path: Path):
+#     """
+#     Merge two AnnData objects from the same experiment that were demultiplexed
+#     under different end-barcoding requirements, using a 1-row CSV for config.
+
+#     CSV must include:
+#       - adata_single_path
+#       - adata_double_path
+
+#     Optional columns:
+#       - adata_single_backups_path
+#       - adata_double_backups_path
+#       - output_path            (file or directory; default: ./merged_output/)
+#       - merged_filename        (default: merged_<single>__<double>.h5ad)
+
+#     Example CSV:
+
+#         adata_single_path,adata_double_path,adata_single_backups_path,adata_double_backups_path,output_path,merged_filename
+#         /path/single.h5ad,/path/double.h5ad,,,,merged_output,merged_run.h5ad
+#     """
+#     try:
+#         cfg = _read_config_csv(config_path)
+
+#         single_path = Path(cfg["adata_single_path"]).expanduser().resolve()
+#         double_path = Path(cfg["adata_double_path"]).expanduser().resolve()
+
+#         for p, label in [(single_path, "adata_single_path"), (double_path, "adata_double_path")]:
+#             if not p.exists():
+#                 raise click.ClickException(f"{label} does not exist: {p}")
+
+#         single_backups = Path(cfg["adata_single_backups_path"]).expanduser().resolve() if cfg.get("adata_single_backups_path") else None
+#         double_backups = Path(cfg["adata_double_backups_path"]).expanduser().resolve() if cfg.get("adata_double_backups_path") else None
+
+#         if single_backups and not single_backups.exists():
+#             raise click.ClickException(f"adata_single_backups_path does not exist: {single_backups}")
+#         if double_backups and not double_backups.exists():
+#             raise click.ClickException(f"adata_double_backups_path does not exist: {double_backups}")
+
+#         output_path = _resolve_output_path(cfg, single_path, double_path)
+
+#         # Load
+#         adata_single, read_report_single = _maybe_read_adata("single-barcoded AnnData", single_path, single_backups)
+#         adata_double, read_report_double = _maybe_read_adata("double-barcoded AnnData", double_path, double_backups)
+
+#         click.echo("Merging AnnDatas ...")
+#         merged = merge_barcoded_anndatas_core(adata_single, adata_double)
+
+#         click.echo(f"Writing merged AnnData to: {output_path}")
+#         backup_dir = output_path.cwd() / "merged_backups"
+#         safe_write_h5ad(merged, output_path, backup=True, backup_dir=backup_dir)
+
+#         click.secho(f"Done. Merged AnnData saved to {output_path}", fg="green")
+
+#     except click.ClickException:
+#         raise
+#     except Exception as e:
+#         # Surface unexpected errors cleanly
+#         raise click.ClickException(f"Unexpected error: {e}") from e
+################################################################################################################

smftools/config/conversion.yaml

@@ -1,11 +1,15 @@
 # Conversion (Bisulfite/APOBEC)footprinting defaults
 extends: default
+
+######## smftools load params #########
 conversion_types: 
   - '5mC' # 5mC
 
+######## smftools preprocess params #########
 # Read QC Params
 read_mod_filtering_use_other_c_as_background: True
 
+######## smftools hmm params #########
 # HMM
 cpg: True # whether to use the default HMM endogenous CpG patch params
 hmm_methbases:
@@ -14,16 +18,17 @@ hmm_feature_sets:
   footprint:
     state: "Non-Modified"
     features:
-      small_bound_stretch: [0, 20]
-      medium_bound_stretch: [20, 50]
-      putative_nucleosome: [50, 200]
+      small_bound_stretch: [10, 30]
+      medium_bound_stretch: [30, 110]
+      putative_nucleosome: [110, 200]
       large_bound_stretch: [200, inf]
   accessible:
     state: "Modified"
     features:
-      small_accessible_patch: [0, 20]
-      mid_accessible_patch: [20, 80]
-      large_accessible_patch: [80, inf]
+      small_accessible_patch: [3, 20]
+      mid_accessible_patch: [20, 40]
+      mid_large_accessible_patch: [40, 130]
+      large_accessible_patch: [130, inf]
   cpg:
     state: "Modified"
     features:

smftools/config/deaminase.yaml

@@ -1,11 +1,14 @@
 # Deaminase footprinting defaults
 extends: default
+
+######## smftools load params #########
 conversion_types: 
   - '5mC' # 5mC
 
 mod_target_bases:
 - "C"
 
+######## smftools preprocess params #########
 read_mod_filtering_gpc_thresholds:
   - null
   - null
@@ -25,6 +28,7 @@ read_mod_filtering_use_other_c_as_background: False
 duplicate_detection_site_types:
   - "any_C"
 
+######## smftools analyze params #########
 # Autocorrelation params
 autocorr_site_types:
   - "any_C"
@@ -33,7 +37,7 @@ autocorr_site_types:
 correlation_matrix_site_types:
   - "any_C_site"
 
-# HMM
+# ######## smftools hmm params #########
 cpg: False # whether to use the default HMM endogenous CpG patch params
 hmm_methbases:
   - "C"
@@ -41,16 +45,17 @@ hmm_feature_sets:
   footprint:
     state: "Non-Modified"
     features:
-      small_bound_stretch: [0, 25]
-      medium_bound_stretch: [25, 80]
-      putative_nucleosome: [80, 200]
+      small_bound_stretch: [10, 30]
+      medium_bound_stretch: [30, 110]
+      putative_nucleosome: [110, 200]
       large_bound_stretch: [200, inf]
   accessible:
     state: "Modified"
     features:
-      small_accessible_patch: [0, 20]
-      mid_accessible_patch: [20, 100]
-      large_accessible_patch: [100, inf]
+      small_accessible_patch: [3, 20]
+      mid_accessible_patch: [20, 40]
+      mid_large_accessible_patch: [40, 130]
+      large_accessible_patch: [130, inf]
 
 hmm_merge_layer_features:
   - ["C_all_accessible_features", 80]

smftools/config/default.yaml

@@ -1,3 +1,13 @@
+# General
+sample_sheet_path: null # path to sample_sheet to load metadata into anndata.
+sample_sheet_mapping_column: 'Barcode' # The column in the sample sheet and current anndata to use for mapping metadata.
+sample_name_col_for_plotting: 'Barcode'
+
+# Compute params
+threads: 4
+device: "auto"
+
+######## smftools load params #########
 # Generic i/o
 bam_suffix: ".bam"
 recursive_input_search: True
@@ -7,16 +17,12 @@ strands:
   - top
 conversions:
   - unconverted
-sample_sheet_path: null # path to sample_sheet to load metadata into anndata.
-sample_sheet_mapping_column: 'Barcode' # The column in the sample sheet and current anndata to use for mapping metadata.
 fastq_barcode_map: null # For FASTQ files, an optional map of file paths to barcodes can be provided. Default is autodetecting barcodes.
 fastq_auto_pairing: True # For FASTQ files, attempt to find read pair files automatically.
 input_already_demuxed: False # If the input files are already demultiplexed.
 delete_intermediate_hdfs: True # Whether to delete the intermediate hdfs from the conversion/deamination workflows.
-
-# Compute params
-threads: 4
-device: "auto"
+delete_intermediate_bams: True # Whether to delete intermediate BAM files.
+delete_intermediate_tsvs: True # Whether to delete intermediate TSV files.
 
 # Sequencing modality and general experiment params
 smf_modality: 'conversion' # conversion, deaminase, direct
@@ -70,11 +76,11 @@ aligner_args:
   dorado:
     ont:
       - "--mm2-opts"
-      - "-N"
-      - "5"
+      - "-N 5"
 
 # Sorted BAM and BED specific handling
 make_bigwigs: False # Whether to make coverage bigwigs
+make_beds: False # Whether to make beds from the aligned bams
 
 # Nanopore specific demultiplexing
 barcode_both_ends: False # dorado demultiplexing
@@ -85,24 +91,25 @@ mapping_threshold: 0.01 # Minimum proportion of mapped reads that need to fall w
 reference_column: 'Reference_strand'
 sample_column: 'Barcode'
 
-# Preprocessing -  Read length, quality, and mapping filtering params
+######## smftools preprocess params #########
+# Read length, quality, and mapping filtering params
 read_coord_filter:
  - null
  - null
 read_len_filter_thresholds:
-  - 200
+  - 100
   - null
 read_len_to_ref_ratio_filter_thresholds:
-  - 0.8
+  - 0.5
   - null
 read_quality_filter_thresholds:
-  - 20
+  - 15
   - null
 read_mapping_quality_filter_thresholds:
   - null
   - null
 
-# Preprocessing -  Read modification filtering params
+# Read modification filtering params
 read_mod_filtering_gpc_thresholds:
   - 0.025
   - 0.975
@@ -116,9 +123,9 @@ read_mod_filtering_a_thresholds:
   - 0.025
   - 0.975
 read_mod_filtering_use_other_c_as_background: False
-min_valid_fraction_positions_in_read_vs_ref: 0.8
+min_valid_fraction_positions_in_read_vs_ref: 0.5
 
-# Preprocessing - Duplicate detection params
+# Duplicate detection params
 duplicate_detection_site_types: # Site types to consider for duplicate detection workflow
   - "GpC"
   - "CpG"
@@ -133,11 +140,10 @@ duplicate_detection_do_hierarchical: True # Whether to follow up fwd/rev lexicog
 duplicate_detection_hierarchical_linkage: "average" # Method for hierarchical clustering distance calculation
 duplicate_detection_do_pca: False # Whether to do PCA before hierarchical linkage based duplicate detection.
 
-# Preprocessing - Complexity analysis params
-
-# General Plotting params
-sample_name_col_for_plotting: 'Barcode'
+# Position QC params
+position_max_nan_threshold: 0.1 # The maximum amount of nans to tolerate in a column
 
+######## smftools analyze params #########
 # Basic Analysis - QC Plotting params
 rows_per_qc_histogram_grid: 12
 
@@ -169,6 +175,7 @@ correlation_matrix_cmaps:
 correlation_matrix_site_types:
   - "GpC_site"
 
+######## smftools hmm params #########
 # HMM params
 hmm_n_states: 2 # Number of HMM states
 hmm_init_emission_probs:
@@ -197,19 +204,23 @@ hmm_feature_sets:
   footprint:
     state: "Non-Modified"
     features:
-      small_bound_stretch: [0, 25]
-      medium_bound_stretch: [25, 80]
-      putative_nucleosome: [80, 200]
+      small_bound_stretch: [10, 40]
+      medium_bound_stretch: [40, 110]
+      putative_nucleosome: [110, 200]
       large_bound_stretch: [200, inf]
   accessible:
     state: "Modified"
     features:
-      small_accessible_patch: [0, 20]
-      mid_accessible_patch: [20, 100]
-      large_accessible_patch: [100, inf]
+      small_accessible_patch: [3, 20]
+      mid_accessible_patch: [20, 40]
+      mid_large_accessible_patch: [40, 110]
+      large_accessible_patch: [110, inf]
 hmm_merge_layer_features:
   - [null, 80]
 
+# Pipeline control flow - load adata
+force_redo_load_adata: False # Whether to perform load adata command from start
+
 # Pipeline control flow - Preprocessing and QC
 force_redo_preprocessing: False # Whether to force redo the entire preprocessing workflow from the initial raw anndata.
 force_reload_sample_sheet: True # Whether to force redo sample sheet loading