smftools 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

smftools/cli/spatial_adata.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
+import logging
 from pathlib import Path
 from typing import Optional, Tuple
 
 import anndata as ad
 
-from smftools.logging_utils import get_logger
+from smftools.constants import LOGGING_DIR, SEQUENCE_INTEGER_ENCODING, SPATIAL_DIR
+from smftools.logging_utils import get_logger, setup_logging
 from smftools.optional_imports import require
 
 logger = get_logger(__name__)
@@ -35,15 +37,13 @@ def spatial_adata(
         Path to the “current” spatial AnnData (or hmm AnnData if we skip to that).
     """
     from ..readwrite import add_or_update_column_in_csv, safe_read_h5ad
-    from .helpers import get_adata_paths
-    from .load_adata import load_adata
-    from .preprocess_adata import preprocess_adata
+    from .helpers import get_adata_paths, load_experiment_config
 
     # 1) Ensure config + basic paths via load_adata
-    loaded_adata, loaded_path, cfg = load_adata(config_path)
+    cfg = load_experiment_config(config_path)
+
     paths = get_adata_paths(cfg)
 
-    raw_path = paths.raw
     pp_path = paths.pp
     pp_dedup_path = paths.pp_dedup
     spatial_path = paths.spatial
@@ -51,47 +51,34 @@ def spatial_adata(
 
     # Stage-skipping logic for spatial
     if not getattr(cfg, "force_redo_spatial_analyses", False):
-        # If HMM exists, it's the most processed stage — reuse it.
-        if hmm_path.exists():
-            logger.info(f"HMM AnnData found: {hmm_path}\nSkipping smftools spatial")
-            return None, hmm_path
-
         # If spatial exists, we consider spatial analyses already done.
         if spatial_path.exists():
             logger.info(f"Spatial AnnData found: {spatial_path}\nSkipping smftools spatial")
             return None, spatial_path
 
-    # 2) Ensure preprocessing has been run
-    #    This will create pp/pp_dedup as needed or return them if they already exist.
-    pp_adata, pp_adata_path_ret, pp_dedup_adata, pp_dedup_adata_path_ret = preprocess_adata(
-        config_path
-    )
-
     # Helper to load from disk, reusing loaded_adata if it matches
     def _load(path: Path):
-        if loaded_adata is not None and loaded_path == path:
-            return loaded_adata
         adata, _ = safe_read_h5ad(path)
         return adata
 
     # 3) Decide which AnnData to use as the *starting point* for spatial analyses
-    # Prefer in-memory pp_dedup_adata when preprocess_adata just ran.
-    if pp_dedup_adata is not None:
-        start_adata = pp_dedup_adata
-        source_path = pp_dedup_adata_path_ret
+    if hmm_path.exists():
+        start_adata = _load(hmm_path)
+        source_path = hmm_path
+    elif spatial_path.exists():
+        start_adata = _load(spatial_path)
+        source_path = spatial_path
+    elif pp_dedup_path.exists():
+        start_adata = _load(pp_dedup_path)
+        source_path = pp_dedup_path
+    elif pp_path.exists():
+        start_adata = _load(pp_path)
+        source_path = pp_path
     else:
-        if pp_dedup_path.exists():
-            start_adata = _load(pp_dedup_path)
-            source_path = pp_dedup_path
-        elif pp_path.exists():
-            start_adata = _load(pp_path)
-            source_path = pp_path
-        elif raw_path.exists():
-            start_adata = _load(raw_path)
-            source_path = raw_path
-        else:
-            logger.warning("No suitable AnnData found for spatial analyses (need at least raw).")
-            return None, None
+        logger.warning(
+            "No suitable AnnData found for spatial analyses (need at least preprocessed)."
+        )
+        return None, None
 
     # 4) Run the spatial core
     adata_spatial, spatial_path = spatial_adata_core(
@@ -99,15 +86,10 @@ def spatial_adata(
         cfg=cfg,
         spatial_adata_path=spatial_path,
         pp_adata_path=pp_path,
-        pp_dup_rem_adata_path=pp_dedup_path,
-        pp_adata_in_memory=pp_adata,
         source_adata_path=source_path,
         config_path=config_path,
     )
 
-    # 5) Register spatial path in summary CSV
-    add_or_update_column_in_csv(cfg.summary_file, "spatial_adata", spatial_path)
-
     return adata_spatial, spatial_path
 
 
@@ -116,8 +98,6 @@ def spatial_adata_core(
     cfg,
     spatial_adata_path: Path,
     pp_adata_path: Path,
-    pp_dup_rem_adata_path: Path,
-    pp_adata_in_memory: Optional[ad.AnnData] = None,
     source_adata_path: Optional[Path] = None,
     config_path: Optional[str] = None,
 ) -> Tuple[ad.AnnData, Path]:
@@ -129,8 +109,6 @@ def spatial_adata_core(
     - `cfg` is the ExperimentConfig.
     - `spatial_adata_path`, `pp_adata_path`, `pp_dup_rem_adata_path` are canonical paths
       from `get_adata_paths`.
-    - `pp_adata_in_memory` optionally holds the preprocessed (non-dedup) AnnData from
-      the same run of `preprocess_adata`, to avoid re-reading from disk.
 
     Does:
     - Optional sample sheet load.
@@ -152,17 +130,17 @@ def spatial_adata_core(
     """
     import os
     import warnings
+    from datetime import datetime
     from pathlib import Path
 
     import numpy as np
     import pandas as pd
 
-    sc = require("scanpy", extra="scanpy", purpose="spatial analyses")
-
     from ..metadata import record_smftools_metadata
     from ..plotting import (
         combined_raw_clustermap,
         plot_rolling_grid,
+        plot_rolling_nn_and_layer,
         plot_spatial_autocorr_grid,
     )
     from ..preprocessing import (
@@ -171,11 +149,12 @@ def spatial_adata_core(
         reindex_references_adata,
     )
     from ..readwrite import make_dirs, safe_read_h5ad
-    from ..tools import calculate_umap
+    from ..tools import rolling_window_nn_distance
     from ..tools.position_stats import (
         compute_positionwise_statistics,
         plot_positionwise_matrices,
     )
+    from ..tools.rolling_nn_distance import assign_rolling_nn_results
     from ..tools.spatial_autocorrelation import (
         analyze_autocorr_matrix,
         binary_autocorrelation_with_spacing,
@@ -187,8 +166,24 @@ def spatial_adata_core(
     # -----------------------------
     # General setup
     # -----------------------------
+    date_str = datetime.today().strftime("%y%m%d")
+    now = datetime.now()
+    time_str = now.strftime("%H%M%S")
+    log_level = getattr(logging, cfg.log_level.upper(), logging.INFO)
+
     output_directory = Path(cfg.output_directory)
-    make_dirs([output_directory])
+    spatial_directory = output_directory / SPATIAL_DIR
+    logging_directory = spatial_directory / LOGGING_DIR
+
+    make_dirs([output_directory, spatial_directory])
+
+    if cfg.emit_log_file:
+        log_file = logging_directory / f"{date_str}_{time_str}_log.log"
+        make_dirs([logging_directory])
+    else:
+        log_file = None
+
+    setup_logging(level=log_level, log_file=log_file, reconfigure=log_file is not None)
 
     smf_modality = cfg.smf_modality
     if smf_modality == "conversion":
@@ -196,8 +191,6 @@ def spatial_adata_core(
     else:
         deaminase = True
 
-    first_pp_run = pp_adata_in_memory is not None and pp_dup_rem_adata_path.exists()
-
     # -----------------------------
     # Optional sample sheet metadata
     # -----------------------------
@@ -231,7 +224,6 @@ def spatial_adata_core(
     else:
         reindex_suffix = None
 
-    pp_dir = output_directory / "preprocessed"
     references = adata.obs[cfg.reference_column].cat.categories
 
     # ============================================================
@@ -241,7 +233,7 @@ def spatial_adata_core(
         preprocessed_version_available = pp_adata_path.exists()
 
         if preprocessed_version_available:
-            pp_clustermap_dir = pp_dir / "06_clustermaps"
+            pp_clustermap_dir = spatial_directory / "06_clustermaps"
 
             if pp_clustermap_dir.is_dir() and not getattr(
                 cfg, "force_redo_spatial_analyses", False
@@ -250,12 +242,9 @@ def spatial_adata_core(
                     f"{pp_clustermap_dir} already exists. Skipping clustermap plotting for preprocessed AnnData."
                 )
             else:
-                make_dirs([pp_dir, pp_clustermap_dir])
+                make_dirs([spatial_directory, pp_clustermap_dir])
 
-                if first_pp_run and (pp_adata_in_memory is not None):
-                    pp_adata = pp_adata_in_memory
-                else:
-                    pp_adata, _ = safe_read_h5ad(pp_adata_path)
+                pp_adata, _ = safe_read_h5ad(pp_adata_path)
 
                 # -----------------------------
                 # Optional sample sheet metadata
@@ -304,7 +293,7 @@ def spatial_adata_core(
                         0
                     ],
                     min_position_valid_fraction=cfg.min_valid_fraction_positions_in_read_vs_ref,
-                    demux_types=("double", "already"),
+                    demux_types=cfg.clustermap_demux_types_to_plot,
                     bins=None,
                     sample_mapping=None,
                     save_path=pp_clustermap_dir,
@@ -314,19 +303,18 @@ def spatial_adata_core(
                 )
 
     # ============================================================
-    # 2) Clustermaps + UMAP on *deduplicated* preprocessed AnnData
+    # 2) Clustermaps on *deduplicated* preprocessed AnnData
     # ============================================================
-    pp_dir_dedup = pp_dir / "deduplicated"
-    pp_clustermap_dir_dedup = pp_dir_dedup / "06_clustermaps"
-    pp_umap_dir = pp_dir_dedup / "07_umaps"
+    spatial_dir_dedup = spatial_directory / "deduplicated"
+    clustermap_dir_dedup = spatial_dir_dedup / "06_clustermaps"
 
     # Clustermaps on deduplicated adata
-    if pp_clustermap_dir_dedup.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
+    if clustermap_dir_dedup.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
         logger.debug(
-            f"{pp_clustermap_dir_dedup} already exists. Skipping clustermap plotting for deduplicated AnnData."
+            f"{clustermap_dir_dedup} already exists. Skipping clustermap plotting for deduplicated AnnData."
         )
     else:
-        make_dirs([pp_dir_dedup, pp_clustermap_dir_dedup])
+        make_dirs([spatial_dir_dedup, clustermap_dir_dedup])
         combined_raw_clustermap(
             adata,
             sample_col=cfg.sample_name_col_for_plotting,
@@ -346,53 +334,113 @@ def spatial_adata_core(
                 0
             ],
             min_position_valid_fraction=1 - cfg.position_max_nan_threshold,
-            demux_types=("double", "already"),
+            demux_types=cfg.clustermap_demux_types_to_plot,
             bins=None,
             sample_mapping=None,
-            save_path=pp_clustermap_dir_dedup,
+            save_path=clustermap_dir_dedup,
             sort_by=cfg.spatial_clustermap_sortby,
             deaminase=deaminase,
             index_col_suffix=reindex_suffix,
         )
 
-    # UMAP / Leiden
-    if pp_umap_dir.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
-        logger.debug(f"{pp_umap_dir} already exists. Skipping UMAP plotting.")
+    # ============================================================
+    # 2b) Rolling NN distances + layer clustermaps
+    # ============================================================
+    pp_rolling_nn_dir = spatial_dir_dedup / "06b_rolling_nn_clustermaps"
+
+    if pp_rolling_nn_dir.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
+        logger.debug(f"{pp_rolling_nn_dir} already exists. Skipping rolling NN distance plots.")
     else:
-        make_dirs([pp_umap_dir])
-
-        var_filters = []
-        if smf_modality == "direct":
-            for ref in references:
-                for base in cfg.mod_target_bases:
-                    var_filters.append(f"{ref}_{base}_site")
-        elif deaminase:
-            for ref in references:
-                var_filters.append(f"{ref}_C_site")
-        else:
-            for ref in references:
-                for base in cfg.mod_target_bases:
-                    var_filters.append(f"{ref}_{base}_site")
-
-        adata = calculate_umap(
-            adata,
-            layer=cfg.layer_for_umap_plotting,
-            var_filters=var_filters,
-            n_pcs=10,
-            knn_neighbors=15,
+        make_dirs([pp_rolling_nn_dir])
+        samples = (
+            adata.obs[cfg.sample_name_col_for_plotting].astype("category").cat.categories.tolist()
         )
+        references = adata.obs[cfg.reference_column].astype("category").cat.categories.tolist()
 
-        sc.tl.leiden(adata, resolution=0.1, flavor="igraph", n_iterations=2)
+        for reference in references:
+            for sample in samples:
+                mask = (adata.obs[cfg.sample_name_col_for_plotting] == sample) & (
+                    adata.obs[cfg.reference_column] == reference
+                )
+                if not mask.any():
+                    continue
+
+                subset = adata[mask]
+                site_mask = (
+                    adata.var[[f"{reference}_{st}_site" for st in cfg.rolling_nn_site_types]]
+                    .fillna(False)
+                    .any(axis=1)
+                )
+                subset = subset[:, site_mask].copy()
+                try:
+                    rolling_values, rolling_starts = rolling_window_nn_distance(
+                        subset,
+                        layer=cfg.rolling_nn_layer,
+                        window=cfg.rolling_nn_window,
+                        step=cfg.rolling_nn_step,
+                        min_overlap=cfg.rolling_nn_min_overlap,
+                        return_fraction=cfg.rolling_nn_return_fraction,
+                        store_obsm=cfg.rolling_nn_obsm_key,
+                    )
+                except Exception as exc:
+                    logger.warning(
+                        "Rolling NN distance computation failed for sample=%s ref=%s: %s",
+                        sample,
+                        reference,
+                        exc,
+                    )
+                    continue
 
-        sc.settings.figdir = pp_umap_dir
-        umap_layers = ["leiden", cfg.sample_name_col_for_plotting, "Reference_strand"]
-        umap_layers += cfg.umap_layers_to_plot
-        sc.pl.umap(adata, color=umap_layers, show=False, save=True)
+                safe_sample = str(sample).replace(os.sep, "_")
+                safe_ref = str(reference).replace(os.sep, "_")
+                parent_obsm_key = f"{cfg.rolling_nn_obsm_key}__{safe_ref}"
+                try:
+                    assign_rolling_nn_results(
+                        adata,
+                        subset,
+                        rolling_values,
+                        rolling_starts,
+                        obsm_key=parent_obsm_key,
+                        window=cfg.rolling_nn_window,
+                        step=cfg.rolling_nn_step,
+                        min_overlap=cfg.rolling_nn_min_overlap,
+                        return_fraction=cfg.rolling_nn_return_fraction,
+                        layer=cfg.rolling_nn_layer,
+                    )
+                except Exception as exc:
+                    logger.warning(
+                        "Failed to merge rolling NN results for sample=%s ref=%s: %s",
+                        sample,
+                        reference,
+                        exc,
+                    )
+                adata.uns.setdefault(f"{cfg.rolling_nn_obsm_key}_reference_map", {})[reference] = (
+                    parent_obsm_key
+                )
+                out_png = pp_rolling_nn_dir / f"{safe_sample}__{safe_ref}.png"
+                title = f"{sample} {reference}"
+                try:
+                    plot_rolling_nn_and_layer(
+                        subset,
+                        obsm_key=cfg.rolling_nn_obsm_key,
+                        layer_key=cfg.rolling_nn_plot_layer,
+                        max_nan_fraction=cfg.position_max_nan_threshold,
+                        var_valid_fraction_col=f"{reference}_valid_fraction",
+                        title=title,
+                        save_name=out_png,
+                    )
+                except Exception as exc:
+                    logger.warning(
+                        "Failed rolling NN plot for sample=%s ref=%s: %s",
+                        sample,
+                        reference,
+                        exc,
+                    )
 
     # ============================================================
     # 3) Spatial autocorrelation + rolling metrics
     # ============================================================
-    pp_autocorr_dir = pp_dir_dedup / "08_autocorrelations"
+    pp_autocorr_dir = spatial_dir_dedup / "08_autocorrelations"
 
     if pp_autocorr_dir.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
         logger.debug(f"{pp_autocorr_dir} already exists. Skipping autocorrelation plotting.")
@@ -735,10 +783,10 @@ def spatial_adata_core(
     # ============================================================
     # 4) Pearson / correlation matrices
     # ============================================================
-    pp_corr_dir = pp_dir_dedup / "09_correlation_matrices"
+    corr_dir = spatial_dir_dedup / "09_correlation_matrices"
 
-    if pp_corr_dir.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
-        logger.debug(f"{pp_corr_dir} already exists. Skipping correlation matrix plotting.")
+    if corr_dir.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
+        logger.debug(f"{corr_dir} already exists. Skipping correlation matrix plotting.")
     else:
         compute_positionwise_statistics(
             adata,
@@ -763,7 +811,7 @@ def spatial_adata_core(
             cmaps=cfg.correlation_matrix_cmaps,
             vmin=None,
             vmax=None,
-            output_dir=pp_corr_dir,
+            output_dir=corr_dir,
             output_key="positionwise_result",
         )
 

smftools/cli_entry.py

@@ -8,6 +8,7 @@ import click
 import pandas as pd
 
 from .cli.hmm_adata import hmm_adata
+from .cli.latent_adata import latent_adata
 from .cli.load_adata import load_adata
 from .cli.preprocess_adata import preprocess_adata
 from .cli.spatial_adata import spatial_adata
@@ -103,6 +104,17 @@ def hmm(config_path):
 ##########################################
 
 
+####### Latent ###########
+@cli.command()
+@click.argument("config_path", type=click.Path(exists=True))
+def latent(config_path):
+    """Process data from CONFIG_PATH."""
+    latent_adata(config_path)
+
+
+##########################################
+
+
 ####### batch command ###########
 @cli.command()
 @click.argument(

smftools/config/conversion.yaml

@@ -15,6 +15,16 @@ autocorr_site_types:
 
 # Spatial Analysis - Clustermap params
 layer_for_clustermap_plotting: 'nan0_0minus1'
+rolling_nn_layer: "nan0_0minus1"
+rolling_nn_plot_layer: "nan0_0minus1"
+rolling_nn_window: 30
+rolling_nn_step: 2
+rolling_nn_min_overlap: 20
+rolling_nn_return_fraction: true
+rolling_nn_obsm_key: "rolling_nn_dist"
+rolling_nn_site_types: 
+  - "GpC"
+  - "CpG"
 clustermap_cmap_c: "coolwarm"
 clustermap_cmap_gpc: "coolwarm"
 clustermap_cmap_cpg: "viridis"
@@ -46,4 +56,4 @@ hmm_feature_sets:
       cpg_patch: [0, inf]
 
 hmm_merge_layer_features:
-  - ["all_accessible_features", 60]
+  - ["all_accessible_features", 60]

smftools/config/default.yaml

@@ -18,8 +18,9 @@ conversions:
 fastq_barcode_map: null # For FASTQ files, an optional map of file paths to barcodes can be provided. Default is autodetecting barcodes.
 fastq_auto_pairing: True # For FASTQ files, attempt to find read pair files automatically.
 input_already_demuxed: False # If the input files are already demultiplexed.
+
 delete_intermediate_hdfs: True # Whether to delete the intermediate hdfs from the conversion/deamination workflows.
-delete_intermediate_bams: True # Whether to delete intermediate BAM files.
+delete_intermediate_bams: False # Whether to delete intermediate BAM files.
 delete_intermediate_tsvs: True # Whether to delete intermediate TSV files.
 
 # Sequencing modality and general experiment params
@@ -77,6 +78,7 @@ aligner_args:
 # Sorted BAM and BED specific handling
 make_bigwigs: False # Whether to make coverage bigwigs
 make_beds: False # Whether to make beds from the aligned bams
+annotate_secondary_supplementary: True # Whether to annotate reads with secondary/supplementary alignments from the aligned BAM
 samtools_backend: auto # auto|python|cli for samtools-compatible operations
 bedtools_backend: auto # auto|python|cli for bedtools-compatible operations
 bigwig_backend: auto # auto|python|cli for bedGraphToBigWig conversion
@@ -90,6 +92,12 @@ mapping_threshold: 0.10 # Minimum proportion of mapped reads that need to fall w
 reference_column: 'Reference_strand'
 sample_column: 'Experiment_name_and_barcode'
 
+# Plotting params
+clustermap_demux_types_to_plot: 
+  - "single"
+  - "double"
+  - "already"
+
 ######## smftools preprocess params #########
 # Read length, quality, and mapping filtering params
 read_coord_filter:
@@ -140,6 +148,10 @@ duplicate_detection_site_types: # Site types to consider for duplicate detection
   - "CpG"
   - "ambiguous_GpC_CpG"
 duplicate_detection_distance_threshold: 0.07 # Hamming distance based similarity threshold to use for marking duplicate reads.
+duplicate_detection_demux_types_to_use: 
+  - "single"
+  - "double"
+  - "already"
 hamming_vs_metric_keys: # Metrics to plot the hamming distance against.
   - Fraction_C_site_modified
 duplicate_detection_keep_best_metric: "read_quality" # Obs metric to use to keep a representative read from a read duplicate cluster
@@ -151,6 +163,11 @@ duplicate_detection_do_pca: False # Whether to do PCA before hierarchical linkag
 
 # Position QC params
 position_max_nan_threshold: 0.1 # The maximum amount of nans to tolerate in a column
+mismatch_frequency_range:
+  - 0.01
+  - 0.99
+mismatch_frequency_layer: "mismatch_integer_encoding"
+mismatch_frequency_read_span_layer: "read_span_mask"
 
 ######## smftools spatial params #########
 invert_adata: False # Whether to invert the AnnData along the positions axis.
@@ -169,6 +186,9 @@ clustermap_cmap_gpc: "coolwarm"
 clustermap_cmap_cpg: "coolwarm"
 clustermap_cmap_a: "coolwarm"
 spatial_clustermap_sortby: "gpc"
+rolling_nn_site_types: 
+  - "GpC"
+  - "CpG"
 
 # Spatial Analysis - UMAP/Leiden params
 layer_for_umap_plotting: 'nan_half'
@@ -243,6 +263,18 @@ hmm_feature_sets:
       mid_accessible_patch: [20, 40]
       large_accessible_patch: [40, 110]
       nucleosome_depleted_region: [110, inf]
+hmm_feature_colormaps:
+  small_accessible_patch: "#A5D6A7"
+  mid_accessible_patch: "#2E7D32"
+  large_accessible_patch: "#006400"
+  nucleosome_depleted_region: "#00441B"
+  all_accessible_features: "#2E7D32"
+  small_bound_stretch: "#1E88E5"
+  medium_bound_stretch: "#6A1B9A"
+  large_bound_stretch: "#FB8C00"
+  putative_nucleosome: "#6D4C41"
+  all_footprint_features: "#6A1B9A"
+  cpg_patch: "#6D4C41"
 hmm_merge_layer_features:
   - ["all_accessible_features", 60]
 clustermap_cmap_hmm: "coolwarm"
@@ -259,6 +291,11 @@ hmm_clustermap_feature_layers:
   - medium_bound_stretch
   - putative_nucleosome
   - large_bound_stretch
+  - all_footprint_features
+hmm_clustermap_length_layers:
+  - all_accessible_features
+  - all_accessible_features_merged
+  - all_footprint_features
 hmm_clustermap_sortby: "hmm"
 hmm_peak_feature_configs:
   all_accessible_features: