smftools 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

smftools/cli/load_adata.py

@@ -7,7 +7,7 @@ from typing import Iterable, Union
 
 import numpy as np
 
-from smftools.constants import HMM_DIR, LOAD_DIR, LOGGING_DIR, PREPROCESS_DIR, SPATIAL_DIR
+from smftools.constants import LOAD_DIR, LOGGING_DIR
 from smftools.logging_utils import get_logger, setup_logging
 
 from .helpers import AdataPaths
@@ -105,7 +105,7 @@ def load_adata(config_path: str):
     from datetime import datetime
     from importlib import resources
 
-    from ..readwrite import add_or_update_column_in_csv, make_dirs
+    from ..readwrite import make_dirs
     from .helpers import get_adata_paths, load_experiment_config
 
     # -----------------------------

smftools/cli/preprocess_adata.py

@@ -6,7 +6,13 @@ from typing import Optional, Tuple
 
 import anndata as ad
 
-from smftools.constants import LOGGING_DIR, PREPROCESS_DIR
+from smftools.constants import (
+    BASE_QUALITY_SCORES,
+    DEMUX_TYPE,
+    LOGGING_DIR,
+    PREPROCESS_DIR,
+    READ_SPAN_MASK,
+)
 from smftools.logging_utils import get_logger, setup_logging
 
 logger = get_logger(__name__)
@@ -175,10 +181,6 @@ def preprocess_adata_core(
     - `pp_adata_path` and `pp_dup_rem_adata_path` are the target output paths for
       preprocessed and preprocessed+deduplicated AnnData.
 
-    Does NOT:
-    - Decide which stage to load from (that's the wrapper's job).
-    - Decide whether to skip entirely; it always runs its steps, but individual
-      sub-steps may skip based on `cfg.bypass_*` or directory existence.
 
     Returns
     -------
@@ -198,12 +200,10 @@ def preprocess_adata_core(
     from ..plotting import (
         plot_read_qc_histograms,
         plot_read_span_quality_clustermaps,
-        plot_sequence_integer_encoding_clustermaps,
     )
     from ..preprocessing import (
         append_base_context,
         append_binary_layer_by_base_context,
-        append_mismatch_frequency_sites,
         binarize_adata,
         binarize_on_Youden,
         calculate_complexity_II,
@@ -476,6 +476,22 @@ def preprocess_adata_core(
         from_valid_sites_only=True,
     )
 
+    # -----------------------------
+    # Optional inversion along positions axis
+    # -----------------------------
+    if getattr(cfg, "invert_adata", False):
+        adata = invert_adata(adata)
+
+    # -----------------------------
+    # Optional reindexing by reference
+    # -----------------------------
+    reindex_references_adata(
+        adata,
+        reference_col=cfg.reference_column,
+        offsets=cfg.reindexing_offsets,
+        new_col=cfg.reindexed_var_suffix,
+    )
+
     ############### Duplicate detection for conversion/deamination SMF ###############
     if smf_modality != "direct":
         references = adata.obs[cfg.reference_column].cat.categories
@@ -511,7 +527,7 @@ def preprocess_adata_core(
             hierarchical_metric="euclidean",
             hierarchical_window=cfg.duplicate_detection_window_size_for_hamming_neighbors,
             demux_types=cfg.duplicate_detection_demux_types_to_use,
-            demux_col="demux_type",
+            demux_col=DEMUX_TYPE,
         )
 
         # Use the flagged duplicate read groups and perform complexity analysis
@@ -537,96 +553,19 @@ def preprocess_adata_core(
         adata_unique = adata
     ########################################################################################################################
 
-    # -----------------------------
-    # Optional inversion along positions axis
-    # -----------------------------
-    if getattr(cfg, "invert_adata", False):
-        adata = invert_adata(adata)
-
-    # -----------------------------
-    # Optional reindexing by reference
-    # -----------------------------
-    reindex_references_adata(
-        adata,
-        reference_col=cfg.reference_column,
-        offsets=cfg.reindexing_offsets,
-        new_col=cfg.reindexed_var_suffix,
-    )
-
-    ############################################### Append mismatch frequency per position ###############################################
-    append_mismatch_frequency_sites(
-        adata_unique,
-        ref_column=cfg.reference_column,
-        mismatch_layer=cfg.mismatch_frequency_layer,
-        read_span_layer=cfg.mismatch_frequency_read_span_layer,
-        mismatch_frequency_range=cfg.mismatch_frequency_range,
-        bypass=cfg.bypass_append_mismatch_frequency_sites,
-        force_redo=cfg.force_redo_append_mismatch_frequency_sites,
-    )
-
-    ############################################### Plot integer sequence encoding clustermaps ###############################################
-    if "sequence_integer_encoding" not in adata.layers:
-        logger.debug(
-            "sequence_integer_encoding layer not found; skipping integer encoding clustermaps."
-        )
-    else:
-        pp_seq_clustermap_dir = preprocess_directory / "06_sequence_integer_encoding_clustermaps"
-        if pp_seq_clustermap_dir.is_dir() and not cfg.force_redo_preprocessing:
-            logger.debug(
-                f"{pp_seq_clustermap_dir} already exists. Skipping sequence integer encoding clustermaps."
-            )
-        else:
-            make_dirs([pp_seq_clustermap_dir])
-            plot_sequence_integer_encoding_clustermaps(
-                adata,
-                sample_col=cfg.sample_name_col_for_plotting,
-                reference_col=cfg.reference_column,
-                demux_types=cfg.clustermap_demux_types_to_plot,
-                min_quality=None,
-                min_length=None,
-                min_mapped_length_to_reference_length_ratio=None,
-                sort_by="none",
-                max_unknown_fraction=0.5,
-                save_path=pp_seq_clustermap_dir,
-                show_position_axis=True,
-            )
-
-        pp_dedup_seq_clustermap_dir = (
-            preprocess_directory / "deduplicated" / "06_sequence_integer_encoding_clustermaps"
-        )
-        if pp_dedup_seq_clustermap_dir.is_dir() and not cfg.force_redo_preprocessing:
-            logger.debug(
-                f"{pp_dedup_seq_clustermap_dir} already exists. Skipping sequence integer encoding clustermaps."
-            )
-        else:
-            make_dirs([pp_dedup_seq_clustermap_dir])
-            plot_sequence_integer_encoding_clustermaps(
-                adata_unique,
-                sample_col=cfg.sample_name_col_for_plotting,
-                reference_col=cfg.reference_column,
-                demux_types=cfg.clustermap_demux_types_to_plot,
-                min_quality=None,
-                min_length=None,
-                min_mapped_length_to_reference_length_ratio=None,
-                sort_by="none",
-                max_unknown_fraction=0.5,
-                save_path=pp_dedup_seq_clustermap_dir,
-                show_position_axis=True,
-            )
-
     ############################################### Plot read span mask + base quality clustermaps ###############################################
     quality_layer = None
-    if "base_quality_scores" in adata.layers:
-        quality_layer = "base_quality_scores"
+    if BASE_QUALITY_SCORES in adata.layers:
+        quality_layer = BASE_QUALITY_SCORES
     elif "base_qualities" in adata.layers:
         quality_layer = "base_qualities"
 
-    if "read_span_mask" not in adata.layers or quality_layer is None:
+    if READ_SPAN_MASK not in adata.layers or quality_layer is None:
         logger.debug(
             "read_span_mask and base quality layers not found; skipping read span/base quality clustermaps."
         )
     else:
-        pp_span_quality_dir = preprocess_directory / "07_read_span_quality_clustermaps"
+        pp_span_quality_dir = preprocess_directory / "06_read_span_and_quality_clustermaps"
         if pp_span_quality_dir.is_dir() and not cfg.force_redo_preprocessing:
             logger.debug(
                 f"{pp_span_quality_dir} already exists. Skipping read span/base quality clustermaps."
@@ -638,7 +577,7 @@ def preprocess_adata_core(
                 sample_col=cfg.sample_name_col_for_plotting,
                 reference_col=cfg.reference_column,
                 quality_layer=quality_layer,
-                read_span_layer="read_span_mask",
+                read_span_layer=READ_SPAN_MASK,
                 demux_types=cfg.clustermap_demux_types_to_plot,
                 save_path=pp_span_quality_dir,
                 show_position_axis=True,
@@ -646,20 +585,20 @@ def preprocess_adata_core(
             )
 
         pp_dedup_span_quality_dir = (
-            preprocess_directory / "deduplicated" / "07_read_span_quality_clustermaps"
+            preprocess_directory / "deduplicated" / "06_read_span_and_quality_clustermaps"
         )
         if pp_dedup_span_quality_dir.is_dir() and not cfg.force_redo_preprocessing:
             logger.debug(
                 f"{pp_dedup_span_quality_dir} already exists. Skipping read span/base quality clustermaps."
             )
-        elif quality_layer in adata_unique.layers and "read_span_mask" in adata_unique.layers:
+        elif quality_layer in adata_unique.layers and READ_SPAN_MASK in adata_unique.layers:
             make_dirs([pp_dedup_span_quality_dir])
             plot_read_span_quality_clustermaps(
                 adata_unique,
                 sample_col=cfg.sample_name_col_for_plotting,
                 reference_col=cfg.reference_column,
                 quality_layer=quality_layer,
-                read_span_layer="read_span_mask",
+                read_span_layer=READ_SPAN_MASK,
                 demux_types=cfg.clustermap_demux_types_to_plot,
                 save_path=pp_dedup_span_quality_dir,
                 show_position_axis=True,

smftools/cli/recipes.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional, Tuple
+
+import anndata as ad
+
+from ..cli.chimeric_adata import chimeric_adata
+from ..cli.hmm_adata import hmm_adata
+from ..cli.latent_adata import latent_adata
+from ..cli.load_adata import load_adata
+from ..cli.preprocess_adata import preprocess_adata
+from ..cli.spatial_adata import spatial_adata
+from ..cli.variant_adata import variant_adata
+
+
+def full_flow(
+    config_path: str,
+) -> Tuple[Optional[ad.AnnData], Optional[Path]]:
+    load_adata(config_path)
+    preprocess_adata(config_path)
+    spatial_adata(config_path)
+    variant_adata(config_path)
+    chimeric_adata(config_path)
+    hmm_adata(config_path)
+    latent_adata(config_path)

smftools/cli/spatial_adata.py

@@ -6,9 +6,8 @@ from typing import Optional, Tuple
 
 import anndata as ad
 
-from smftools.constants import LOGGING_DIR, SEQUENCE_INTEGER_ENCODING, SPATIAL_DIR
+from smftools.constants import LOGGING_DIR, SPATIAL_DIR
 from smftools.logging_utils import get_logger, setup_logging
-from smftools.optional_imports import require
 
 logger = get_logger(__name__)
 
@@ -36,7 +35,7 @@ def spatial_adata(
     spatial_adata_path : Path | None
         Path to the “current” spatial AnnData (or hmm AnnData if we skip to that).
     """
-    from ..readwrite import add_or_update_column_in_csv, safe_read_h5ad
+    from ..readwrite import safe_read_h5ad
     from .helpers import get_adata_paths, load_experiment_config
 
     # 1) Ensure config + basic paths via load_adata
@@ -47,7 +46,10 @@ def spatial_adata(
     pp_path = paths.pp
     pp_dedup_path = paths.pp_dedup
     spatial_path = paths.spatial
+    chimeric_path = paths.chimeric
+    variant_path = paths.variant
     hmm_path = paths.hmm
+    latent_path = paths.latent
 
     # Stage-skipping logic for spatial
     if not getattr(cfg, "force_redo_spatial_analyses", False):
@@ -65,9 +67,18 @@ def spatial_adata(
     if hmm_path.exists():
         start_adata = _load(hmm_path)
         source_path = hmm_path
+    elif latent_path.exists():
+        start_adata = _load(latent_path)
+        source_path = latent_path
     elif spatial_path.exists():
         start_adata = _load(spatial_path)
         source_path = spatial_path
+    elif chimeric_path.exists():
+        start_adata = _load(chimeric_path)
+        source_path = chimeric_path
+    elif variant_path.exists():
+        start_adata = _load(variant_path)
+        source_path = variant_path
     elif pp_dedup_path.exists():
         start_adata = _load(pp_dedup_path)
         source_path = pp_dedup_path
@@ -140,7 +151,6 @@ def spatial_adata_core(
     from ..plotting import (
         combined_raw_clustermap,
         plot_rolling_grid,
-        plot_rolling_nn_and_layer,
         plot_spatial_autocorr_grid,
     )
     from ..preprocessing import (
@@ -149,12 +159,10 @@ def spatial_adata_core(
         reindex_references_adata,
     )
     from ..readwrite import make_dirs, safe_read_h5ad
-    from ..tools import rolling_window_nn_distance
     from ..tools.position_stats import (
         compute_positionwise_statistics,
         plot_positionwise_matrices,
     )
-    from ..tools.rolling_nn_distance import assign_rolling_nn_results
     from ..tools.spatial_autocorrelation import (
         analyze_autocorr_matrix,
         binary_autocorrelation_with_spacing,
@@ -227,13 +235,13 @@ def spatial_adata_core(
     references = adata.obs[cfg.reference_column].cat.categories
 
     # ============================================================
-    # 1) Clustermaps (non-direct modalities) on *preprocessed* data
+    # 1) Clustermaps (non-direct modalities) on preprocessed adata
     # ============================================================
     if smf_modality != "direct":
         preprocessed_version_available = pp_adata_path.exists()
 
         if preprocessed_version_available:
-            pp_clustermap_dir = spatial_directory / "06_clustermaps"
+            pp_clustermap_dir = spatial_directory / "01_clustermaps"
 
             if pp_clustermap_dir.is_dir() and not getattr(
                 cfg, "force_redo_spatial_analyses", False
@@ -292,7 +300,7 @@ def spatial_adata_core(
                     min_mapped_length_to_reference_length_ratio=cfg.read_len_to_ref_ratio_filter_thresholds[
                         0
                     ],
-                    min_position_valid_fraction=cfg.min_valid_fraction_positions_in_read_vs_ref,
+                    min_position_valid_fraction=1 - cfg.position_max_nan_threshold,
                     demux_types=cfg.clustermap_demux_types_to_plot,
                     bins=None,
                     sample_mapping=None,
@@ -303,10 +311,10 @@ def spatial_adata_core(
                 )
 
     # ============================================================
-    # 2) Clustermaps on *deduplicated* preprocessed AnnData
+    # 2) Clustermaps on deduplicated preprocessed AnnDatas
     # ============================================================
     spatial_dir_dedup = spatial_directory / "deduplicated"
-    clustermap_dir_dedup = spatial_dir_dedup / "06_clustermaps"
+    clustermap_dir_dedup = spatial_dir_dedup / "01_clustermaps"
 
     # Clustermaps on deduplicated adata
     if clustermap_dir_dedup.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
@@ -343,104 +351,10 @@ def spatial_adata_core(
             index_col_suffix=reindex_suffix,
         )
 
-    # ============================================================
-    # 2b) Rolling NN distances + layer clustermaps
-    # ============================================================
-    pp_rolling_nn_dir = spatial_dir_dedup / "06b_rolling_nn_clustermaps"
-
-    if pp_rolling_nn_dir.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
-        logger.debug(f"{pp_rolling_nn_dir} already exists. Skipping rolling NN distance plots.")
-    else:
-        make_dirs([pp_rolling_nn_dir])
-        samples = (
-            adata.obs[cfg.sample_name_col_for_plotting].astype("category").cat.categories.tolist()
-        )
-        references = adata.obs[cfg.reference_column].astype("category").cat.categories.tolist()
-
-        for reference in references:
-            for sample in samples:
-                mask = (adata.obs[cfg.sample_name_col_for_plotting] == sample) & (
-                    adata.obs[cfg.reference_column] == reference
-                )
-                if not mask.any():
-                    continue
-
-                subset = adata[mask]
-                site_mask = (
-                    adata.var[[f"{reference}_{st}_site" for st in cfg.rolling_nn_site_types]]
-                    .fillna(False)
-                    .any(axis=1)
-                )
-                subset = subset[:, site_mask].copy()
-                try:
-                    rolling_values, rolling_starts = rolling_window_nn_distance(
-                        subset,
-                        layer=cfg.rolling_nn_layer,
-                        window=cfg.rolling_nn_window,
-                        step=cfg.rolling_nn_step,
-                        min_overlap=cfg.rolling_nn_min_overlap,
-                        return_fraction=cfg.rolling_nn_return_fraction,
-                        store_obsm=cfg.rolling_nn_obsm_key,
-                    )
-                except Exception as exc:
-                    logger.warning(
-                        "Rolling NN distance computation failed for sample=%s ref=%s: %s",
-                        sample,
-                        reference,
-                        exc,
-                    )
-                    continue
-
-                safe_sample = str(sample).replace(os.sep, "_")
-                safe_ref = str(reference).replace(os.sep, "_")
-                parent_obsm_key = f"{cfg.rolling_nn_obsm_key}__{safe_ref}"
-                try:
-                    assign_rolling_nn_results(
-                        adata,
-                        subset,
-                        rolling_values,
-                        rolling_starts,
-                        obsm_key=parent_obsm_key,
-                        window=cfg.rolling_nn_window,
-                        step=cfg.rolling_nn_step,
-                        min_overlap=cfg.rolling_nn_min_overlap,
-                        return_fraction=cfg.rolling_nn_return_fraction,
-                        layer=cfg.rolling_nn_layer,
-                    )
-                except Exception as exc:
-                    logger.warning(
-                        "Failed to merge rolling NN results for sample=%s ref=%s: %s",
-                        sample,
-                        reference,
-                        exc,
-                    )
-                adata.uns.setdefault(f"{cfg.rolling_nn_obsm_key}_reference_map", {})[reference] = (
-                    parent_obsm_key
-                )
-                out_png = pp_rolling_nn_dir / f"{safe_sample}__{safe_ref}.png"
-                title = f"{sample} {reference}"
-                try:
-                    plot_rolling_nn_and_layer(
-                        subset,
-                        obsm_key=cfg.rolling_nn_obsm_key,
-                        layer_key=cfg.rolling_nn_plot_layer,
-                        max_nan_fraction=cfg.position_max_nan_threshold,
-                        var_valid_fraction_col=f"{reference}_valid_fraction",
-                        title=title,
-                        save_name=out_png,
-                    )
-                except Exception as exc:
-                    logger.warning(
-                        "Failed rolling NN plot for sample=%s ref=%s: %s",
-                        sample,
-                        reference,
-                        exc,
-                    )
-
     # ============================================================
     # 3) Spatial autocorrelation + rolling metrics
     # ============================================================
-    pp_autocorr_dir = spatial_dir_dedup / "08_autocorrelations"
+    pp_autocorr_dir = spatial_dir_dedup / "02_autocorrelations"
 
     if pp_autocorr_dir.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
         logger.debug(f"{pp_autocorr_dir} already exists. Skipping autocorrelation plotting.")
@@ -783,7 +697,7 @@ def spatial_adata_core(
     # ============================================================
     # 4) Pearson / correlation matrices
     # ============================================================
-    corr_dir = spatial_dir_dedup / "09_correlation_matrices"
+    corr_dir = spatial_dir_dedup / "03_correlation_matrices"
 
     if corr_dir.is_dir() and not getattr(cfg, "force_redo_spatial_analyses", False):
         logger.debug(f"{corr_dir} already exists. Skipping correlation matrix plotting.")
@@ -816,10 +730,10 @@ def spatial_adata_core(
         )
 
     # ============================================================
-    # 5) Save spatial AnnData
+    # 4) Save spatial AnnData
     # ============================================================
     if (not spatial_adata_path.exists()) or getattr(cfg, "force_redo_spatial_analyses", False):
-        logger.info("Saving spatial analyzed AnnData (post preprocessing and duplicate removal).")
+        logger.info("Saving spatial analyzed AnnData.")
         record_smftools_metadata(
             adata,
             step_name="spatial",