smftools 0.2.3__py3-none-any.whl → 0.2.4__py3-none-any.whl

smftools/_version.py

@@ -1 +1 @@
-__version__ = "0.2.3"
+__version__ = "0.2.4"

smftools/cli/helpers.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+from pathlib import Path
+import anndata as ad
+from ..readwrite import safe_write_h5ad
+
+@dataclass
+class AdataPaths:
+    raw: Path
+    pp: Path
+    pp_dedup: Path
+    spatial: Path
+    hmm: Path
+
+
+def get_adata_paths(cfg) -> AdataPaths:
+    """
+    Central helper: given cfg, compute all standard AnnData paths.
+    """
+    h5_dir = Path(cfg.output_directory) / "h5ads"
+
+    raw = h5_dir / f"{cfg.experiment_name}.h5ad.gz"
+
+    pp = h5_dir / f"{cfg.experiment_name}_preprocessed.h5ad.gz"
+
+    if cfg.smf_modality == "direct":
+        # direct SMF: duplicate-removed path is just preprocessed path
+        pp_dedup = pp
+    else:
+        pp_dedup = h5_dir / f"{cfg.experiment_name}_preprocessed_duplicates_removed.h5ad.gz"
+
+    pp_dedup_base = pp_dedup.name.removesuffix(".h5ad.gz")
+
+    spatial = h5_dir / f"{pp_dedup_base}_spatial.h5ad.gz"
+    hmm = h5_dir / f"{pp_dedup_base}_spatial_hmm.h5ad.gz"
+
+    return AdataPaths(
+        raw=raw,
+        pp=pp,
+        pp_dedup=pp_dedup,
+        spatial=spatial,
+        hmm=hmm,
+    )
+
+def write_gz_h5ad(adata: ad.AnnData, path: Path) -> Path:
+    if path.suffix != ".gz":
+        path = path.with_name(path.name + ".gz")
+    safe_write_h5ad(adata, path, compression="gzip", backup=True)
+    return path

smftools/cli/hmm_adata.py

@@ -63,9 +63,11 @@ def hmm_adata(config_path):
         preprocessed_dedup_spatial_version_available = spatial_adata_path.exists()
         preprocessed_dedup_spatial_hmm_version_available = hmm_adata_path.exists()
 
-        if cfg.force_redo_hmm_fit:
-            print(f"Forcing redo of basic analysis workflow, starting from the preprocessed adata if available. Otherwise, will use the raw adata.")
-            if preprocessed_dedup_spatial_version_available:
+        if cfg.force_redo_hmm_fit or cfg.force_redo_hmm_apply:
+            print(f"Forcing redo of hmm analysis workflow.")
+            if preprocessed_dedup_spatial_hmm_version_available:
+                adata, load_report = safe_read_h5ad(hmm_adata_path)
+            elif preprocessed_dedup_spatial_version_available:
                 adata, load_report = safe_read_h5ad(spatial_adata_path)
             elif preprocessed_dup_removed_version_available:
                 adata, load_report = safe_read_h5ad(pp_dup_rem_adata_path)
@@ -74,7 +76,7 @@ def hmm_adata(config_path):
             else:
                 print(f"Can not redo duplicate detection when there is no compatible adata available: either raw or preprocessed are required")
         elif preprocessed_dedup_spatial_hmm_version_available:
-            return (None, hmm_adata_path)
+            adata, load_report = safe_read_h5ad(hmm_adata_path)
         else:
             if preprocessed_dedup_spatial_version_available:
                 adata, load_report = safe_read_h5ad(spatial_adata_path)
@@ -110,96 +112,126 @@ def hmm_adata(config_path):
         if adata.uns.get(uns_key) is None:
             adata.uns[uns_key] = []
 
-        for sample in samples:
-            for ref in references:
-                mask = (adata.obs[cfg.sample_name_col_for_plotting] == sample) & (adata.obs[cfg.reference_column] == ref)
-                subset = adata[mask].copy()
-                if subset.shape[0] < 1:
-                    continue
-
-                for mod_site in cfg.hmm_methbases:
-                    mod_label = {'C': 'C'}.get(mod_site, mod_site)
-                    hmm_path = hmm_dir / f"{sample}_{ref}_{mod_label}_hmm_model.pth"
-
-                    # ensure the input obsm exists
-                    obsm_key = f'{ref}_{mod_label}_site'
-                    if obsm_key not in subset.obsm:
-                        print(f"Skipping {sample} {ref} {mod_label}: missing obsm '{obsm_key}'")
+        if adata.uns.get('hmm_annotated', False) and not cfg.force_redo_hmm_fit and not cfg.force_redo_hmm_apply:
+            pass
+        else:
+            for sample in samples:
+                for ref in references:
+                    mask = (adata.obs[cfg.sample_name_col_for_plotting] == sample) & (adata.obs[cfg.reference_column] == ref)
+                    subset = adata[mask].copy()
+                    if subset.shape[0] < 1:
                         continue
 
-                    # Fit or load model
-                    if os.path.exists(hmm_path) and not cfg.force_redo_hmm_fit:
-                        hmm = HMM.load(hmm_path)
-                        hmm.print_params()
-                    else:
-                        print(f"Fitting HMM for {sample} {ref} {mod_label}")
-                        hmm = HMM.from_config(cfg)
-                        # fit expects a list-of-seqs or 2D ndarray in the obsm
-                        seqs = subset.obsm[obsm_key]
-                        hmm.fit(seqs)
-                        hmm.print_params()
-                        hmm.save(hmm_path)
-
-                    # Apply / annotate on the subset, then copy layers back to final_adata
-                    if (not cfg.bypass_hmm_apply) or cfg.force_redo_hmm_apply:
-                        print(f"Applying HMM on subset for {sample} {ref} {mod_label}")
-                        # Use the new uns_key argument so subset will record appended layer names
-                        # (annotate_adata modifies subset.obs/layers in-place and should write subset.uns[uns_key])
-                        hmm.annotate_adata(subset,
-                                        obs_column=cfg.reference_column,
-                                        layer=cfg.layer_for_umap_plotting,
-                                        config=cfg)
-                        
-                        #to_merge = [("C_all_accessible_features", 80)]
-                        to_merge = cfg.hmm_merge_layer_features
-                        for layer_to_merge, merge_distance in to_merge:
-                            if layer_to_merge:
-                                hmm.merge_intervals_in_layer(subset,
-                                                            layer=layer_to_merge,
-                                                            distance_threshold=merge_distance,
-                                                            overwrite=True
-                                                            )
-                            else:
-                                pass
+                    for mod_site in cfg.hmm_methbases:
+                        mod_label = {'C': 'C'}.get(mod_site, mod_site)
+                        hmm_path = hmm_dir / f"{sample}_{ref}_{mod_label}_hmm_model.pth"
 
-                        # collect appended layers from subset.uns
-                        appended = list(subset.uns.get(uns_key, []))
-                        print(appended)
-                        if len(appended) == 0:
-                            # nothing appended for this subset; continue
+                        # ensure the input obsm exists
+                        obsm_key = f'{ref}_{mod_label}_site'
+                        if obsm_key not in subset.obsm:
+                            print(f"Skipping {sample} {ref} {mod_label}: missing obsm '{obsm_key}'")
                             continue
 
-                        # copy each appended layer into adata
-                        subset_mask_bool = mask.values if hasattr(mask, "values") else np.asarray(mask)
-                        for layer_name in appended:
-                            if layer_name not in subset.layers:
-                                # defensive: skip
-                                warnings.warn(f"Expected layer {layer_name} in subset but not found; skipping copy.")
-                                continue
-                            sub_layer = subset.layers[layer_name]
-                            # ensure final layer exists and assign rows
-                            try:
-                               hmm._ensure_final_layer_and_assign(adata, layer_name, subset_mask_bool, sub_layer)
-                            except Exception as e:
-                                warnings.warn(f"Failed to copy layer {layer_name} into adata: {e}", stacklevel=2)
-                                # fallback: if dense and small, try to coerce
-                                if issparse(sub_layer):
-                                    arr = sub_layer.toarray()
-                                else:
-                                    arr = np.asarray(sub_layer)
-                                adata.layers[layer_name] = adata.layers.get(layer_name, np.zeros((adata.shape[0], arr.shape[1]), dtype=arr.dtype))
-                                final_idx = np.nonzero(subset_mask_bool)[0]
-                                adata.layers[layer_name][final_idx, :] = arr
-
-                        # merge appended layer names into adata.uns
-                        existing = list(adata.uns.get(uns_key, []))
-                        for ln in appended:
-                            if ln not in existing:
-                                existing.append(ln)
-                        adata.uns[uns_key] = existing
+                        # Fit or load model
+                        if hmm_path.exists() and not cfg.force_redo_hmm_fit:
+                            hmm = HMM.load(hmm_path)
+                            hmm.print_params()
+                        else:
+                            print(f"Fitting HMM for {sample} {ref} {mod_label}")
+                            hmm = HMM.from_config(cfg)
+                            # fit expects a list-of-seqs or 2D ndarray in the obsm
+                            seqs = subset.obsm[obsm_key]
+                            hmm.fit(seqs)
+                            hmm.print_params()
+                            hmm.save(hmm_path)
+
+                        # Apply / annotate on the subset, then copy layers back to final_adata
+                        if cfg.bypass_hmm_apply:
+                            pass
+                        else:
+                            print(f"Applying HMM on subset for {sample} {ref} {mod_label}")
+                            # Use the new uns_key argument so subset will record appended layer names
+                            # (annotate_adata modifies subset.obs/layers in-place and should write subset.uns[uns_key])
+                            if smf_modality == "direct":
+                                hmm_layer = cfg.output_binary_layer_name
+                            else:
+                                hmm_layer = None
+
+                            hmm.annotate_adata(subset,
+                                            obs_column=cfg.reference_column,
+                                            layer=hmm_layer,
+                                            config=cfg,
+                                            force_redo=cfg.force_redo_hmm_apply
+                                            )
+                            
+                            if adata.uns.get('hmm_annotated', False) and not cfg.force_redo_hmm_apply:
+                                pass
+                            else:
+                                to_merge = cfg.hmm_merge_layer_features
+                                for layer_to_merge, merge_distance in to_merge:
+                                    if layer_to_merge:
+                                        hmm.merge_intervals_in_layer(subset,
+                                                                    layer=layer_to_merge,
+                                                                    distance_threshold=merge_distance,
+                                                                    overwrite=True
+                                                                    )
+                                    else:
+                                        pass
+
+                                # collect appended layers from subset.uns
+                                appended = list(subset.uns.get(uns_key, []))
+                                print(appended)
+                                if len(appended) == 0:
+                                    # nothing appended for this subset; continue
+                                    continue
+
+                                # copy each appended layer into adata
+                                subset_mask_bool = mask.values if hasattr(mask, "values") else np.asarray(mask)
+                                for layer_name in appended:
+                                    if layer_name not in subset.layers:
+                                        # defensive: skip
+                                        warnings.warn(f"Expected layer {layer_name} in subset but not found; skipping copy.")
+                                        continue
+                                    sub_layer = subset.layers[layer_name]
+                                    # ensure final layer exists and assign rows
+                                    try:
+                                        hmm._ensure_final_layer_and_assign(adata, layer_name, subset_mask_bool, sub_layer)
+                                    except Exception as e:
+                                        warnings.warn(f"Failed to copy layer {layer_name} into adata: {e}", stacklevel=2)
+                                        # fallback: if dense and small, try to coerce
+                                        if issparse(sub_layer):
+                                            arr = sub_layer.toarray()
+                                        else:
+                                            arr = np.asarray(sub_layer)
+                                        adata.layers[layer_name] = adata.layers.get(layer_name, np.zeros((adata.shape[0], arr.shape[1]), dtype=arr.dtype))
+                                        final_idx = np.nonzero(subset_mask_bool)[0]
+                                        adata.layers[layer_name][final_idx, :] = arr
+
+                                # merge appended layer names into adata.uns
+                                existing = list(adata.uns.get(uns_key, []))
+                                for ln in appended:
+                                    if ln not in existing:
+                                        existing.append(ln)
+                                adata.uns[uns_key] = existing
 
     else:
         pass
+
+    from ..hmm import call_hmm_peaks
+    hmm_dir = pp_dir / "11_hmm_peak_calling"
+    if hmm_dir.is_dir():
+        pass
+    else:
+        make_dirs([pp_dir, hmm_dir])
+
+        call_hmm_peaks(
+                adata,
+                feature_configs=cfg.hmm_peak_feature_configs,
+                ref_column=cfg.reference_column,
+                site_types=cfg.mod_target_bases,
+                save_plot=True,
+                output_dir=hmm_dir,
+                index_col_suffix=cfg.reindexed_var_suffix)
     
     ## Save HMM annotated adata
     if not hmm_adata_path.exists():
@@ -215,85 +247,69 @@ def hmm_adata(config_path):
     ########################################################################################################################
 
 ############################################### HMM based feature plotting ###############################################
-    
-    hmm_dir = pp_dir / "11_hmm_clustermaps"
+    from ..plotting import combined_hmm_raw_clustermap
+    hmm_dir = pp_dir / "12_hmm_clustermaps"
+    make_dirs([pp_dir, hmm_dir])
 
-    if hmm_dir.is_dir():
-        print(f'{hmm_dir} already exists.')
-    else:
-        make_dirs([pp_dir, hmm_dir])
-        from ..plotting import combined_hmm_raw_clustermap
-        feature_layers = [
-            "all_accessible_features",
-            "large_accessible_patch",
-            "small_bound_stretch",
-            "medium_bound_stretch",
-            "putative_nucleosome",
-            "all_accessible_features_merged",
-        ]
-
-        layers: list[str] = []
-
-        if any(base in ["C", "CpG", "GpC"] for base in cfg.mod_target_bases):
-            if smf_modality == 'deaminase':
-                layers.extend([f"C_{layer}" for layer in feature_layers])
-            elif smf_modality == 'conversion':
-                layers.extend([f"GpC_{layer}" for layer in feature_layers])
-
-        if 'A' in cfg.mod_target_bases:
-            layers.extend([f"A_{layer}" for layer in feature_layers])
-
-        if not layers:
-            raise ValueError(
-                f"No HMM feature layers matched mod_target_bases={cfg.mod_target_bases} "
-                f"and smf_modality={smf_modality}"
-            )
-        
-        if smf_modality == 'direct':
-            sort_by = "any_a"
-        else:
-            sort_by = 'gpc'
+    layers: list[str] = []
 
-        for layer in layers:
-            save_path = hmm_dir / layer
-            make_dirs([save_path])
+    for base in cfg.hmm_methbases:
+        layers.extend([f"{base}_{layer}" for layer in cfg.hmm_clustermap_feature_layers])
+
+    if cfg.cpg:
+        layers.extend(["CpG_cpg_patch"])
+
+    if not layers:
+        raise ValueError(
+            f"No HMM feature layers matched mod_target_bases={cfg.mod_target_bases} "
+            f"and smf_modality={smf_modality}"
+        )
+
+    for layer in layers:
+        hmm_cluster_save_dir = hmm_dir / layer
+        if hmm_cluster_save_dir.is_dir():
+            pass
+        else:
+            make_dirs([hmm_cluster_save_dir])
 
             combined_hmm_raw_clustermap(
             adata,
             sample_col=cfg.sample_name_col_for_plotting,
             reference_col=cfg.reference_column,
             hmm_feature_layer=layer,
-            layer_gpc="nan0_0minus1",
-            layer_cpg="nan0_0minus1",
-            layer_any_c="nan0_0minus1",
-            layer_a= "nan0_0minus1",
-            cmap_hmm="coolwarm",
-            cmap_gpc="coolwarm",
-            cmap_cpg="viridis",
-            cmap_any_c='coolwarm',
-            cmap_a= "coolwarm",
+            layer_gpc=cfg.layer_for_clustermap_plotting,
+            layer_cpg=cfg.layer_for_clustermap_plotting,
+            layer_c=cfg.layer_for_clustermap_plotting,
+            layer_a=cfg.layer_for_clustermap_plotting,
+            cmap_hmm=cfg.clustermap_cmap_hmm,
+            cmap_gpc=cfg.clustermap_cmap_gpc,
+            cmap_cpg=cfg.clustermap_cmap_cpg,
+            cmap_c=cfg.clustermap_cmap_c,
+            cmap_a=cfg.clustermap_cmap_a,
             min_quality=cfg.read_quality_filter_thresholds[0],
             min_length=cfg.read_len_filter_thresholds[0],
             min_mapped_length_to_reference_length_ratio=cfg.read_len_to_ref_ratio_filter_thresholds[0],
             min_position_valid_fraction=1-cfg.position_max_nan_threshold,
-            save_path=save_path,
+            save_path=hmm_cluster_save_dir,
             normalize_hmm=False,
-            sort_by=sort_by,  # options: 'gpc', 'cpg', 'gpc_cpg', 'none', or 'obs:<column>'
+            sort_by=cfg.hmm_clustermap_sortby,  # options: 'gpc', 'cpg', 'gpc_cpg', 'none', or 'obs:<column>'
             bins=None,
             deaminase=deaminase,
-            min_signal=0
+            min_signal=0,
+            index_col_suffix=cfg.reindexed_var_suffix
             )
 
-    hmm_dir = pp_dir / "12_hmm_bulk_traces"
+    hmm_dir = pp_dir / "13_hmm_bulk_traces"
 
     if hmm_dir.is_dir():
         print(f'{hmm_dir} already exists.')
     else:
         make_dirs([pp_dir, hmm_dir])
         from ..plotting import plot_hmm_layers_rolling_by_sample_ref
+        bulk_hmm_layers = [layer for layer in adata.uns['hmm_appended_layers'] if "_lengths" not in layer]
         saved = plot_hmm_layers_rolling_by_sample_ref(
             adata,
-            layers=adata.uns['hmm_appended_layers'],
+            layers=bulk_hmm_layers,
             sample_col=cfg.sample_name_col_for_plotting,
             ref_col=cfg.reference_column,
             window=101,
@@ -304,7 +320,7 @@ def hmm_adata(config_path):
             show_raw=False
         )
 
-    hmm_dir = pp_dir / "13_hmm_fragment_distributions"
+    hmm_dir = pp_dir / "14_hmm_fragment_distributions"
 
     if hmm_dir.is_dir():
         print(f'{hmm_dir} already exists.')
@@ -312,7 +328,14 @@ def hmm_adata(config_path):
         make_dirs([pp_dir, hmm_dir])
         from ..plotting import plot_hmm_size_contours
 
-        for layer, max in [('C_all_accessible_features_lengths', 400), ('C_all_footprint_features_lengths', 160), ('C_all_accessible_features_merged_lengths', 800)]:
+        if smf_modality == 'deaminase':
+            fragments = [('C_all_accessible_features_lengths', 400), ('C_all_footprint_features_lengths', 250), ('C_all_accessible_features_merged_lengths', 800)]
+        elif smf_modality == 'conversion':
+            fragments = [('GpC_all_accessible_features_lengths', 400), ('GpC_all_footprint_features_lengths', 250), ('GpC_all_accessible_features_merged_lengths', 800)]
+        elif smf_modality == "direct":
+            fragments = [('A_all_accessible_features_lengths', 400), ('A_all_footprint_features_lengths', 200), ('A_all_accessible_features_merged_lengths', 800)]
+
+        for layer, max in fragments:
             save_path = hmm_dir / layer
             make_dirs([save_path])
 
@@ -328,9 +351,9 @@ def hmm_adata(config_path):
                 save_pdf=False,
                 save_each_page=True,
                 dpi=200,
-                smoothing_sigma=None,
-                normalize_after_smoothing=False,
-                cmap='viridis', 
+                smoothing_sigma=(10, 10),
+                normalize_after_smoothing=True,
+                cmap='Greens', 
                 log_scale_z=True
             )
     ########################################################################################################################