smftools 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

smftools/config/default.yaml

@@ -18,8 +18,9 @@ conversions:
 fastq_barcode_map: null # For FASTQ files, an optional map of file paths to barcodes can be provided. Default is autodetecting barcodes.
 fastq_auto_pairing: True # For FASTQ files, attempt to find read pair files automatically.
 input_already_demuxed: False # If the input files are already demultiplexed.
+
 delete_intermediate_hdfs: True # Whether to delete the intermediate hdfs from the conversion/deamination workflows.
-delete_intermediate_bams: True # Whether to delete intermediate BAM files.
+delete_intermediate_bams: False # Whether to delete intermediate BAM files.
 delete_intermediate_tsvs: True # Whether to delete intermediate TSV files.
 
 # Sequencing modality and general experiment params
@@ -77,6 +78,7 @@ aligner_args:
 # Sorted BAM and BED specific handling
 make_bigwigs: False # Whether to make coverage bigwigs
 make_beds: False # Whether to make beds from the aligned bams
+annotate_secondary_supplementary: True # Whether to annotate reads with secondary/supplementary alignments from the aligned BAM
 samtools_backend: auto # auto|python|cli for samtools-compatible operations
 bedtools_backend: auto # auto|python|cli for bedtools-compatible operations
 bigwig_backend: auto # auto|python|cli for bedGraphToBigWig conversion
@@ -90,6 +92,12 @@ mapping_threshold: 0.10 # Minimum proportion of mapped reads that need to fall w
 reference_column: 'Reference_strand'
 sample_column: 'Experiment_name_and_barcode'
 
+# Plotting params
+clustermap_demux_types_to_plot: 
+  - "single"
+  - "double"
+  - "already"
+
 ######## smftools preprocess params #########
 # Read length, quality, and mapping filtering params
 read_coord_filter:
@@ -102,7 +110,7 @@ read_len_to_ref_ratio_filter_thresholds:
   - null
   - null
 read_quality_filter_thresholds:
-  - 15
+  - 10
   - null
 read_mapping_quality_filter_thresholds:
   - null
@@ -122,7 +130,7 @@ read_mod_filtering_a_thresholds:
   - 0.025
   - 0.975
 read_mod_filtering_use_other_c_as_background: False
-min_valid_fraction_positions_in_read_vs_ref: 0.5
+min_valid_fraction_positions_in_read_vs_ref: 0.2
 
 # Plotting params for read length histograms
 obs_to_plot_pp_qc:
@@ -140,6 +148,10 @@ duplicate_detection_site_types: # Site types to consider for duplicate detection
   - "CpG"
   - "ambiguous_GpC_CpG"
 duplicate_detection_distance_threshold: 0.07 # Hamming distance based similarity threshold to use for marking duplicate reads.
+duplicate_detection_demux_types_to_use: 
+  - "single"
+  - "double"
+  - "already"
 hamming_vs_metric_keys: # Metrics to plot the hamming distance against.
   - Fraction_C_site_modified
 duplicate_detection_keep_best_metric: "read_quality" # Obs metric to use to keep a representative read from a read duplicate cluster
@@ -150,7 +162,13 @@ duplicate_detection_hierarchical_linkage: "average" # Method for hierarchical cl
 duplicate_detection_do_pca: False # Whether to do PCA before hierarchical linkage based duplicate detection.
 
 # Position QC params
-position_max_nan_threshold: 0.1 # The maximum amount of nans to tolerate in a column
+position_max_nan_threshold: 0.8 # The maximum amount of nans to tolerate in a column
+mismatch_frequency_range:
+  - 0.01
+  - 0.99
+mismatch_frequency_layer: "mismatch_integer_encoding"
+mismatch_frequency_read_span_layer: "read_span_mask"
+mismatch_base_frequency_exclude_mod_sites: True
 
 ######## smftools spatial params #########
 invert_adata: False # Whether to invert the AnnData along the positions axis.
@@ -170,9 +188,55 @@ clustermap_cmap_cpg: "coolwarm"
 clustermap_cmap_a: "coolwarm"
 spatial_clustermap_sortby: "gpc"
 
-# Spatial Analysis - UMAP/Leiden params
+# Clustermap variant params
+overlay_variant_calls: false
+variant_overlay_seq1_color: "black"
+variant_overlay_seq2_color: "white"
+variant_overlay_marker_size: 4.0
+
+# Spatial analysis - Rolling NN Hamming
+rolling_nn_layer: "nan0_0minus1"
+rolling_nn_plot_layer: "nan0_0minus1"
+rolling_nn_plot_layers:
+  - "nan0_0minus1"
+  - "zero_hamming_distance_spans"
+rolling_nn_window: 10
+rolling_nn_step: 1
+rolling_nn_min_overlap: 8
+rolling_nn_return_fraction: true
+rolling_nn_obsm_key: "rolling_nn_dist"
+rolling_nn_site_types: 
+  - "GpC"
+  - "CpG"
+rolling_nn_write_zero_pairs_csvs: true
+rolling_nn_zero_pairs_uns_key: null
+rolling_nn_zero_pairs_segments_key: null
+rolling_nn_zero_pairs_layer_key: null
+rolling_nn_zero_pairs_refine: true
+rolling_nn_zero_pairs_max_nan_run: 2
+rolling_nn_zero_pairs_merge_gap: 1
+rolling_nn_zero_pairs_max_segments_per_read: 2
+rolling_nn_zero_pairs_max_overlap: 5
+rolling_nn_zero_pairs_layer_overlap_mode: "sum"
+rolling_nn_zero_pairs_layer_overlap_value: null
+rolling_nn_zero_pairs_keep_uns: true
+rolling_nn_zero_pairs_segments_keep_uns: true
+rolling_nn_zero_pairs_top_segments_per_read: 3
+rolling_nn_zero_pairs_top_segments_max_overlap: 5
+rolling_nn_zero_pairs_top_segments_min_span: 300
+rolling_nn_zero_pairs_top_segments_write_csvs: true
+rolling_nn_zero_pairs_segment_histogram_bins: 30
+
+# Cross-sample rolling NN analysis
+cross_sample_analysis: true
+cross_sample_grouping_col: null
+cross_sample_random_seed: 42
+delta_hamming_chimeric_span_threshold: 200
+
+# Latent Analysis - UMAP/Leiden params
 layer_for_umap_plotting: 'nan_half'
 umap_layers_to_plot:
+  - "leiden"
   - "mapped_length"
   - "Raw_modification_signal"
 
@@ -243,22 +307,31 @@ hmm_feature_sets:
       mid_accessible_patch: [20, 40]
       large_accessible_patch: [40, 110]
       nucleosome_depleted_region: [110, inf]
+hmm_feature_colormaps:
+  small_accessible_patch: "#A5D6A7"
+  mid_accessible_patch: "#2E7D32"
+  large_accessible_patch: "#006400"
+  nucleosome_depleted_region: "#00441B"
+  all_accessible_features: "#2E7D32"
+  small_bound_stretch: "#1E88E5"
+  medium_bound_stretch: "#6A1B9A"
+  large_bound_stretch: "#FB8C00"
+  putative_nucleosome: "#6D4C41"
+  all_footprint_features: "#6A1B9A"
+  cpg_patch: "#6D4C41"
 hmm_merge_layer_features:
   - ["all_accessible_features", 60]
 clustermap_cmap_hmm: "coolwarm"
 hmm_clustermap_feature_layers:
-  - all_accessible_features
   - all_accessible_features_merged
-  - small_accessible_patch
-  - mid_accessible_patch
-  - large_accessible_patch
-  - large_accessible_patch_merged
-  - nucleosome_depleted_region
   - nucleosome_depleted_region_merged
   - small_bound_stretch
   - medium_bound_stretch
   - putative_nucleosome
-  - large_bound_stretch
+  - all_footprint_features
+hmm_clustermap_length_layers:
+  - all_accessible_features_merged
+  - all_footprint_features
 hmm_clustermap_sortby: "hmm"
 hmm_peak_feature_configs:
   all_accessible_features:

smftools/config/experiment_config.py

@@ -12,6 +12,7 @@ from smftools.constants import (
     BAM_SUFFIX,
     BARCODE_BOTH_ENDS,
     CONVERSIONS,
+    LOAD_DIR,
     MOD_LIST,
     MOD_MAP,
     REF_COL,
@@ -664,6 +665,8 @@ class ExperimentConfig:
     # General I/O
     input_data_path: Optional[str] = None
     output_directory: Optional[str] = None
+    emit_log_file: Optional[bool] = True
+    log_level: Optional[str] = "INFO"
     fasta: Optional[str] = None
     bam_suffix: str = BAM_SUFFIX
     recursive_input_search: bool = True
@@ -736,6 +739,7 @@ class ExperimentConfig:
     aligner_args: Optional[List[str]] = None
     make_bigwigs: bool = False
     make_beds: bool = False
+    annotate_secondary_supplementary: bool = True
     samtools_backend: str = "auto"
     bedtools_backend: str = "auto"
     bigwig_backend: str = "auto"
@@ -747,6 +751,9 @@ class ExperimentConfig:
     # General Plotting
     sample_name_col_for_plotting: Optional[str] = "Barcode"
     rows_per_qc_histogram_grid: int = 12
+    clustermap_demux_types_to_plot: List[str] = field(
+        default_factory=lambda: ["single", "double", "already"]
+    )
 
     # Preprocessing - Read length and quality filter params
     read_coord_filter: Optional[Sequence[float]] = field(default_factory=lambda: [None, None])
@@ -816,6 +823,9 @@ class ExperimentConfig:
     duplicate_detection_site_types: List[str] = field(
         default_factory=lambda: ["GpC", "CpG", "ambiguous_GpC_CpG"]
     )
+    duplicate_detection_demux_types_to_use: List[str] = field(
+        default_factory=lambda: ["single", "double", "already"]
+    )
     duplicate_detection_distance_threshold: float = 0.07
     hamming_vs_metric_keys: List[str] = field(default_factory=lambda: ["Fraction_C_site_modified"])
     duplicate_detection_keep_best_metric: str = "read_quality"
@@ -827,6 +837,13 @@ class ExperimentConfig:
 
     # Preprocessing - Position QC
     position_max_nan_threshold: float = 0.1
+    mismatch_frequency_range: Sequence[float] = field(default_factory=lambda: [0.05, 0.95])
+    mismatch_frequency_layer: str = "mismatch_integer_encoding"
+    mismatch_frequency_read_span_layer: str = "read_span_mask"
+    mismatch_base_frequency_exclude_mod_sites: bool = False
+    references_to_align_for_variant_annotation: List[Optional[str]] = field(
+        default_factory=lambda: [None, None]
+    )
 
     # Spatial Analysis - Clustermap params
     layer_for_clustermap_plotting: Optional[str] = "nan0_0minus1"
@@ -835,6 +852,45 @@ class ExperimentConfig:
     clustermap_cmap_cpg: Optional[str] = "coolwarm"
     clustermap_cmap_a: Optional[str] = "coolwarm"
     spatial_clustermap_sortby: Optional[str] = "gpc"
+    overlay_variant_calls: bool = False
+    variant_overlay_seq1_color: str = "white"
+    variant_overlay_seq2_color: str = "black"
+    variant_overlay_marker_size: float = 4.0
+    rolling_nn_layer: Optional[str] = "nan0_0minus1"
+    rolling_nn_plot_layer: Optional[str] = "nan0_0minus1"
+    rolling_nn_plot_layers: List[str] = field(
+        default_factory=lambda: ["nan0_0minus1", "nan0_0minus1"]
+    )
+    rolling_nn_window: int = 10
+    rolling_nn_step: int = 1
+    rolling_nn_min_overlap: int = 8
+    rolling_nn_return_fraction: bool = True
+    rolling_nn_obsm_key: str = "rolling_nn_dist"
+    rolling_nn_site_types: Optional[List[str]] = None
+    rolling_nn_write_zero_pairs_csvs: bool = True
+    rolling_nn_zero_pairs_uns_key: Optional[str] = None
+    rolling_nn_zero_pairs_segments_key: Optional[str] = None
+    rolling_nn_zero_pairs_layer_key: Optional[str] = None
+    rolling_nn_zero_pairs_refine: bool = True
+    rolling_nn_zero_pairs_max_nan_run: Optional[int] = None
+    rolling_nn_zero_pairs_merge_gap: int = 0
+    rolling_nn_zero_pairs_max_segments_per_read: Optional[int] = None
+    rolling_nn_zero_pairs_max_overlap: Optional[int] = None
+    rolling_nn_zero_pairs_layer_overlap_mode: str = "binary"
+    rolling_nn_zero_pairs_layer_overlap_value: Optional[int] = None
+    rolling_nn_zero_pairs_keep_uns: bool = True
+    rolling_nn_zero_pairs_segments_keep_uns: bool = True
+    rolling_nn_zero_pairs_top_segments_per_read: Optional[int] = None
+    rolling_nn_zero_pairs_top_segments_max_overlap: Optional[int] = None
+    rolling_nn_zero_pairs_top_segments_min_span: Optional[float] = None
+    rolling_nn_zero_pairs_top_segments_write_csvs: bool = True
+    rolling_nn_zero_pairs_segment_histogram_bins: int = 30
+
+    # Cross-sample rolling NN analysis
+    cross_sample_analysis: bool = False
+    cross_sample_grouping_col: Optional[str] = None
+    cross_sample_random_seed: int = 42
+    delta_hamming_chimeric_span_threshold: int = 200
 
     # Spatial Analysis - UMAP/Leiden params
     layer_for_umap_plotting: Optional[str] = "nan_half"
@@ -883,11 +939,15 @@ class ExperimentConfig:
     accessible_patches: Optional[bool] = True
     cpg: Optional[bool] = False
     hmm_feature_sets: Dict[str, Any] = field(default_factory=dict)
+    hmm_feature_colormaps: Dict[str, Any] = field(default_factory=dict)
     hmm_merge_layer_features: Optional[List[Tuple]] = field(default_factory=lambda: [(None, 60)])
     clustermap_cmap_hmm: Optional[str] = "coolwarm"
     hmm_clustermap_feature_layers: List[str] = field(
         default_factory=lambda: ["all_accessible_features"]
     )
+    hmm_clustermap_length_layers: List[str] = field(
+        default_factory=lambda: ["all_accessible_features"]
+    )
     hmm_clustermap_sortby: Optional[str] = "hmm"
     hmm_peak_feature_configs: Dict[str, Any] = field(default_factory=dict)
 
@@ -906,6 +966,8 @@ class ExperimentConfig:
     invert_adata: bool = False
     bypass_append_binary_layer_by_base_context: bool = False
     force_redo_append_binary_layer_by_base_context: bool = False
+    bypass_append_mismatch_frequency_sites: bool = False
+    force_redo_append_mismatch_frequency_sites: bool = False
     bypass_calculate_read_modification_stats: bool = False
     force_redo_calculate_read_modification_stats: bool = False
     bypass_filter_reads_on_modification_thresholds: bool = False
@@ -1110,7 +1172,7 @@ class ExperimentConfig:
 
         # Demultiplexing output path
         split_dir = merged.get("split_dir", SPLIT_DIR)
-        split_path = output_dir / split_dir
+        split_path = output_dir / LOAD_DIR / split_dir
 
         # final normalization
         if "strands" in merged:
@@ -1121,6 +1183,10 @@ class ExperimentConfig:
             merged["mod_target_bases"] = _parse_list(merged["mod_target_bases"])
         if "conversion_types" in merged:
             merged["conversion_types"] = _parse_list(merged["conversion_types"])
+        if "references_to_align_for_variant_annotation" in merged:
+            merged["references_to_align_for_variant_annotation"] = _parse_list(
+                merged["references_to_align_for_variant_annotation"]
+            )
 
         merged["filter_threshold"] = float(_parse_numeric(merged.get("filter_threshold", 0.8), 0.8))
         merged["m6A_threshold"] = float(_parse_numeric(merged.get("m6A_threshold", 0.7), 0.7))
@@ -1197,6 +1263,9 @@ class ExperimentConfig:
         # Final normalization of hmm_feature_sets and canonical local variables
         merged["hmm_feature_sets"] = normalize_hmm_feature_sets(merged.get("hmm_feature_sets", {}))
         hmm_feature_sets = merged.get("hmm_feature_sets", {})
+        hmm_feature_colormaps = merged.get("hmm_feature_colormaps", {})
+        if not isinstance(hmm_feature_colormaps, dict):
+            hmm_feature_colormaps = {}
         hmm_annotation_threshold = merged.get("hmm_annotation_threshold", 0.5)
         hmm_batch_size = int(merged.get("hmm_batch_size", 1024))
         hmm_use_viterbi = bool(merged.get("hmm_use_viterbi", False))
@@ -1211,6 +1280,9 @@ class ExperimentConfig:
         hmm_clustermap_feature_layers = _parse_list(
             merged.get("hmm_clustermap_feature_layers", "all_accessible_features")
         )
+        hmm_clustermap_length_layers = _parse_list(
+            merged.get("hmm_clustermap_length_layers", hmm_clustermap_feature_layers)
+        )
 
         hmm_fit_strategy = str(merged.get("hmm_fit_strategy", "per_group")).strip()
         hmm_shared_scope = _parse_list(merged.get("hmm_shared_scope", ["reference", "methbase"]))
@@ -1231,6 +1303,7 @@ class ExperimentConfig:
 
         # instantiate dataclass
         instance = cls(
+            annotate_secondary_supplementary=merged.get("annotate_secondary_supplementary", True),
             smf_modality=merged.get("smf_modality"),
             input_data_path=input_data_path,
             recursive_input_search=merged.get("recursive_input_search"),
@@ -1257,6 +1330,8 @@ class ExperimentConfig:
             trim=merged.get("trim", TRIM),
             input_already_demuxed=merged.get("input_already_demuxed", False),
             threads=merged.get("threads"),
+            emit_log_file=merged.get("emit_log_file", True),
+            log_level=merged.get("log_level", "INFO"),
             sample_sheet_path=merged.get("sample_sheet_path"),
             sample_sheet_mapping_column=merged.get("sample_sheet_mapping_column"),
             delete_intermediate_bams=merged.get("delete_intermediate_bams", False),
@@ -1313,6 +1388,9 @@ class ExperimentConfig:
             ),
             reindexing_offsets=merged.get("reindexing_offsets", {None: None}),
             reindexed_var_suffix=merged.get("reindexed_var_suffix", "reindexed"),
+            clustermap_demux_types_to_plot=merged.get(
+                "clustermap_demux_types_to_plot", ["single", "double", "already"]
+            ),
             layer_for_clustermap_plotting=merged.get(
                 "layer_for_clustermap_plotting", "nan0_0minus1"
             ),
@@ -1321,6 +1399,65 @@ class ExperimentConfig:
             clustermap_cmap_cpg=merged.get("clustermap_cmap_cpg", "coolwarm"),
             clustermap_cmap_a=merged.get("clustermap_cmap_a", "coolwarm"),
             spatial_clustermap_sortby=merged.get("spatial_clustermap_sortby", "gpc"),
+            overlay_variant_calls=_parse_bool(merged.get("overlay_variant_calls", False)),
+            variant_overlay_seq1_color=merged.get("variant_overlay_seq1_color", "white"),
+            variant_overlay_seq2_color=merged.get("variant_overlay_seq2_color", "black"),
+            variant_overlay_marker_size=float(merged.get("variant_overlay_marker_size", 4.0)),
+            rolling_nn_layer=merged.get("rolling_nn_layer", "nan0_0minus1"),
+            rolling_nn_plot_layer=merged.get("rolling_nn_plot_layer", "nan0_0minus1"),
+            rolling_nn_plot_layers=merged.get(
+                "rolling_nn_plot_layers", ["nan0_0minus1", "nan0_0minus1"]
+            ),
+            rolling_nn_window=merged.get("rolling_nn_window", 15),
+            rolling_nn_step=merged.get("rolling_nn_step", 2),
+            rolling_nn_min_overlap=merged.get("rolling_nn_min_overlap", 10),
+            rolling_nn_return_fraction=merged.get("rolling_nn_return_fraction", True),
+            rolling_nn_obsm_key=merged.get("rolling_nn_obsm_key", "rolling_nn_dist"),
+            rolling_nn_site_types=merged.get("rolling_nn_site_types", None),
+            rolling_nn_write_zero_pairs_csvs=merged.get("rolling_nn_write_zero_pairs_csvs", True),
+            rolling_nn_zero_pairs_uns_key=merged.get("rolling_nn_zero_pairs_uns_key", None),
+            rolling_nn_zero_pairs_segments_key=merged.get(
+                "rolling_nn_zero_pairs_segments_key", None
+            ),
+            rolling_nn_zero_pairs_layer_key=merged.get("rolling_nn_zero_pairs_layer_key", None),
+            rolling_nn_zero_pairs_refine=merged.get("rolling_nn_zero_pairs_refine", True),
+            rolling_nn_zero_pairs_max_nan_run=merged.get("rolling_nn_zero_pairs_max_nan_run", None),
+            rolling_nn_zero_pairs_merge_gap=merged.get("rolling_nn_zero_pairs_merge_gap", 0),
+            rolling_nn_zero_pairs_max_segments_per_read=merged.get(
+                "rolling_nn_zero_pairs_max_segments_per_read", None
+            ),
+            rolling_nn_zero_pairs_max_overlap=merged.get("rolling_nn_zero_pairs_max_overlap", None),
+            rolling_nn_zero_pairs_layer_overlap_mode=merged.get(
+                "rolling_nn_zero_pairs_layer_overlap_mode", "binary"
+            ),
+            rolling_nn_zero_pairs_layer_overlap_value=merged.get(
+                "rolling_nn_zero_pairs_layer_overlap_value", None
+            ),
+            rolling_nn_zero_pairs_keep_uns=merged.get("rolling_nn_zero_pairs_keep_uns", True),
+            rolling_nn_zero_pairs_segments_keep_uns=merged.get(
+                "rolling_nn_zero_pairs_segments_keep_uns", True
+            ),
+            rolling_nn_zero_pairs_top_segments_per_read=merged.get(
+                "rolling_nn_zero_pairs_top_segments_per_read", None
+            ),
+            rolling_nn_zero_pairs_top_segments_max_overlap=merged.get(
+                "rolling_nn_zero_pairs_top_segments_max_overlap", None
+            ),
+            rolling_nn_zero_pairs_top_segments_min_span=merged.get(
+                "rolling_nn_zero_pairs_top_segments_min_span", None
+            ),
+            rolling_nn_zero_pairs_top_segments_write_csvs=merged.get(
+                "rolling_nn_zero_pairs_top_segments_write_csvs", True
+            ),
+            rolling_nn_zero_pairs_segment_histogram_bins=merged.get(
+                "rolling_nn_zero_pairs_segment_histogram_bins", 30
+            ),
+            cross_sample_analysis=merged.get("cross_sample_analysis", False),
+            cross_sample_grouping_col=merged.get("cross_sample_grouping_col", None),
+            cross_sample_random_seed=merged.get("cross_sample_random_seed", 42),
+            delta_hamming_chimeric_span_threshold=merged.get(
+                "delta_hamming_chimeric_span_threshold", 200
+            ),
             layer_for_umap_plotting=merged.get("layer_for_umap_plotting", "nan_half"),
             umap_layers_to_plot=merged.get(
                 "umap_layers_to_plot", ["mapped_length", "Raw_modification_signal"]
@@ -1347,6 +1484,7 @@ class ExperimentConfig:
             hmm_emission_adapt_tol=hmm_emission_adapt_tol,
             hmm_dtype=merged.get("hmm_dtype", "float64"),
             hmm_feature_sets=hmm_feature_sets,
+            hmm_feature_colormaps=hmm_feature_colormaps,
             hmm_annotation_threshold=hmm_annotation_threshold,
             hmm_batch_size=hmm_batch_size,
             hmm_use_viterbi=hmm_use_viterbi,
@@ -1355,6 +1493,7 @@ class ExperimentConfig:
             hmm_merge_layer_features=hmm_merge_layer_features,
             clustermap_cmap_hmm=merged.get("clustermap_cmap_hmm", "coolwarm"),
             hmm_clustermap_feature_layers=hmm_clustermap_feature_layers,
+            hmm_clustermap_length_layers=hmm_clustermap_length_layers,
             hmm_clustermap_sortby=merged.get("hmm_clustermap_sortby", "hmm"),
             hmm_peak_feature_configs=hmm_peak_feature_configs,
             footprints=merged.get("footprints", None),
@@ -1390,6 +1529,9 @@ class ExperimentConfig:
             duplicate_detection_site_types=merged.get(
                 "duplicate_detection_site_types", ["GpC", "CpG", "ambiguous_GpC_CpG"]
             ),
+            duplicate_detection_demux_types_to_use=merged.get(
+                "duplicate_detection_demux_types_to_use", ["single", "double", "already"]
+            ),
             duplicate_detection_distance_threshold=merged.get(
                 "duplicate_detection_distance_threshold", 0.07
             ),
@@ -1479,6 +1621,9 @@ class ExperimentConfig:
             force_redo_hmm_fit=merged.get("force_redo_hmm_fit", False),
             bypass_hmm_apply=merged.get("bypass_hmm_apply", False),
             force_redo_hmm_apply=merged.get("force_redo_hmm_apply", False),
+            references_to_align_for_variant_annotation=merged.get(
+                "references_to_align_for_variant_annotation", [None, None]
+            ),
             config_source=config_source or "<var_dict>",
         )
 

smftools/constants.py

@@ -21,7 +21,34 @@ BAM_SUFFIX: Final[str] = ".bam"
 BARCODE_BOTH_ENDS: Final[bool] = False
 REF_COL: Final[str] = "Reference_strand"
 SAMPLE_COL: Final[str] = "Experiment_name_and_barcode"
+SAMPLE: Final[str] = "Sample"
 SPLIT_DIR: Final[str] = "demultiplexed_BAMs"
+H5_DIR: Final[str] = "h5ads"
+DEMUX_TYPE: Final[str] = "demux_type"
+BARCODE: Final[str] = "Barcode"
+REFERENCE: Final[str] = "Reference"
+REFERENCE_STRAND: Final[str] = "Reference_strand"
+REFERENCE_DATASET_STRAND: Final[str] = "Reference_dataset_strand"
+STRAND: Final[str] = "Strand"
+DATASET: Final[str] = "Dataset"
+READ_MISMATCH_TREND: Final[str] = "Read_mismatch_trend"
+READ_MAPPING_DIRECTION: Final[str] = "Read_mapping_direction"
+SEQUENCE_INTEGER_ENCODING: Final[str] = "sequence_integer_encoding"
+SEQUENCE_INTEGER_DECODING: Final[str] = "sequence_integer_decoding"
+MISMATCH_INTEGER_ENCODING: Final[str] = "mismatch_integer_encoding"
+BASE_QUALITY_SCORES: Final[str] = "base_quality_scores"
+READ_SPAN_MASK: Final[str] = "read_span_mask"
+
+LOAD_DIR: Final[str] = "load_adata_outputs"
+PREPROCESS_DIR: Final[str] = "preprocess_adata_outputs"
+SPATIAL_DIR: Final[str] = "spatial_adata_outputs"
+HMM_DIR: Final[str] = "hmm_adata_outputs"
+LATENT_DIR: Final[str] = "latent_adata_outputs"
+VARIANT_DIR: Final[str] = "variant_adata_outputs"
+CHIMERIC_DIR: Final[str] = "chimeric_adata_outputs"
+
+LOGGING_DIR: Final[str] = "logs"
+
 TRIM: Final[bool] = False
 
 _private_conversions = ["unconverted"]
@@ -35,3 +62,45 @@ MOD_MAP: Final[Mapping[str, str]] = _deep_freeze(_private_mod_map)
 
 _private_strands = ("bottom", "top")
 STRANDS: Final[tuple[str, ...]] = _deep_freeze(_private_strands)
+
+MODKIT_EXTRACT_TSV_COLUMN_CHROM: Final[str] = "chrom"
+MODKIT_EXTRACT_TSV_COLUMN_REF_POSITION: Final[str] = "ref_position"
+MODKIT_EXTRACT_TSV_COLUMN_MODIFIED_PRIMARY_BASE: Final[str] = "modified_primary_base"
+MODKIT_EXTRACT_TSV_COLUMN_REF_STRAND: Final[str] = "ref_strand"
+MODKIT_EXTRACT_TSV_COLUMN_READ_ID: Final[str] = "read_id"
+MODKIT_EXTRACT_TSV_COLUMN_CALL_CODE: Final[str] = "call_code"
+MODKIT_EXTRACT_TSV_COLUMN_CALL_PROB: Final[str] = "call_prob"
+
+MODKIT_EXTRACT_MODIFIED_BASE_A: Final[str] = "A"
+MODKIT_EXTRACT_MODIFIED_BASE_C: Final[str] = "C"
+MODKIT_EXTRACT_REF_STRAND_PLUS: Final[str] = "+"
+MODKIT_EXTRACT_REF_STRAND_MINUS: Final[str] = "-"
+
+_private_modkit_extract_call_code_modified = ("a", "h", "m")
+MODKIT_EXTRACT_CALL_CODE_MODIFIED: Final[tuple[str, ...]] = _deep_freeze(
+    _private_modkit_extract_call_code_modified
+)
+_private_modkit_extract_call_code_canonical = ("-",)
+MODKIT_EXTRACT_CALL_CODE_CANONICAL: Final[tuple[str, ...]] = _deep_freeze(
+    _private_modkit_extract_call_code_canonical
+)
+
+MODKIT_EXTRACT_SEQUENCE_BASES: Final[tuple[str, ...]] = _deep_freeze(("A", "C", "G", "T", "N"))
+MODKIT_EXTRACT_SEQUENCE_PADDING_BASE: Final[str] = "PAD"
+_private_modkit_extract_base_to_int: Dict[str, int] = {
+    "A": 0,
+    "C": 1,
+    "G": 2,
+    "T": 3,
+    "N": 4,
+    "PAD": 5,
+}
+MODKIT_EXTRACT_SEQUENCE_BASE_TO_INT: Final[Mapping[str, int]] = _deep_freeze(
+    _private_modkit_extract_base_to_int
+)
+_private_modkit_extract_int_to_base: Dict[int, str] = {
+    value: key for key, value in _private_modkit_extract_base_to_int.items()
+}
+MODKIT_EXTRACT_SEQUENCE_INT_TO_BASE: Final[Mapping[int, str]] = _deep_freeze(
+    _private_modkit_extract_int_to_base
+)

smftools/hmm/HMM.py

@@ -144,6 +144,83 @@ def _safe_int_coords(var_names) -> Tuple[np.ndarray, bool]:
         return np.arange(len(var_names), dtype=int), False
 
 
+def mask_layers_outside_read_span(
+    adata,
+    layers: Sequence[str],
+    *,
+    start_key: str = "reference_start",
+    end_key: str = "reference_end",
+    use_original_var_names: bool = True,
+) -> List[str]:
+    """Mask layer values outside read reference spans with NaN.
+
+    This uses integer coordinate comparisons against either ``adata.var["Original_var_names"]``
+    (when present) or ``adata.var_names``. Values strictly less than ``start_key`` or greater
+    than ``end_key`` are set to NaN for each read.
+
+    Args:
+        adata: AnnData object to modify in-place.
+        layers: Layer names to mask.
+        start_key: obs column holding reference start positions.
+        end_key: obs column holding reference end positions.
+        use_original_var_names: Use ``adata.var["Original_var_names"]`` when available.
+
+    Returns:
+        List of layer names that were masked.
+    """
+    if not layers:
+        return []
+
+    if start_key not in adata.obs or end_key not in adata.obs:
+        raise KeyError(f"Missing {start_key!r} or {end_key!r} in adata.obs.")
+
+    coord_source = adata.var_names
+    if use_original_var_names and "Original_var_names" in adata.var:
+        orig = np.asarray(adata.var["Original_var_names"])
+        if orig.size == adata.n_vars:
+            try:
+                orig_numeric = np.asarray(orig, dtype=float)
+            except (TypeError, ValueError):
+                orig_numeric = None
+            if orig_numeric is not None and np.isfinite(orig_numeric).any():
+                coord_source = orig
+
+    coords, _ = _safe_int_coords(coord_source)
+    if coords.shape[0] != adata.n_vars:
+        raise ValueError("Coordinate source length does not match adata.n_vars.")
+
+    try:
+        starts = np.asarray(adata.obs[start_key], dtype=float)
+        ends = np.asarray(adata.obs[end_key], dtype=float)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("Start/end positions must be numeric.") from exc
+
+    masked = []
+    for layer in layers:
+        if layer not in adata.layers:
+            raise KeyError(f"Layer {layer!r} not found in adata.layers.")
+
+        arr = np.asarray(adata.layers[layer])
+        if not np.issubdtype(arr.dtype, np.floating):
+            arr = arr.astype(float, copy=True)
+
+        for i in range(adata.n_obs):
+            start = starts[i]
+            end = ends[i]
+            if not np.isfinite(start) or not np.isfinite(end):
+                continue
+            start_i = int(start)
+            end_i = int(end)
+            row_mask = (coords < start_i) | (coords > end_i)
+            if row_mask.any():
+                arr[i, row_mask] = np.nan
+
+        adata.layers[layer] = arr
+        masked.append(layer)
+
+    return masked
+
+
 def _logsumexp(x: torch.Tensor, dim: int) -> torch.Tensor:
     """Compute log-sum-exp in a numerically stable way.
 
@@ -1064,6 +1141,8 @@ class BaseHMM(nn.Module):
         uns_key: str = "hmm_appended_layers",
         uns_flag: str = "hmm_annotated",
         force_redo: bool = False,
+        mask_to_read_span: bool = True,
+        mask_use_original_var_names: bool = True,
         device: Optional[Union[str, torch.device]] = None,
         **kwargs,
     ):
@@ -1085,6 +1164,8 @@ class BaseHMM(nn.Module):
             uns_key: .uns key to track appended layers.
             uns_flag: .uns flag to mark annotations.
             force_redo: Whether to overwrite existing layers.
+            mask_to_read_span: Whether to mask appended layers outside read spans.
+            mask_use_original_var_names: Use ``adata.var["Original_var_names"]`` when available.
             device: Device specifier.
             **kwargs: Additional parameters for specialized workflows.
 
@@ -1245,6 +1326,13 @@ class BaseHMM(nn.Module):
                     np.asarray(adata.layers[nm])
                 )
 
+        if mask_to_read_span and appended:
+            mask_layers_outside_read_span(
+                adata,
+                appended,
+                use_original_var_names=mask_use_original_var_names,
+            )
+
         adata.uns[uns_key] = appended
         adata.uns[uns_flag] = True
         return None

smftools/hmm/call_hmm_peaks.py

@@ -51,7 +51,7 @@ def call_hmm_peaks(
         raise KeyError(f"obs column '{ref_column}' not found")
 
     # Ensure categorical for predictable ref iteration
-    if not pd.api.types.is_categorical_dtype(adata.obs[ref_column]):
+    if not isinstance(adata.obs[ref_column].dtype, pd.CategoricalDtype):
         adata.obs[ref_column] = adata.obs[ref_column].astype("category")
 
     # Optional: drop duplicate obs columns once to avoid Pandas/AnnData view quirks