smftools 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

smftools/config/default.yaml

@@ -1,7 +1,7 @@
 # General
 sample_sheet_path: null # path to sample_sheet to load metadata into anndata.
-sample_sheet_mapping_column: 'Barcode' # The column in the sample sheet and current anndata to use for mapping metadata.
-sample_name_col_for_plotting: 'Barcode'
+sample_sheet_mapping_column: 'Experiment_name_and_barcode' # The column in the sample sheet and current anndata to use for mapping metadata.
+sample_name_col_for_plotting: 'Experiment_name_and_barcode'
 
 # Compute params
 threads: 4
@@ -9,9 +9,7 @@ device: "auto"
 
 ######## smftools load params #########
 # Generic i/o
-bam_suffix: ".bam"
 recursive_input_search: True
-split_dir: "demultiplexed_BAMs"
 strands:
   - bottom
   - top
@@ -40,7 +38,8 @@ model: "hac" # needed for dorado basecaller
 filter_threshold: 0.8 # Dorado probability filter threshold for base calling.
 
 # Alignment params
-aligner: "minimap2" # Aligner to use: dorado, minimap2
+aligner: "dorado" # Aligner to use: dorado, minimap2
+align_from_bam: False # Whether to run alignment from a bam file for minimap2. If False, runs alignment from a FASTQ file.
 aligner_args:
   minimap2:
     ont:
@@ -52,7 +51,6 @@ aligner_args:
       - '-y'
       - '-N'
       - '5'
-      - '--secondary=no' 
     pacbio:
       - '-a'
       - '-x'
@@ -62,7 +60,6 @@ aligner_args:
       - '-y'
       - '-N'
       - '5'
-      - '--secondary=no' 
     illumina:
       - '-a'
       - '-x'
@@ -72,7 +69,6 @@ aligner_args:
       - '-y'
       - '-N'
       - '5'
-      - '--secondary=no'     
   dorado:
     ont:
       - "--mm2-opts"
@@ -87,9 +83,9 @@ barcode_both_ends: False # dorado demultiplexing
 trim: False # dorado adapter and barcode removal during demultiplexing
 
 # Anndata structure
-mapping_threshold: 0.01 # Minimum proportion of mapped reads that need to fall within a region to include in the final AnnData.
+mapping_threshold: 0.10 # Minimum proportion of mapped reads that need to fall within a region to include in the final AnnData.
 reference_column: 'Reference_strand'
-sample_column: 'Barcode'
+sample_column: 'Experiment_name_and_barcode'
 
 ######## smftools preprocess params #########
 # Read length, quality, and mapping filtering params
@@ -100,7 +96,7 @@ read_len_filter_thresholds:
   - 100
   - null
 read_len_to_ref_ratio_filter_thresholds:
-  - 0.5
+  - null
   - null
 read_quality_filter_thresholds:
   - 15
@@ -116,7 +112,7 @@ read_mod_filtering_gpc_thresholds:
 read_mod_filtering_cpg_thresholds:
   - 0.0
   - 1.0
-read_mod_filtering_any_c_thresholds:
+read_mod_filtering_c_thresholds:
   - 0.025
   - 0.975
 read_mod_filtering_a_thresholds:
@@ -125,6 +121,16 @@ read_mod_filtering_a_thresholds:
 read_mod_filtering_use_other_c_as_background: False
 min_valid_fraction_positions_in_read_vs_ref: 0.5
 
+# Plotting params for read length histograms
+obs_to_plot_pp_qc:
+  - read_length
+  - mapped_length
+  - read_quality
+  - mapping_quality
+  - mapped_length_to_reference_length_ratio
+  - mapped_length_to_read_length_ratio
+  - Raw_modification_signal
+
 # Duplicate detection params
 duplicate_detection_site_types: # Site types to consider for duplicate detection workflow
   - "GpC"
@@ -132,7 +138,7 @@ duplicate_detection_site_types: # Site types to consider for duplicate detection
   - "ambiguous_GpC_CpG"
 duplicate_detection_distance_threshold: 0.07 # Hamming distance based similarity threshold to use for marking duplicate reads.
 hamming_vs_metric_keys: # Metrics to plot the hamming distance against.
-  - Fraction_any_C_site_modified
+  - Fraction_C_site_modified
 duplicate_detection_keep_best_metric: "read_quality" # Obs metric to use to keep a representative read from a read duplicate cluster
 duplicate_detection_window_size_for_hamming_neighbors: 50 # How many neighboring reads to look at for calculating hamming distance pairs
 duplicate_detection_min_overlapping_positions: 20 # The minimum amount of valid overlapping positions that will allow duplicate detection to work
@@ -143,29 +149,39 @@ duplicate_detection_do_pca: False # Whether to do PCA before hierarchical linkag
 # Position QC params
 position_max_nan_threshold: 0.1 # The maximum amount of nans to tolerate in a column
 
-######## smftools analyze params #########
-# Basic Analysis - QC Plotting params
+######## smftools spatial params #########
+invert_adata: False # Whether to invert the AnnData along the positions axis.
+# Reindexing params
+reindexing_offsets:
+  null : null
+reindexed_var_suffix: "reindexed"
+
+# Spatial Analysis - QC Plotting params
 rows_per_qc_histogram_grid: 12
 
-# Basic Analysis - Clustermap params
+# Spatial Analysis - Clustermap params
 layer_for_clustermap_plotting: 'nan0_0minus1'
+clustermap_cmap_c: "coolwarm"
+clustermap_cmap_gpc: "coolwarm"
+clustermap_cmap_cpg: "coolwarm"
+clustermap_cmap_a: "coolwarm"
+spatial_clustermap_sortby: "gpc"
 
-# Basic Analysis - UMAP/Leiden params
+# Spatial Analysis - UMAP/Leiden params
 layer_for_umap_plotting: 'nan_half'
 umap_layers_to_plot:
   - "mapped_length"
   - "Raw_modification_signal"
 
-# Basic Analysis - Spatial Autocorrelation params
+# Spatial Analysis - Spatial Autocorrelation params
+autocorr_normalization_method: "pearson" # options are pearson or sum
 rows_per_qc_autocorr_grid: 6
 autocorr_rolling_window_size: 25
 autocorr_max_lag: 800
 autocorr_site_types:
   - "GpC"
-  - "CpG"
-  - "any_C"
 
-# Basic Analysis - Correlation Matrix params
+# Spatial Analysis - Correlation Matrix params
 correlation_matrix_types:
   - "pearson"
   - "binary_covariance"
@@ -188,10 +204,19 @@ hmm_init_start_probs:
   - 0.5
   - 0.5
 hmm_eps: 1e-8
+# Fitting strategy
+hmm_fit_strategy: "per_group"      # "per_group" | "shared_transitions"
+hmm_shared_scope: ["reference", "methbase"]
+hmm_groupby: ["sample", "reference", "methbase"]
+# If hmm_fit_strategy == shared_transitions
+hmm_adapt_emissions: true
+hmm_adapt_startprobs: true
+hmm_emission_adapt_iters: 5
+hmm_emission_adapt_tol: 1.0e-4
 hmm_dtype: "float64"
-hmm_annotation_threshold: 0.5
-hmm_batch_size: 1024
-hmm_use_viterbi: False
+hmm_annotation_threshold: 0.5 # The minimum probability threshold of a feature interval to accept it for layer annotation.
+hmm_batch_size: 1024 # hmm batch size
+hmm_use_viterbi: False # Whether to use viterbi decoding. If False, uses forward-backward gammas. Viterbi is smoother, but less sensitive.
 footprints: True # whether to use the default HMM footprint params
 accessible_patches: True # whether to use the default HMM accessible patch params
 cpg: False # whether to use the default HMM endogenous CpG patch params
@@ -204,19 +229,104 @@ hmm_feature_sets:
   footprint:
     state: "Non-Modified"
     features:
-      small_bound_stretch: [10, 40]
-      medium_bound_stretch: [40, 110]
-      putative_nucleosome: [110, 200]
+      small_bound_stretch: [6, 40]
+      medium_bound_stretch: [40, 100]
+      putative_nucleosome: [100, 200]
       large_bound_stretch: [200, inf]
   accessible:
     state: "Modified"
     features:
       small_accessible_patch: [3, 20]
       mid_accessible_patch: [20, 40]
-      mid_large_accessible_patch: [40, 110]
-      large_accessible_patch: [110, inf]
+      large_accessible_patch: [40, 110]
+      nucleosome_depleted_region: [110, inf]
 hmm_merge_layer_features:
-  - [null, 80]
+  - ["all_accessible_features", 60]
+clustermap_cmap_hmm: "coolwarm"
+hmm_clustermap_feature_layers:
+  - all_accessible_features
+  - all_accessible_features_merged
+  - small_accessible_patch
+  - mid_accessible_patch
+  - large_accessible_patch
+  - large_accessible_patch_merged
+  - nucleosome_depleted_region
+  - nucleosome_depleted_region_merged
+  - small_bound_stretch
+  - medium_bound_stretch
+  - putative_nucleosome
+  - large_bound_stretch
+hmm_clustermap_sortby: "hmm"
+hmm_peak_feature_configs:
+  all_accessible_features:
+    min_distance: 200 # The minimum distance in between called peaks
+    peak_width: 200 # The window width to calculate sum/mean hmm signal per read centered at the peak center.
+    peak_prominence: 0.1 # The minimum prominence to call a peak
+    peak_threshold: 0.80 # The minimum mean hmm signal in each molecule within the peak window to mark the molecule as positive for the feature.
+    rolling_window: 50 # Window size for the rolling average smoothing before peak calling
+
+  all_accessible_features_merged:
+    min_distance: 250
+    peak_width: 250
+    peak_prominence: 0.05
+    peak_threshold: 0.80
+    rolling_window: 50 
+
+  small_accessible_patch:
+    min_distance: 40
+    peak_width: 30
+    peak_prominence: 0.1
+    peak_threshold: 0.8
+    rolling_window: 40 
+
+  mid_accessible_patch:
+    min_distance: 100
+    peak_width: 60
+    peak_prominence: 0.025
+    peak_threshold: 0.80
+    rolling_window: 50 
+
+  large_accessible_patch:
+    min_distance: 100
+    peak_width: 100
+    peak_prominence: 0.025
+    peak_threshold: 0.80
+    rolling_window: 50 
+
+  nucleosome_depleted_region:
+    min_distance: 200
+    peak_width: 200
+    peak_prominence: 0.025
+    peak_threshold: 0.80
+    rolling_window: 50 
+
+  small_bound_stretch:
+    min_distance: 20
+    peak_width: 20
+    peak_prominence: 0.01
+    peak_threshold: 0.50
+    rolling_window: 10 
+
+  medium_bound_stretch:
+    min_distance: 40
+    peak_width: 40
+    peak_prominence: 0.01
+    peak_threshold: 0.50
+    rolling_window: 20 
+
+  putative_nucleosome:
+    min_distance: 160
+    peak_width: 147     # canonical nucleosome footprint
+    peak_prominence: 0.025
+    peak_threshold: 0.60
+    rolling_window: 20 
+
+  large_bound_stretch:
+    min_distance: 250
+    peak_width: 300
+    peak_prominence: 0.20
+    peak_threshold: 0.80
+    rolling_window: 50 
 
 # Pipeline control flow - load adata
 force_redo_load_adata: False # Whether to perform load adata command from start
@@ -230,7 +340,6 @@ bypass_clean_nan: False # Whether to skip NaN cleaning
 force_redo_clean_nan: False # Whether to redo NaN cleaning
 bypass_append_base_context: False # Whether to skip adding per reference base context additions.
 force_redo_append_base_context: False # Whether to redo per reference base context additions.
-invert_adata: False # Whether to invert the AnnData along the positions axis.
 bypass_append_binary_layer_by_base_context: False # Whether to skip adding new binary layers for each specific base context.
 force_redo_append_binary_layer_by_base_context: False # Whether to redo adding new binary layers for each specific base context.
 bypass_calculate_read_modification_stats: False # Whether to skip adding read level modification statistics.
@@ -242,8 +351,8 @@ force_redo_flag_duplicate_reads: False # Whether to redo flagging duplicate read
 bypass_complexity_analysis: False # Whether to skip complexity analysis
 force_redo_complexity_analysis: False # Whether to redo complexity analysis
 
-# Pipeline control flow - Basic Analyses
-force_redo_basic_analyses: False # Whether to force redo the entire basic analysis pipeline from the AnnData
+# Pipeline control flow - Spatial Analyses
+force_redo_spatial_analyses: False # Whether to force redo the entire basic analysis pipeline from the AnnData
 bypass_basic_clustermaps: False # Whether to skip basic clustermap plotting
 force_redo_basic_clustermaps: False # Whether to redo basic clustermap plotting
 bypass_basic_umap: False # Whether to skip basic UMAP calculation/plotting

smftools/config/direct.yaml

@@ -14,6 +14,9 @@ thresholds:
 mod_list: 
   - '5mC_5hmC'
   - '6mA' # mods to detect
+mod_map:
+  5mC_5hmC: 5mC
+  6mA: 6mA
 mod_target_bases:
   - "A"
 enzyme_target_bases:
@@ -24,10 +27,10 @@ delete_batch_hdfs: True # Whether to delete intermediate barcode level hdfs afte
 
 ######## smftools preprocess params ########
 fit_position_methylation_thresholds: False # Whether to use Youden J-stat to determine position by positions thresholds for modification binarization.
-binarize_on_fixed_methlyation_threshold: 0.7 # The threshold used to binarize the anndata using a fixed value if fitting parameter above is False.
+binarize_on_fixed_methlyation_threshold: 0.5 # The threshold used to binarize the anndata using a fixed value if fitting parameter above is False.
 positive_control_sample_methylation_fitting: null # A positive control Sample_name to use for fully modified template data
 negative_control_sample_methylation_fitting: null # A negative control Sample_name to use for fully unmodified template data
-infer_on_percentile_sample_methylation_fitting: 10 # If a positive/negative control are not provided and fitting the data is requested, use the indicated percentile windows from the top and bottom of the dataset.
+infer_on_percentile_sample_methylation_fitting: 5 # If a positive/negative control are not provided and fitting the data is requested, use the indicated percentile windows from the top and bottom of the dataset.
 inference_variable_sample_methylation_fitting: "Raw_modification_signal" # The obs column value used for the percentile metric above.
 fit_j_threshold: 0.5 # The J-statistic threhold to use for determining which positions pass qc for mod detection thresholding
 output_binary_layer_name: "binarized_methylation" # The layer to store the binarized methylation data in
@@ -36,6 +39,11 @@ output_binary_layer_name: "binarized_methylation" # The layer to store the binar
 autocorr_site_types:
   - "A"
 
+spatial_clustermap_sortby: "a"
+
 ######## smftools hmm params #########
 hmm_methbases:
-  - "A"
+  - "A"
+
+hmm_merge_layer_features:
+  - ["A_all_accessible_features", 60]

smftools/config/discover_input_files.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import Dict, List, Any, Iterable, Union
+from typing import Any, Dict, List, Union
+
+from smftools.constants import BAM_SUFFIX
+
 
 def discover_input_files(
     input_data_path: Union[str, Path],
-    bam_suffix: str = ".bam",
+    bam_suffix: str = BAM_SUFFIX,
     recursive: bool = False,
     follow_symlinks: bool = False,
 ) -> Dict[str, Any]:
@@ -30,10 +33,21 @@ def discover_input_files(
     bam_suffix = bam_suffix.lower()
 
     # Sets of canonical extension keys we’ll compare against
-    pod5_exts  = {".pod5", ".p5"}
+    pod5_exts = {".pod5", ".p5"}
     fast5_exts = {".fast5", ".f5"}
-    fastq_exts = {".fastq", ".fq", ".fastq.gz", ".fq.gz", ".fastq.bz2", ".fq.bz2", ".fastq.xz", ".fq.xz", ".fastq.zst", ".fq.zst"}
-    h5ad_exts  = {".h5ad", ".h5"}
+    fastq_exts = {
+        ".fastq",
+        ".fq",
+        ".fastq.gz",
+        ".fq.gz",
+        ".fastq.bz2",
+        ".fq.bz2",
+        ".fastq.xz",
+        ".fq.xz",
+        ".fastq.zst",
+        ".fq.zst",
+    }
+    h5ad_exts = {".h5ad", ".h5"}
     compressed_exts = {".gz", ".bz2", ".xz", ".zst"}
 
     def ext_key(pp: Path) -> str: