smftools 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

smftools/config/experiment_config.py

@@ -12,6 +12,7 @@ from smftools.constants import (
     BAM_SUFFIX,
     BARCODE_BOTH_ENDS,
     CONVERSIONS,
+    LOAD_DIR,
     MOD_LIST,
     MOD_MAP,
     REF_COL,
@@ -664,6 +665,8 @@ class ExperimentConfig:
     # General I/O
     input_data_path: Optional[str] = None
     output_directory: Optional[str] = None
+    emit_log_file: Optional[bool] = True
+    log_level: Optional[str] = "INFO"
     fasta: Optional[str] = None
     bam_suffix: str = BAM_SUFFIX
     recursive_input_search: bool = True
@@ -736,6 +739,7 @@ class ExperimentConfig:
     aligner_args: Optional[List[str]] = None
     make_bigwigs: bool = False
     make_beds: bool = False
+    annotate_secondary_supplementary: bool = True
     samtools_backend: str = "auto"
     bedtools_backend: str = "auto"
     bigwig_backend: str = "auto"
@@ -747,6 +751,9 @@ class ExperimentConfig:
     # General Plotting
     sample_name_col_for_plotting: Optional[str] = "Barcode"
     rows_per_qc_histogram_grid: int = 12
+    clustermap_demux_types_to_plot: List[str] = field(
+        default_factory=lambda: ["single", "double", "already"]
+    )
 
     # Preprocessing - Read length and quality filter params
     read_coord_filter: Optional[Sequence[float]] = field(default_factory=lambda: [None, None])
@@ -816,6 +823,9 @@ class ExperimentConfig:
     duplicate_detection_site_types: List[str] = field(
         default_factory=lambda: ["GpC", "CpG", "ambiguous_GpC_CpG"]
     )
+    duplicate_detection_demux_types_to_use: List[str] = field(
+        default_factory=lambda: ["single", "double", "already"]
+    )
     duplicate_detection_distance_threshold: float = 0.07
     hamming_vs_metric_keys: List[str] = field(default_factory=lambda: ["Fraction_C_site_modified"])
     duplicate_detection_keep_best_metric: str = "read_quality"
@@ -827,6 +837,9 @@ class ExperimentConfig:
 
     # Preprocessing - Position QC
     position_max_nan_threshold: float = 0.1
+    mismatch_frequency_range: Sequence[float] = field(default_factory=lambda: [0.05, 0.95])
+    mismatch_frequency_layer: str = "mismatch_integer_encoding"
+    mismatch_frequency_read_span_layer: str = "read_span_mask"
 
     # Spatial Analysis - Clustermap params
     layer_for_clustermap_plotting: Optional[str] = "nan0_0minus1"
@@ -835,6 +848,14 @@ class ExperimentConfig:
     clustermap_cmap_cpg: Optional[str] = "coolwarm"
     clustermap_cmap_a: Optional[str] = "coolwarm"
     spatial_clustermap_sortby: Optional[str] = "gpc"
+    rolling_nn_layer: Optional[str] = "nan0_0minus1"
+    rolling_nn_plot_layer: Optional[str] = "nan0_0minus1"
+    rolling_nn_window: int = 15
+    rolling_nn_step: int = 2
+    rolling_nn_min_overlap: int = 10
+    rolling_nn_return_fraction: bool = True
+    rolling_nn_obsm_key: str = "rolling_nn_dist"
+    rolling_nn_site_types: Optional[List[str]] = None
 
     # Spatial Analysis - UMAP/Leiden params
     layer_for_umap_plotting: Optional[str] = "nan_half"
@@ -883,11 +904,15 @@ class ExperimentConfig:
     accessible_patches: Optional[bool] = True
     cpg: Optional[bool] = False
     hmm_feature_sets: Dict[str, Any] = field(default_factory=dict)
+    hmm_feature_colormaps: Dict[str, Any] = field(default_factory=dict)
     hmm_merge_layer_features: Optional[List[Tuple]] = field(default_factory=lambda: [(None, 60)])
     clustermap_cmap_hmm: Optional[str] = "coolwarm"
     hmm_clustermap_feature_layers: List[str] = field(
         default_factory=lambda: ["all_accessible_features"]
     )
+    hmm_clustermap_length_layers: List[str] = field(
+        default_factory=lambda: ["all_accessible_features"]
+    )
     hmm_clustermap_sortby: Optional[str] = "hmm"
     hmm_peak_feature_configs: Dict[str, Any] = field(default_factory=dict)
 
@@ -906,6 +931,8 @@ class ExperimentConfig:
     invert_adata: bool = False
     bypass_append_binary_layer_by_base_context: bool = False
     force_redo_append_binary_layer_by_base_context: bool = False
+    bypass_append_mismatch_frequency_sites: bool = False
+    force_redo_append_mismatch_frequency_sites: bool = False
     bypass_calculate_read_modification_stats: bool = False
     force_redo_calculate_read_modification_stats: bool = False
     bypass_filter_reads_on_modification_thresholds: bool = False
@@ -1110,7 +1137,7 @@ class ExperimentConfig:
 
         # Demultiplexing output path
         split_dir = merged.get("split_dir", SPLIT_DIR)
-        split_path = output_dir / split_dir
+        split_path = output_dir / LOAD_DIR / split_dir
 
         # final normalization
         if "strands" in merged:
@@ -1197,6 +1224,9 @@ class ExperimentConfig:
         # Final normalization of hmm_feature_sets and canonical local variables
         merged["hmm_feature_sets"] = normalize_hmm_feature_sets(merged.get("hmm_feature_sets", {}))
         hmm_feature_sets = merged.get("hmm_feature_sets", {})
+        hmm_feature_colormaps = merged.get("hmm_feature_colormaps", {})
+        if not isinstance(hmm_feature_colormaps, dict):
+            hmm_feature_colormaps = {}
         hmm_annotation_threshold = merged.get("hmm_annotation_threshold", 0.5)
         hmm_batch_size = int(merged.get("hmm_batch_size", 1024))
         hmm_use_viterbi = bool(merged.get("hmm_use_viterbi", False))
@@ -1211,6 +1241,9 @@ class ExperimentConfig:
         hmm_clustermap_feature_layers = _parse_list(
             merged.get("hmm_clustermap_feature_layers", "all_accessible_features")
         )
+        hmm_clustermap_length_layers = _parse_list(
+            merged.get("hmm_clustermap_length_layers", hmm_clustermap_feature_layers)
+        )
 
         hmm_fit_strategy = str(merged.get("hmm_fit_strategy", "per_group")).strip()
         hmm_shared_scope = _parse_list(merged.get("hmm_shared_scope", ["reference", "methbase"]))
@@ -1231,6 +1264,7 @@ class ExperimentConfig:
 
         # instantiate dataclass
         instance = cls(
+            annotate_secondary_supplementary=merged.get("annotate_secondary_supplementary", True),
             smf_modality=merged.get("smf_modality"),
             input_data_path=input_data_path,
             recursive_input_search=merged.get("recursive_input_search"),
@@ -1257,6 +1291,8 @@ class ExperimentConfig:
             trim=merged.get("trim", TRIM),
             input_already_demuxed=merged.get("input_already_demuxed", False),
             threads=merged.get("threads"),
+            emit_log_file=merged.get("emit_log_file", True),
+            log_level=merged.get("log_level", "INFO"),
             sample_sheet_path=merged.get("sample_sheet_path"),
             sample_sheet_mapping_column=merged.get("sample_sheet_mapping_column"),
             delete_intermediate_bams=merged.get("delete_intermediate_bams", False),
@@ -1313,6 +1349,9 @@ class ExperimentConfig:
             ),
             reindexing_offsets=merged.get("reindexing_offsets", {None: None}),
             reindexed_var_suffix=merged.get("reindexed_var_suffix", "reindexed"),
+            clustermap_demux_types_to_plot=merged.get(
+                "clustermap_demux_types_to_plot", ["single", "double", "already"]
+            ),
             layer_for_clustermap_plotting=merged.get(
                 "layer_for_clustermap_plotting", "nan0_0minus1"
             ),
@@ -1321,6 +1360,14 @@ class ExperimentConfig:
             clustermap_cmap_cpg=merged.get("clustermap_cmap_cpg", "coolwarm"),
             clustermap_cmap_a=merged.get("clustermap_cmap_a", "coolwarm"),
             spatial_clustermap_sortby=merged.get("spatial_clustermap_sortby", "gpc"),
+            rolling_nn_layer=merged.get("rolling_nn_layer", "nan0_0minus1"),
+            rolling_nn_plot_layer=merged.get("rolling_nn_plot_layer", "nan0_0minus1"),
+            rolling_nn_window=merged.get("rolling_nn_window", 15),
+            rolling_nn_step=merged.get("rolling_nn_step", 2),
+            rolling_nn_min_overlap=merged.get("rolling_nn_min_overlap", 10),
+            rolling_nn_return_fraction=merged.get("rolling_nn_return_fraction", True),
+            rolling_nn_obsm_key=merged.get("rolling_nn_obsm_key", "rolling_nn_dist"),
+            rolling_nn_site_types=merged.get("rolling_nn_site_types", None),
             layer_for_umap_plotting=merged.get("layer_for_umap_plotting", "nan_half"),
             umap_layers_to_plot=merged.get(
                 "umap_layers_to_plot", ["mapped_length", "Raw_modification_signal"]
@@ -1347,6 +1394,7 @@ class ExperimentConfig:
             hmm_emission_adapt_tol=hmm_emission_adapt_tol,
             hmm_dtype=merged.get("hmm_dtype", "float64"),
             hmm_feature_sets=hmm_feature_sets,
+            hmm_feature_colormaps=hmm_feature_colormaps,
             hmm_annotation_threshold=hmm_annotation_threshold,
             hmm_batch_size=hmm_batch_size,
             hmm_use_viterbi=hmm_use_viterbi,
@@ -1355,6 +1403,7 @@ class ExperimentConfig:
             hmm_merge_layer_features=hmm_merge_layer_features,
             clustermap_cmap_hmm=merged.get("clustermap_cmap_hmm", "coolwarm"),
             hmm_clustermap_feature_layers=hmm_clustermap_feature_layers,
+            hmm_clustermap_length_layers=hmm_clustermap_length_layers,
             hmm_clustermap_sortby=merged.get("hmm_clustermap_sortby", "hmm"),
             hmm_peak_feature_configs=hmm_peak_feature_configs,
             footprints=merged.get("footprints", None),
@@ -1390,6 +1439,9 @@ class ExperimentConfig:
             duplicate_detection_site_types=merged.get(
                 "duplicate_detection_site_types", ["GpC", "CpG", "ambiguous_GpC_CpG"]
             ),
+            duplicate_detection_demux_types_to_use=merged.get(
+                "duplicate_detection_demux_types_to_use", ["single", "double", "already"]
+            ),
             duplicate_detection_distance_threshold=merged.get(
                 "duplicate_detection_distance_threshold", 0.07
             ),

smftools/constants.py

@@ -21,7 +21,30 @@ BAM_SUFFIX: Final[str] = ".bam"
 BARCODE_BOTH_ENDS: Final[bool] = False
 REF_COL: Final[str] = "Reference_strand"
 SAMPLE_COL: Final[str] = "Experiment_name_and_barcode"
+SAMPLE: Final[str] = "Sample"
 SPLIT_DIR: Final[str] = "demultiplexed_BAMs"
+H5_DIR: Final[str] = "h5ads"
+DEMUX_TYPE: Final[str] = "demux_type"
+BARCODE: Final[str] = "Barcode"
+REFERENCE: Final[str] = "Reference"
+REFERENCE_STRAND: Final[str] = "Reference_strand"
+REFERENCE_DATASET_STRAND: Final[str] = "Reference_dataset_strand"
+STRAND: Final[str] = "Strand"
+DATASET: Final[str] = "Dataset"
+READ_MISMATCH_TREND: Final[str] = "Read_mismatch_trend"
+READ_MAPPING_DIRECTION: Final[str] = "Read_mapping_direction"
+SEQUENCE_INTEGER_ENCODING: Final[str] = "sequence_integer_encoding"
+SEQUENCE_INTEGER_DECODING: Final[str] = "sequence_integer_decoding"
+MISMATCH_INTEGER_ENCODING: Final[str] = "mismatch_integer_encoding"
+BASE_QUALITY_SCORES: Final[str] = "base_quality_scores"
+READ_SPAN_MASK: Final[str] = "read_span_mask"
+
+LOAD_DIR: Final[str] = "load_adata_outputs"
+PREPROCESS_DIR: Final[str] = "preprocess_adata_outputs"
+SPATIAL_DIR: Final[str] = "spatial_adata_outputs"
+HMM_DIR: Final[str] = "hmm_adata_outputs"
+LATENT_DIR: Final[str] = "latent_adata_outputs"
+LOGGING_DIR: Final[str] = "logs"
 TRIM: Final[bool] = False
 
 _private_conversions = ["unconverted"]
@@ -35,3 +58,45 @@ MOD_MAP: Final[Mapping[str, str]] = _deep_freeze(_private_mod_map)
 
 _private_strands = ("bottom", "top")
 STRANDS: Final[tuple[str, ...]] = _deep_freeze(_private_strands)
+
+MODKIT_EXTRACT_TSV_COLUMN_CHROM: Final[str] = "chrom"
+MODKIT_EXTRACT_TSV_COLUMN_REF_POSITION: Final[str] = "ref_position"
+MODKIT_EXTRACT_TSV_COLUMN_MODIFIED_PRIMARY_BASE: Final[str] = "modified_primary_base"
+MODKIT_EXTRACT_TSV_COLUMN_REF_STRAND: Final[str] = "ref_strand"
+MODKIT_EXTRACT_TSV_COLUMN_READ_ID: Final[str] = "read_id"
+MODKIT_EXTRACT_TSV_COLUMN_CALL_CODE: Final[str] = "call_code"
+MODKIT_EXTRACT_TSV_COLUMN_CALL_PROB: Final[str] = "call_prob"
+
+MODKIT_EXTRACT_MODIFIED_BASE_A: Final[str] = "A"
+MODKIT_EXTRACT_MODIFIED_BASE_C: Final[str] = "C"
+MODKIT_EXTRACT_REF_STRAND_PLUS: Final[str] = "+"
+MODKIT_EXTRACT_REF_STRAND_MINUS: Final[str] = "-"
+
+_private_modkit_extract_call_code_modified = ("a", "h", "m")
+MODKIT_EXTRACT_CALL_CODE_MODIFIED: Final[tuple[str, ...]] = _deep_freeze(
+    _private_modkit_extract_call_code_modified
+)
+_private_modkit_extract_call_code_canonical = ("-",)
+MODKIT_EXTRACT_CALL_CODE_CANONICAL: Final[tuple[str, ...]] = _deep_freeze(
+    _private_modkit_extract_call_code_canonical
+)
+
+MODKIT_EXTRACT_SEQUENCE_BASES: Final[tuple[str, ...]] = _deep_freeze(("A", "C", "G", "T", "N"))
+MODKIT_EXTRACT_SEQUENCE_PADDING_BASE: Final[str] = "PAD"
+_private_modkit_extract_base_to_int: Dict[str, int] = {
+    "A": 0,
+    "C": 1,
+    "G": 2,
+    "T": 3,
+    "N": 4,
+    "PAD": 5,
+}
+MODKIT_EXTRACT_SEQUENCE_BASE_TO_INT: Final[Mapping[str, int]] = _deep_freeze(
+    _private_modkit_extract_base_to_int
+)
+_private_modkit_extract_int_to_base: Dict[int, str] = {
+    value: key for key, value in _private_modkit_extract_base_to_int.items()
+}
+MODKIT_EXTRACT_SEQUENCE_INT_TO_BASE: Final[Mapping[int, str]] = _deep_freeze(
+    _private_modkit_extract_int_to_base
+)

smftools/hmm/HMM.py

@@ -144,6 +144,83 @@ def _safe_int_coords(var_names) -> Tuple[np.ndarray, bool]:
         return np.arange(len(var_names), dtype=int), False
 
 
+def mask_layers_outside_read_span(
+    adata,
+    layers: Sequence[str],
+    *,
+    start_key: str = "reference_start",
+    end_key: str = "reference_end",
+    use_original_var_names: bool = True,
+) -> List[str]:
+    """Mask layer values outside read reference spans with NaN.
+
+    This uses integer coordinate comparisons against either ``adata.var["Original_var_names"]``
+    (when present) or ``adata.var_names``. Values strictly less than ``start_key`` or greater
+    than ``end_key`` are set to NaN for each read.
+
+    Args:
+        adata: AnnData object to modify in-place.
+        layers: Layer names to mask.
+        start_key: obs column holding reference start positions.
+        end_key: obs column holding reference end positions.
+        use_original_var_names: Use ``adata.var["Original_var_names"]`` when available.
+
+    Returns:
+        List of layer names that were masked.
+    """
+    if not layers:
+        return []
+
+    if start_key not in adata.obs or end_key not in adata.obs:
+        raise KeyError(f"Missing {start_key!r} or {end_key!r} in adata.obs.")
+
+    coord_source = adata.var_names
+    if use_original_var_names and "Original_var_names" in adata.var:
+        orig = np.asarray(adata.var["Original_var_names"])
+        if orig.size == adata.n_vars:
+            try:
+                orig_numeric = np.asarray(orig, dtype=float)
+            except (TypeError, ValueError):
+                orig_numeric = None
+            if orig_numeric is not None and np.isfinite(orig_numeric).any():
+                coord_source = orig
+
+    coords, _ = _safe_int_coords(coord_source)
+    if coords.shape[0] != adata.n_vars:
+        raise ValueError("Coordinate source length does not match adata.n_vars.")
+
+    try:
+        starts = np.asarray(adata.obs[start_key], dtype=float)
+        ends = np.asarray(adata.obs[end_key], dtype=float)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("Start/end positions must be numeric.") from exc
+
+    masked = []
+    for layer in layers:
+        if layer not in adata.layers:
+            raise KeyError(f"Layer {layer!r} not found in adata.layers.")
+
+        arr = np.asarray(adata.layers[layer])
+        if not np.issubdtype(arr.dtype, np.floating):
+            arr = arr.astype(float, copy=True)
+
+        for i in range(adata.n_obs):
+            start = starts[i]
+            end = ends[i]
+            if not np.isfinite(start) or not np.isfinite(end):
+                continue
+            start_i = int(start)
+            end_i = int(end)
+            row_mask = (coords < start_i) | (coords > end_i)
+            if row_mask.any():
+                arr[i, row_mask] = np.nan
+
+        adata.layers[layer] = arr
+        masked.append(layer)
+
+    return masked
+
+
 def _logsumexp(x: torch.Tensor, dim: int) -> torch.Tensor:
     """Compute log-sum-exp in a numerically stable way.
 
@@ -1064,6 +1141,8 @@ class BaseHMM(nn.Module):
         uns_key: str = "hmm_appended_layers",
         uns_flag: str = "hmm_annotated",
         force_redo: bool = False,
+        mask_to_read_span: bool = True,
+        mask_use_original_var_names: bool = True,
         device: Optional[Union[str, torch.device]] = None,
         **kwargs,
     ):
@@ -1085,6 +1164,8 @@ class BaseHMM(nn.Module):
             uns_key: .uns key to track appended layers.
             uns_flag: .uns flag to mark annotations.
             force_redo: Whether to overwrite existing layers.
+            mask_to_read_span: Whether to mask appended layers outside read spans.
+            mask_use_original_var_names: Use ``adata.var["Original_var_names"]`` when available.
             device: Device specifier.
             **kwargs: Additional parameters for specialized workflows.
 
@@ -1245,6 +1326,13 @@ class BaseHMM(nn.Module):
                     np.asarray(adata.layers[nm])
                 )
 
+        if mask_to_read_span and appended:
+            mask_layers_outside_read_span(
+                adata,
+                appended,
+                use_original_var_names=mask_use_original_var_names,
+            )
+
         adata.uns[uns_key] = appended
         adata.uns[uns_flag] = True
         return None

smftools/informatics/__init__.py

@@ -6,6 +6,7 @@ _LAZY_ATTRS = {
     "_bed_to_bigwig": "smftools.informatics.bed_functions",
     "_plot_bed_histograms": "smftools.informatics.bed_functions",
     "add_demux_type_annotation": "smftools.informatics.h5ad_functions",
+    "add_read_tag_annotations": "smftools.informatics.h5ad_functions",
     "add_read_length_and_mapping_qc": "smftools.informatics.h5ad_functions",
     "align_and_sort_BAM": "smftools.informatics.bam_functions",
     "bam_qc": "smftools.informatics.bam_functions",
@@ -18,6 +19,7 @@ _LAZY_ATTRS = {
     "extract_base_identities": "smftools.informatics.bam_functions",
     "extract_mods": "smftools.informatics.modkit_functions",
     "extract_read_features_from_bam": "smftools.informatics.bam_functions",
+    "extract_read_tags_from_bam": "smftools.informatics.bam_functions",
     "extract_read_lengths_from_bed": "smftools.informatics.bed_functions",
     "extract_readnames_from_bam": "smftools.informatics.bam_functions",
     "fast5_to_pod5": "smftools.informatics.pod5_functions",
@@ -30,6 +32,8 @@ _LAZY_ATTRS = {
     "modQC": "smftools.informatics.modkit_functions",
     "modcall": "smftools.informatics.basecalling",
     "modkit_extract_to_adata": "smftools.informatics.modkit_extract_to_adata",
+    "decode_int_sequence": "smftools.informatics.sequence_encoding",
+    "encode_sequence_to_int": "smftools.informatics.sequence_encoding",
     "ohe_batching": "smftools.informatics.ohe",
     "ohe_layers_decode": "smftools.informatics.ohe",
     "one_hot_decode": "smftools.informatics.ohe",
@@ -55,6 +59,8 @@ def __getattr__(name: str):
 __all__ = [
     "basecall_pod5s",
     "converted_BAM_to_adata",
+    "decode_int_sequence",
+    "encode_sequence_to_int",
     "subsample_fasta_from_bed",
     "subsample_pod5",
     "fast5_to_pod5",