smftools 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

smftools/schema/__init__.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from importlib import resources
+from pathlib import Path
+
+SCHEMA_REGISTRY_VERSION = "1"
+SCHEMA_REGISTRY_RESOURCE = "anndata_schema_v1.yaml"
+
+
+def get_schema_registry_path() -> Path:
+    return resources.files(__package__).joinpath(SCHEMA_REGISTRY_RESOURCE)

smftools/schema/anndata_schema_v1.yaml

@@ -0,0 +1,227 @@
+schema_version: "1"
+description: "smftools AnnData schema registry (v1)."
+stages:
+  raw:
+    stage_requires: []
+    obs:
+      Experiment_name:
+        dtype: "category"
+        created_by: "smftools.cli.load_adata"
+        modified_by: []
+        notes: "Experiment identifier applied to all reads."
+        requires: []
+        optional_inputs: []
+      Experiment_name_and_barcode:
+        dtype: "category"
+        created_by: "smftools.cli.load_adata"
+        modified_by: []
+        notes: "Concatenated experiment name and barcode."
+        requires: [["obs.Experiment_name", "obs.Barcode"]]
+        optional_inputs: []
+      Barcode:
+        dtype: "category"
+        created_by: "smftools.informatics.modkit_extract_to_adata"
+        modified_by: []
+        notes: "Barcode assigned during demultiplexing or extraction."
+        requires: []
+        optional_inputs: []
+      read_length:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "Read length in bases."
+        requires: []
+        optional_inputs: []
+      mapped_length:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "Aligned length in bases."
+        requires: []
+        optional_inputs: []
+      reference_length:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "Reference length for alignment target."
+        requires: []
+        optional_inputs: []
+      read_quality:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "Per-read quality score."
+        requires: []
+        optional_inputs: []
+      mapping_quality:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "Mapping quality score."
+        requires: []
+        optional_inputs: []
+      read_length_to_reference_length_ratio:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "Read length divided by reference length."
+        requires: [["obs.read_length", "obs.reference_length"]]
+        optional_inputs: []
+      mapped_length_to_reference_length_ratio:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "Mapped length divided by reference length."
+        requires: [["obs.mapped_length", "obs.reference_length"]]
+        optional_inputs: []
+      mapped_length_to_read_length_ratio:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "Mapped length divided by read length."
+        requires: [["obs.mapped_length", "obs.read_length"]]
+        optional_inputs: []
+      Raw_modification_signal:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by:
+          - "smftools.cli.load_adata"
+        notes: "Summed modification signal per read."
+        requires: [["X"], ["layers.raw_mods"]]
+        optional_inputs: []
+      pod5_origin:
+        dtype: "string"
+        created_by: "smftools.informatics.h5ad_functions.annotate_pod5_origin"
+        modified_by: []
+        notes: "POD5 filename source for each read."
+        requires: [["obs_names"]]
+        optional_inputs: []
+      demux_type:
+        dtype: "category"
+        created_by: "smftools.informatics.h5ad_functions.add_demux_type_annotation"
+        modified_by: []
+        notes: "Classification of demultiplexing status."
+        requires: [["obs_names"]]
+        optional_inputs: []
+    var:
+      reference_position:
+        dtype: "int"
+        created_by: "smftools.informatics.modkit_extract_to_adata"
+        modified_by: []
+        notes: "Reference coordinate for each column."
+        requires: []
+        optional_inputs: []
+      reference_id:
+        dtype: "category"
+        created_by: "smftools.informatics.modkit_extract_to_adata"
+        modified_by: []
+        notes: "Reference contig or sequence name."
+        requires: []
+        optional_inputs: []
+    layers:
+      raw_mods:
+        dtype: "float"
+        created_by: "smftools.informatics.modkit_extract_to_adata"
+        modified_by: []
+        notes: "Raw modification scores (modality-dependent)."
+        requires: []
+        optional_inputs: []
+    obsm: {}
+    varm: {}
+    obsp: {}
+    uns:
+      smftools:
+        dtype: "mapping"
+        created_by: "smftools.metadata.record_smftools_metadata"
+        modified_by: []
+        notes: "smftools metadata including history, environment, provenance, schema snapshot."
+        requires: []
+        optional_inputs: []
+  preprocess:
+    stage_requires: ["raw"]
+    obs:
+      sequence__merged_cluster_id:
+        dtype: "category"
+        created_by: "smftools.preprocessing.flag_duplicate_reads"
+        modified_by: []
+        notes: "Cluster identifier for duplicate detection."
+        requires: [["layers.nan0_0minus1"]]
+        optional_inputs: ["obs.demux_type"]
+    layers:
+      nan0_0minus1:
+        dtype: "float"
+        created_by: "smftools.preprocessing.binarize"
+        modified_by:
+          - "smftools.preprocessing.clean_NaN"
+        notes: "Binarized methylation matrix (nan=0, 0=-1)."
+        requires: [["X"]]
+        optional_inputs: []
+    obsm:
+      X_umap:
+        dtype: "float"
+        created_by: "smftools.tools.calculate_umap"
+        modified_by: []
+        notes: "UMAP embedding for preprocessed reads."
+        requires: [["X"]]
+        optional_inputs: []
+    varm: {}
+    obsp: {}
+    uns:
+      duplicate_read_groups:
+        dtype: "mapping"
+        created_by: "smftools.preprocessing.flag_duplicate_reads"
+        modified_by: []
+        notes: "Duplicate read group metadata."
+        requires: [["obs.sequence__merged_cluster_id"]]
+        optional_inputs: []
+  spatial:
+    stage_requires: ["raw", "preprocess"]
+    obs:
+      leiden:
+        dtype: "category"
+        created_by: "smftools.tools.calculate_umap"
+        modified_by: []
+        notes: "Leiden cluster assignments."
+        requires: [["obsm.X_umap"]]
+        optional_inputs: []
+    obsm:
+      X_umap:
+        dtype: "float"
+        created_by: "smftools.tools.calculate_umap"
+        modified_by: []
+        notes: "UMAP embedding for spatial analyses."
+        requires: [["X"]]
+        optional_inputs: []
+    layers: {}
+    varm: {}
+    obsp: {}
+    uns:
+      positionwise_result:
+        dtype: "mapping"
+        created_by: "smftools.tools.position_stats.compute_positionwise_statistics"
+        modified_by: []
+        notes: "Positionwise correlation statistics for spatial analyses."
+        requires: [["X"]]
+        optional_inputs: ["obs.reference_column"]
+  hmm:
+    stage_requires: ["raw", "preprocess", "spatial"]
+    layers:
+      hmm_state_calls:
+        dtype: "int"
+        created_by: "smftools.hmm.call_hmm_peaks"
+        modified_by: []
+        notes: "HMM-derived state calls per read/position."
+        requires: [["layers.nan0_0minus1"]]
+        optional_inputs: []
+    obsm: {}
+    varm: {}
+    obsp: {}
+    obs: {}
+    uns:
+      hmm_annotated:
+        dtype: "bool"
+        created_by: "smftools.cli.hmm_adata"
+        modified_by: []
+        notes: "Flag indicating HMM annotations are present."
+        requires: [["layers.hmm_state_calls"]]
+        optional_inputs: []

smftools/tools/__init__.py

@@ -1,20 +1,27 @@
-from .position_stats import calculate_relative_risk_on_activity, compute_positionwise_statistics
-from .calculate_umap import calculate_umap
-from .cluster_adata_on_methylation import cluster_adata_on_methylation
-from .general_tools import create_nan_mask_from_X, combine_layers, create_nan_or_non_gpc_mask
-from .read_stats import calculate_row_entropy
-from .spatial_autocorrelation import *
-from .subset_adata import subset_adata
+from __future__ import annotations
 
+from importlib import import_module
 
-__all__ = [
-    "compute_positionwise_statistics",
-    "calculate_row_entropy",
-    "calculate_umap",
-    "calculate_relative_risk_on_activity",
-    "cluster_adata_on_methylation",
-    "create_nan_mask_from_X",
-    "create_nan_or_non_gpc_mask",
-    "combine_layers",
-    "subset_adata",
-]
+_LAZY_ATTRS = {
+    "calculate_umap": "smftools.tools.calculate_umap",
+    "cluster_adata_on_methylation": "smftools.tools.cluster_adata_on_methylation",
+    "combine_layers": "smftools.tools.general_tools",
+    "create_nan_mask_from_X": "smftools.tools.general_tools",
+    "create_nan_or_non_gpc_mask": "smftools.tools.general_tools",
+    "calculate_relative_risk_on_activity": "smftools.tools.position_stats",
+    "compute_positionwise_statistics": "smftools.tools.position_stats",
+    "calculate_row_entropy": "smftools.tools.read_stats",
+    "subset_adata": "smftools.tools.subset_adata",
+}
+
+
+def __getattr__(name: str):
+    if name in _LAZY_ATTRS:
+        module = import_module(_LAZY_ATTRS[name])
+        attr = getattr(module, name)
+        globals()[name] = attr
+        return attr
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = list(_LAZY_ATTRS.keys())

smftools/tools/archived/apply_hmm.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import numpy as np
 import pandas as pd
 import torch

smftools/tools/archived/classifiers.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 ## Train CNN, RNN, Random Forest models on double barcoded, low contamination datasets
 import torch
 import torch.nn as nn
@@ -21,13 +23,29 @@ device = (
 
 # ------------------------- Utilities -------------------------
 def random_fill_nans(X):
+    """Replace NaNs in an array with random values.
+
+    Args:
+        X: Input NumPy array.
+
+    Returns:
+        NumPy array with NaNs replaced.
+    """
     nan_mask = np.isnan(X)
     X[nan_mask] = np.random.rand(*X[nan_mask].shape)
     return X
 
 # ------------------------- Model Definitions -------------------------
 class CNNClassifier(nn.Module):
+    """Simple 1D CNN classifier for fixed-length inputs."""
+
     def __init__(self, input_size, num_classes):
+        """Initialize CNN classifier layers.
+
+        Args:
+            input_size: Length of the 1D input.
+            num_classes: Number of output classes.
+        """
         super().__init__()
         self.conv1 = nn.Conv1d(1, 16, kernel_size=3, padding=1)
         self.conv2 = nn.Conv1d(16, 32, kernel_size=3, padding=1)
@@ -39,11 +57,13 @@ class CNNClassifier(nn.Module):
         self.fc2 = nn.Linear(64, num_classes)
 
     def _forward_conv(self, x):
+        """Apply convolutional layers and activation."""
         x = self.relu(self.conv1(x))
         x = self.relu(self.conv2(x))
         return x
 
     def forward(self, x):
+        """Run the forward pass."""
         x = x.unsqueeze(1)
         x = self._forward_conv(x)
         x = x.view(x.size(0), -1)
@@ -51,7 +71,15 @@ class CNNClassifier(nn.Module):
         return self.fc2(x)
     
 class MLPClassifier(nn.Module):
+    """Simple MLP classifier."""
+
     def __init__(self, input_dim, num_classes):
+        """Initialize MLP layers.
+
+        Args:
+            input_dim: Input feature dimension.
+            num_classes: Number of output classes.
+        """
         super().__init__()
         self.model = nn.Sequential(
             nn.Linear(input_dim, 128),
@@ -64,10 +92,20 @@ class MLPClassifier(nn.Module):
         )
 
     def forward(self, x):
+        """Run the forward pass."""
         return self.model(x)
 
 class RNNClassifier(nn.Module):
+    """LSTM-based classifier for sequential inputs."""
+
     def __init__(self, input_size, hidden_dim, num_classes):
+        """Initialize RNN classifier layers.
+
+        Args:
+            input_size: Input feature dimension.
+            hidden_dim: Hidden state dimension.
+            num_classes: Number of output classes.
+        """
         super().__init__()
         # Define LSTM layer
         self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_dim, batch_first=True)
@@ -75,18 +113,29 @@ class RNNClassifier(nn.Module):
         self.fc = nn.Linear(hidden_dim, num_classes)
 
     def forward(self, x):
+        """Run the forward pass."""
         x = x.unsqueeze(1)
         _, (h_n, _) = self.lstm(x)
         return self.fc(h_n.squeeze(0))
     
 class AttentionRNNClassifier(nn.Module):
+    """LSTM classifier with simple attention."""
+
     def __init__(self, input_size, hidden_dim, num_classes):
+        """Initialize attention-based RNN layers.
+
+        Args:
+            input_size: Input feature dimension.
+            hidden_dim: Hidden state dimension.
+            num_classes: Number of output classes.
+        """
         super().__init__()
         self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_dim, batch_first=True)
         self.attn = nn.Linear(hidden_dim, 1)  # Simple attention scores
         self.fc = nn.Linear(hidden_dim, num_classes)
 
     def forward(self, x):
+        """Run the forward pass."""
         x = x.unsqueeze(1)  # shape: (batch, 1, seq_len)
         lstm_out, _ = self.lstm(x)  # shape: (batch, 1, hidden_dim)
         attn_weights = torch.softmax(self.attn(lstm_out), dim=1)  # (batch, 1, 1)
@@ -94,7 +143,15 @@ class AttentionRNNClassifier(nn.Module):
         return self.fc(context)
     
 class PositionalEncoding(nn.Module):
+    """Positional encoding module for transformer models."""
+
     def __init__(self, d_model, max_len=5000):
+        """Initialize positional encoding buffer.
+
+        Args:
+            d_model: Model embedding dimension.
+            max_len: Maximum sequence length.
+        """
         super().__init__()
         pe = torch.zeros(max_len, d_model)
         position = torch.arange(0, max_len).unsqueeze(1).float()
@@ -104,11 +161,23 @@ class PositionalEncoding(nn.Module):
         self.pe = pe.unsqueeze(0)  # (1, max_len, d_model)
 
     def forward(self, x):
+        """Add positional encoding to inputs."""
         x = x + self.pe[:, :x.size(1)].to(x.device)
         return x
     
 class TransformerClassifier(nn.Module):
+    """Transformer encoder-based classifier."""
+
     def __init__(self, input_dim, model_dim, num_classes, num_heads=4, num_layers=2):
+        """Initialize transformer classifier layers.
+
+        Args:
+            input_dim: Input feature dimension.
+            model_dim: Transformer model dimension.
+            num_classes: Number of output classes.
+            num_heads: Number of attention heads.
+            num_layers: Number of encoder layers.
+        """
         super().__init__()
         self.input_fc = nn.Linear(input_dim, model_dim)
         self.pos_encoder = PositionalEncoding(model_dim)
@@ -119,6 +188,7 @@ class TransformerClassifier(nn.Module):
         self.cls_head = nn.Linear(model_dim, num_classes)
 
     def forward(self, x):
+        """Run the forward pass."""
         # x: [batch_size, input_dim]
         x = self.input_fc(x).unsqueeze(1)  # -> [batch_size, 1, model_dim]
         x = self.pos_encoder(x)
@@ -128,6 +198,19 @@ class TransformerClassifier(nn.Module):
         return self.cls_head(pooled)
 
 def train_model(model, loader, optimizer, criterion, device, ref_name="", model_name="", epochs=20, patience=5):
+    """Train a model with early stopping.
+
+    Args:
+        model: PyTorch model.
+        loader: DataLoader for training data.
+        optimizer: Optimizer instance.
+        criterion: Loss function.
+        device: Torch device.
+        ref_name: Reference label for logging.
+        model_name: Model label for logging.
+        epochs: Maximum epochs.
+        patience: Early-stopping patience.
+    """
     model.train()
     best_loss = float('inf')
     trigger_times = 0
@@ -154,6 +237,17 @@ def train_model(model, loader, optimizer, criterion, device, ref_name="", model_
                 break
 
 def evaluate_model(model, X_tensor, y_encoded, device):
+    """Evaluate a trained model and compute metrics.
+
+    Args:
+        model: Trained model.
+        X_tensor: Input tensor.
+        y_encoded: Encoded labels.
+        device: Torch device.
+
+    Returns:
+        Tuple of metrics dict, predicted labels, and probabilities.
+    """
     model.eval()
     with torch.no_grad():
         outputs = model(X_tensor.to(device))
@@ -176,6 +270,18 @@ def evaluate_model(model, X_tensor, y_encoded, device):
     }, preds, probs
 
 def train_rf(X_tensor, y_tensor, train_indices, test_indices, n_estimators=500):
+    """Train a random forest classifier.
+
+    Args:
+        X_tensor: Input tensor.
+        y_tensor: Label tensor.
+        train_indices: Indices for training.
+        test_indices: Indices for testing.
+        n_estimators: Number of trees.
+
+    Returns:
+        Tuple of (model, preds, probs).
+    """
     model = RandomForestClassifier(n_estimators=n_estimators, random_state=42, class_weight='balanced')
     model.fit(X_tensor[train_indices].numpy(), y_tensor[train_indices].numpy())
     probs = model.predict_proba(X_tensor[test_indices].cpu().numpy())[:, 1]
@@ -186,6 +292,25 @@ def train_rf(X_tensor, y_tensor, train_indices, test_indices, n_estimators=500):
 def run_training_loop(adata, site_config, layer_name=None,
                        mlp=False, cnn=False, rnn=False, arnn=False, transformer=False, rf=False, nb=False, rr_bayes=False,
                        max_epochs=10, max_patience=5, n_estimators=500, training_split=0.5):
+    """Train one or more classifier types on AnnData.
+
+    Args:
+        adata: AnnData object containing data and labels.
+        site_config: Mapping of reference to site list.
+        layer_name: Optional layer to use as input.
+        mlp: Whether to train an MLP model.
+        cnn: Whether to train a CNN model.
+        rnn: Whether to train an RNN model.
+        arnn: Whether to train an attention RNN model.
+        transformer: Whether to train a transformer model.
+        rf: Whether to train a random forest model.
+        nb: Whether to train a Naive Bayes model.
+        rr_bayes: Whether to train a ridge regression model.
+        max_epochs: Maximum training epochs.
+        max_patience: Early stopping patience.
+        n_estimators: Random forest estimator count.
+        training_split: Fraction of data used for training.
+    """
     device = (
     torch.device('cuda') if torch.cuda.is_available() else
     torch.device('mps') if torch.backends.mps.is_available() else
@@ -701,6 +826,20 @@ def evaluate_model_by_subgroups(
     label_col="activity_status",
     min_samples=10,
     exclude_training_data=True):
+    """Evaluate predictions within categorical subgroups.
+
+    Args:
+        adata: AnnData with prediction columns.
+        model_prefix: Prediction column prefix.
+        suffix: Prediction column suffix.
+        groupby_cols: Columns to group by.
+        label_col: Ground-truth label column.
+        min_samples: Minimum samples per group.
+        exclude_training_data: Whether to exclude training rows.
+
+    Returns:
+        DataFrame of subgroup-level metrics.
+    """
     import pandas as pd
     from sklearn.metrics import accuracy_score, f1_score, roc_auc_score
 
@@ -745,6 +884,18 @@ def evaluate_model_by_subgroups(
     return pd.DataFrame(results)
 
 def evaluate_models_by_subgroup(adata, model_prefixes, groupby_cols, label_col, exclude_training_data=True):
+    """Evaluate multiple model prefixes across subgroups.
+
+    Args:
+        adata: AnnData with prediction columns.
+        model_prefixes: Iterable of model prefixes.
+        groupby_cols: Columns to group by.
+        label_col: Ground-truth label column.
+        exclude_training_data: Whether to exclude training rows.
+
+    Returns:
+        Concatenated DataFrame of subgroup-level metrics.
+    """
     import pandas as pd
     all_metrics = []
     for model_prefix in model_prefixes:
@@ -758,6 +909,20 @@ def evaluate_models_by_subgroup(adata, model_prefixes, groupby_cols, label_col,
     return final_df
 
 def prepare_melted_model_data(adata, outkey='melted_model_df', groupby=['Enhancer_Open', 'Promoter_Open'], label_col='activity_status', model_names = ['cnn', 'mlp', 'rf'], suffix='GpC_site_CpG_site', omit_training=True):
+    """Prepare a long-format DataFrame for model performance plots.
+
+    Args:
+        adata: AnnData with prediction columns.
+        outkey: Key to store the melted DataFrame in ``adata.uns``.
+        groupby: Grouping columns to include.
+        label_col: Ground-truth label column.
+        model_names: Model prefixes to include.
+        suffix: Prediction column suffix.
+        omit_training: Whether to exclude training rows.
+
+    Returns:
+        Melted DataFrame of predictions.
+    """
     import pandas as pd
     import seaborn as sns
     import matplotlib.pyplot as plt

smftools/tools/archived/classify_methylated_features.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # classify_methylated_features
 
 def classify_methylated_features(read, model, coordinates, classification_mapping={}):

smftools/tools/archived/classify_non_methylated_features.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # classify_non_methylated_features
 
 def classify_non_methylated_features(read, model, coordinates, classification_mapping={}):

smftools/tools/archived/subset_adata_v1.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # subset_adata
 
 def subset_adata(adata, obs_columns):
@@ -13,6 +15,15 @@ def subset_adata(adata, obs_columns):
     """
 
     def subset_recursive(adata_subset, columns):
+        """Recursively subset AnnData by categorical columns.
+
+        Args:
+            adata_subset: AnnData subset to split.
+            columns: Remaining columns to split on.
+
+        Returns:
+            Dictionary mapping category tuples to AnnData subsets.
+        """
         if not columns:
             return {(): adata_subset}
         
@@ -29,4 +40,4 @@ def subset_adata(adata, obs_columns):
     # Start the recursive subset process
     subsets_dict = subset_recursive(adata, obs_columns)
     
-    return subsets_dict
+    return subsets_dict

smftools/tools/archived/subset_adata_v2.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # subset_adata
 
 def subset_adata(adata, columns, cat_type='obs'):
@@ -14,6 +16,17 @@ def subset_adata(adata, columns, cat_type='obs'):
     """
 
     def subset_recursive(adata_subset, columns, cat_type, key_prefix=()):
+        """Recursively subset AnnData by categorical columns.
+
+        Args:
+            adata_subset: AnnData subset to split.
+            columns: Remaining columns to split on.
+            cat_type: Whether to use obs or var categories.
+            key_prefix: Tuple of previous category keys.
+
+        Returns:
+            Dictionary mapping category tuples to AnnData subsets.
+        """
         # Returns when the bottom of the stack is reached
         if not columns:
             # If there's only one column, return the key as a single value, not a tuple
@@ -43,4 +56,4 @@ def subset_adata(adata, columns, cat_type='obs'):
     # Start the recursive subset process
     subsets_dict = subset_recursive(adata, columns, cat_type)
     
-    return subsets_dict
+    return subsets_dict