smftools 0.2.5__py3-none-any.whl → 0.3.1__py3-none-any.whl

smftools/preprocessing/flag_duplicate_reads.py

@@ -1,51 +1,64 @@
+from __future__ import annotations
+
 # duplicate_detection_with_hier_and_plots.py
 import copy
 import math
 import os
 import warnings
 from collections import defaultdict
-from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
+from importlib.util import find_spec
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union
 
-import anndata as ad
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-import torch
+from scipy.cluster import hierarchy as sch
+from scipy.spatial.distance import pdist, squareform
+from scipy.stats import gaussian_kde
 
 from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
 
 from ..readwrite import make_dirs
 
 logger = get_logger(__name__)
 
-# optional imports for clustering / PCA / KDE
-try:
-    from scipy.cluster import hierarchy as sch
-    from scipy.spatial.distance import pdist, squareform
-
-    SCIPY_AVAILABLE = True
-except Exception:
-    sch = None
-    pdist = None
-    squareform = None
-    SCIPY_AVAILABLE = False
+plt = require("matplotlib.pyplot", extra="plotting", purpose="duplicate read plots")
+torch = require("torch", extra="torch", purpose="duplicate read detection")
 
-try:
-    from sklearn.cluster import DBSCAN, KMeans
-    from sklearn.decomposition import PCA
-    from sklearn.metrics import silhouette_score
-    from sklearn.mixture import GaussianMixture
+if TYPE_CHECKING:
+    import anndata as ad
 
-    SKLEARN_AVAILABLE = True
-except Exception:
-    PCA = None
-    KMeans = DBSCAN = GaussianMixture = silhouette_score = None
-    SKLEARN_AVAILABLE = False
+SCIPY_AVAILABLE = True
+SKLEARN_AVAILABLE = find_spec("sklearn") is not None
 
-try:
-    from scipy.stats import gaussian_kde
-except Exception:
-    gaussian_kde = None
+PCA = None
+KMeans = DBSCAN = GaussianMixture = silhouette_score = None
+if SKLEARN_AVAILABLE:
+    sklearn_cluster = require(
+        "sklearn.cluster",
+        extra="ml-base",
+        purpose="duplicate read clustering",
+    )
+    sklearn_decomp = require(
+        "sklearn.decomposition",
+        extra="ml-base",
+        purpose="duplicate read PCA",
+    )
+    sklearn_metrics = require(
+        "sklearn.metrics",
+        extra="ml-base",
+        purpose="duplicate read clustering diagnostics",
+    )
+    sklearn_mixture = require(
+        "sklearn.mixture",
+        extra="ml-base",
+        purpose="duplicate read clustering",
+    )
+    DBSCAN = sklearn_cluster.DBSCAN
+    KMeans = sklearn_cluster.KMeans
+    PCA = sklearn_decomp.PCA
+    silhouette_score = sklearn_metrics.silhouette_score
+    GaussianMixture = sklearn_mixture.GaussianMixture
 
 
 def merge_uns_preserve(orig_uns: dict, new_uns: dict, prefer: str = "orig") -> dict:
@@ -153,24 +166,6 @@ def flag_duplicate_reads(
     import numpy as np
     import pandas as pd
 
-    # optional imports already guarded at module import time, but re-check
-    try:
-        from scipy.cluster import hierarchy as sch
-        from scipy.spatial.distance import pdist
-
-        SCIPY_AVAILABLE = True
-    except Exception:
-        sch = None
-        pdist = None
-        SCIPY_AVAILABLE = False
-    try:
-        from sklearn.decomposition import PCA
-
-        SKLEARN_AVAILABLE = True
-    except Exception:
-        PCA = None
-        SKLEARN_AVAILABLE = False
-
     # -------- helper: demux-aware keeper selection --------
     def _choose_keeper_with_demux_preference(
         members_idx: List[int],
@@ -1577,13 +1572,6 @@ def _run_clustering(
     Run clustering on 2D points (x,y). Returns labels (len = npoints) and diagnostics dict.
     Labels follow sklearn conventions (noise -> -1 for DBSCAN/HDBSCAN).
     """
-    try:
-        from sklearn.cluster import DBSCAN, KMeans
-        from sklearn.metrics import silhouette_score
-        from sklearn.mixture import GaussianMixture
-    except Exception:
-        KMeans = DBSCAN = GaussianMixture = silhouette_score = None
-
     pts = np.column_stack([x, y])
     diagnostics: Dict[str, Any] = {"method": method, "n_input": len(x)}
     if len(x) < min_points:

smftools/preprocessing/make_dirs.py

@@ -1,5 +1,6 @@
-## make_dirs
+from __future__ import annotations
 
+## make_dirs
 from smftools.logging_utils import get_logger
 
 logger = get_logger(__name__)

smftools/preprocessing/min_non_diagonal.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 ## min_non_diagonal
 
 

smftools/preprocessing/recipes.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # recipes
 
 

smftools/readwrite.py

@@ -431,6 +431,8 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
         "layers_skipped": [],
         "obsm_converted": [],
         "obsm_skipped": [],
+        "varm_converted": [],
+        "varm_skipped": [],
         "X_replaced_or_converted": None,
         "errors": [],
     }
@@ -605,10 +607,16 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
 
     def _sanitize_layers_obsm(src_dict, which: str):
         """
-        Ensure arrays in layers/obsm are numeric and non-object dtype.
+        Ensure arrays in layers/obsm/varm are numeric and non-object dtype.
         Returns a cleaned dict suitable to pass into AnnData(...)
         If an entry is not convertible, it is backed up & skipped.
         """
+        report_map = {
+            "layers": ("layers_converted", "layers_skipped"),
+            "obsm": ("obsm_converted", "obsm_skipped"),
+            "varm": ("varm_converted", "varm_skipped"),
+        }
+        converted_key, skipped_key = report_map[which]
         cleaned = {}
         for k, v in src_dict.items():
             try:
@@ -618,9 +626,7 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
                         arr_f = arr.astype(float)
                         cleaned[k] = arr_f
                         report_key = f"{which}.{k}"
-                        report["layers_converted"].append(
-                            report_key
-                        ) if which == "layers" else report["obsm_converted"].append(report_key)
+                        report[converted_key].append(report_key)
                         if verbose:
                             print(f"  {which}.{k} object array coerced to float.")
                     except Exception:
@@ -628,18 +634,13 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
                             arr_i = arr.astype(int)
                             cleaned[k] = arr_i
                             report_key = f"{which}.{k}"
-                            report["layers_converted"].append(
-                                report_key
-                            ) if which == "layers" else report["obsm_converted"].append(report_key)
+                            report[converted_key].append(report_key)
                             if verbose:
                                 print(f"  {which}.{k} object array coerced to int.")
                         except Exception:
                             if backup:
                                 _backup(v, f"{which}_{k}_backup")
-                            if which == "layers":
-                                report["layers_skipped"].append(k)
-                            else:
-                                report["obsm_skipped"].append(k)
+                            report[skipped_key].append(k)
                             if verbose:
                                 print(
                                     f"  SKIPPING {which}.{k} (object dtype not numeric). Backed up: {backup}"
@@ -650,10 +651,7 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
             except Exception as e:
                 if backup:
                     _backup(v, f"{which}_{k}_backup")
-                if which == "layers":
-                    report["layers_skipped"].append(k)
-                else:
-                    report["obsm_skipped"].append(k)
+                report[skipped_key].append(k)
                 msg = f"  SKIPPING {which}.{k} due to conversion error: {e}"
                 report["errors"].append(msg)
                 if verbose:
@@ -693,6 +691,7 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
     # ---------- sanitize layers and obsm ----------
     layers_src = getattr(adata, "layers", {})
     obsm_src = getattr(adata, "obsm", {})
+    varm_src = getattr(adata, "varm", {})
 
     try:
         layers_clean = _sanitize_layers_obsm(layers_src, "layers")
@@ -712,6 +711,15 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
             print(msg)
         obsm_clean = {}
 
+    try:
+        varm_clean = _sanitize_layers_obsm(varm_src, "varm")
+    except Exception as e:
+        msg = f"Failed to sanitize varm: {e}"
+        report["errors"].append(msg)
+        if verbose:
+            print(msg)
+        varm_clean = {}
+
     # ---------- handle X ----------
     X_to_use = adata.X
     try:
@@ -747,7 +755,7 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
             layers=layers_clean,
             uns=uns_clean,
             obsm=obsm_clean,
-            varm=getattr(adata, "varm", None),
+            varm=varm_clean,
         )
 
         # preserve names (as strings)
@@ -872,6 +880,16 @@ def safe_write_h5ad(adata, path, compression="gzip", backup=False, backup_dir=No
                 }
             )
 
+        # varm
+        for k, v in adata_copy.varm.items():
+            meta_rows.append(
+                {
+                    "kind": "varm",
+                    "name": k,
+                    "dtype": str(np.asarray(v).dtype),
+                }
+            )
+
         # uns
         for k, v in adata_copy.uns.items():
             meta_rows.append(
@@ -977,6 +995,7 @@ def safe_read_h5ad(
         "parsed_uns_json_keys": [],
         "restored_layers": [],
         "restored_obsm": [],
+        "restored_varm": [],
         "recategorized_obs": [],
         "recategorized_var": [],
         "missing_backups": [],
@@ -1215,7 +1234,7 @@ def safe_read_h5ad(
                 print(f"[safe_read_h5ad] restored adata.uns['{key}'] from {full}")
 
     # 5) Restore layers and obsm from backups if present
-    # expected backup names: layers_<name>_backup.pkl, obsm_<name>_backup.pkl
+    # expected backup names: layers_<name>_backup.pkl, obsm_<name>_backup.pkl, varm_<name>_backup.pkl
     if os.path.isdir(backup_dir):
         for fname in os.listdir(backup_dir):
             if fname.startswith("layers_") and fname.endswith("_backup.pkl"):
@@ -1248,6 +1267,21 @@ def safe_read_h5ad(
                             f"Failed to restore obsm['{obsm_name}'] from {full}: {e}"
                         )
 
+            if fname.startswith("varm_") and fname.endswith("_backup.pkl"):
+                varm_name = fname[len("varm_") : -len("_backup.pkl")]
+                full = os.path.join(backup_dir, fname)
+                val = _load_pickle_if_exists(full)
+                if val is not None:
+                    try:
+                        adata.varm[varm_name] = np.asarray(val)
+                        report["restored_varm"].append((varm_name, full))
+                        if verbose:
+                            print(f"[safe_read_h5ad] restored varm['{varm_name}'] from {full}")
+                    except Exception as e:
+                        report["errors"].append(
+                            f"Failed to restore varm['{varm_name}'] from {full}: {e}"
+                        )
+
     # 6) If restore_backups True but some expected backups missing, note them
     if restore_backups and os.path.isdir(backup_dir):
         # detect common expected names from obs/var/uns/layers in adata
@@ -1297,6 +1331,8 @@ def safe_read_h5ad(
             print("Restored layers:", report["restored_layers"])
         if report["restored_obsm"]:
             print("Restored obsm:", report["restored_obsm"])
+        if report["restored_varm"]:
+            print("Restored varm:", report["restored_varm"])
         if report["recategorized_obs"] or report["recategorized_var"]:
             print(
                 "Recategorized columns (obs/var):",

smftools/schema/anndata_schema_v1.yaml

@@ -60,6 +60,20 @@ stages:
         notes: "Mapping quality score."
         requires: []
         optional_inputs: []
+      reference_start:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "0-based reference start position for the alignment."
+        requires: []
+        optional_inputs: []
+      reference_end:
+        dtype: "float"
+        created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
+        modified_by: []
+        notes: "0-based reference end position (exclusive) for the alignment."
+        requires: []
+        optional_inputs: []
       read_length_to_reference_length_ratio:
         dtype: "float"
         created_by: "smftools.informatics.h5ad_functions.add_read_length_and_mapping_qc"
@@ -179,7 +193,7 @@ stages:
     obs:
       leiden:
         dtype: "category"
-        created_by: "smftools.tools.calculate_umap"
+        created_by: "smftools.tools.calculate_leiden"
         modified_by: []
         notes: "Leiden cluster assignments."
         requires: [["obsm.X_umap"]]

smftools/tools/__init__.py

@@ -1,19 +1,31 @@
-from .calculate_umap import calculate_umap
-from .cluster_adata_on_methylation import cluster_adata_on_methylation
-from .general_tools import combine_layers, create_nan_mask_from_X, create_nan_or_non_gpc_mask
-from .position_stats import calculate_relative_risk_on_activity, compute_positionwise_statistics
-from .read_stats import calculate_row_entropy
-from .spatial_autocorrelation import *
-from .subset_adata import subset_adata
+from __future__ import annotations
 
-__all__ = [
-    "compute_positionwise_statistics",
-    "calculate_row_entropy",
-    "calculate_umap",
-    "calculate_relative_risk_on_activity",
-    "cluster_adata_on_methylation",
-    "create_nan_mask_from_X",
-    "create_nan_or_non_gpc_mask",
-    "combine_layers",
-    "subset_adata",
-]
+from importlib import import_module
+
+_LAZY_ATTRS = {
+    "calculate_leiden": "smftools.tools.calculate_leiden",
+    "calculate_nmf": "smftools.tools.calculate_nmf",
+    "calculate_sequence_cp_decomposition": "smftools.tools.tensor_factorization",
+    "calculate_umap": "smftools.tools.calculate_umap",
+    "cluster_adata_on_methylation": "smftools.tools.cluster_adata_on_methylation",
+    "combine_layers": "smftools.tools.general_tools",
+    "create_nan_mask_from_X": "smftools.tools.general_tools",
+    "create_nan_or_non_gpc_mask": "smftools.tools.general_tools",
+    "calculate_relative_risk_on_activity": "smftools.tools.position_stats",
+    "compute_positionwise_statistics": "smftools.tools.position_stats",
+    "calculate_row_entropy": "smftools.tools.read_stats",
+    "rolling_window_nn_distance": "smftools.tools.rolling_nn_distance",
+    "subset_adata": "smftools.tools.subset_adata",
+}
+
+
+def __getattr__(name: str):
+    if name in _LAZY_ATTRS:
+        module = import_module(_LAZY_ATTRS[name])
+        attr = getattr(module, name)
+        globals()[name] = attr
+        return attr
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = list(_LAZY_ATTRS.keys())

smftools/tools/archived/apply_hmm.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import numpy as np
 import pandas as pd
 import torch

smftools/tools/archived/classifiers.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 ## Train CNN, RNN, Random Forest models on double barcoded, low contamination datasets
 import torch
 import torch.nn as nn

smftools/tools/archived/classify_methylated_features.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # classify_methylated_features
 
 def classify_methylated_features(read, model, coordinates, classification_mapping={}):

smftools/tools/archived/classify_non_methylated_features.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # classify_non_methylated_features
 
 def classify_non_methylated_features(read, model, coordinates, classification_mapping={}):

smftools/tools/archived/subset_adata_v1.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # subset_adata
 
 def subset_adata(adata, obs_columns):

smftools/tools/archived/subset_adata_v2.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # subset_adata
 
 def subset_adata(adata, columns, cat_type='obs'):

smftools/tools/calculate_leiden.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pandas as pd
+
+from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
+
+if TYPE_CHECKING:
+    import anndata as ad
+
+logger = get_logger(__name__)
+
+
+def calculate_leiden(
+    adata: "ad.AnnData",
+    *,
+    resolution: float = 0.1,
+    key_added: str = "leiden",
+    connectivities_key: str = "connectivities",
+) -> "ad.AnnData":
+    """Compute Leiden clusters from a connectivity graph.
+
+    Args:
+        adata: AnnData object with ``obsp[connectivities_key]`` set.
+        resolution: Resolution parameter for Leiden clustering.
+        key_added: Column name to store cluster assignments in ``adata.obs``.
+        connectivities_key: Key in ``adata.obsp`` containing a sparse adjacency matrix.
+
+    Returns:
+        Updated AnnData object with Leiden labels in ``adata.obs``.
+    """
+    if connectivities_key not in adata.obsp:
+        raise KeyError(f"Missing connectivities '{connectivities_key}' in adata.obsp.")
+
+    igraph = require("igraph", extra="cluster", purpose="Leiden clustering")
+    leidenalg = require("leidenalg", extra="cluster", purpose="Leiden clustering")
+
+    connectivities = adata.obsp[connectivities_key]
+    coo = connectivities.tocoo()
+    edges = list(zip(coo.row.tolist(), coo.col.tolist()))
+    graph = igraph.Graph(n=connectivities.shape[0], edges=edges, directed=False)
+    graph.es["weight"] = coo.data.tolist()
+
+    partition = leidenalg.find_partition(
+        graph,
+        leidenalg.RBConfigurationVertexPartition,
+        weights=graph.es["weight"],
+        resolution_parameter=resolution,
+    )
+
+    labels = np.array(partition.membership, dtype=str)
+    adata.obs[key_added] = pd.Categorical(labels)
+    logger.info("Stored Leiden clusters in adata.obs['%s'].", key_added)
+    return adata

smftools/tools/calculate_nmf.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Sequence
+
+import numpy as np
+
+from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
+
+if TYPE_CHECKING:
+    import anndata as ad
+
+logger = get_logger(__name__)
+
+
+def calculate_nmf(
+    adata: "ad.AnnData",
+    layer: str | None = "nan_half",
+    var_filters: Sequence[str] | None = None,
+    n_components: int = 2,
+    max_iter: int = 200,
+    random_state: int = 0,
+    overwrite: bool = True,
+    embedding_key: str = "X_nmf",
+    components_key: str = "H_nmf",
+    uns_key: str = "nmf",
+) -> "ad.AnnData":
+    """Compute a low-dimensional NMF embedding.
+
+    Args:
+        adata: AnnData object to update.
+        layer: Layer name to use for NMF (``None`` uses ``adata.X``).
+        var_filters: Optional list of var masks to subset features.
+        n_components: Number of NMF components to compute.
+        max_iter: Maximum number of NMF iterations.
+        random_state: Random seed for the NMF initializer.
+        overwrite: Whether to recompute if the embedding already exists.
+        embedding_key: Key for the embedding in ``adata.obsm``.
+        components_key: Key for the components matrix in ``adata.varm``.
+        uns_key: Key for metadata stored in ``adata.uns``.
+
+    Returns:
+        anndata.AnnData: Updated AnnData object.
+    """
+    from scipy.sparse import issparse
+
+    require("sklearn", extra="ml-base", purpose="NMF calculation")
+    from sklearn.decomposition import NMF
+
+    has_embedding = embedding_key in adata.obsm
+    has_components = components_key in adata.varm
+    if has_embedding and has_components and not overwrite:
+        logger.info("NMF embedding and components already present; skipping recomputation.")
+        return adata
+    if has_embedding and not has_components and not overwrite:
+        logger.info("NMF embedding present without components; recomputing to store components.")
+
+    subset_mask = None
+    if var_filters:
+        subset_mask = np.logical_or.reduce([adata.var[f].values for f in var_filters])
+        adata_subset = adata[:, subset_mask].copy()
+        logger.info(
+            "Subsetting adata: retained %s features based on filters %s",
+            adata_subset.shape[1],
+            var_filters,
+        )
+    else:
+        adata_subset = adata.copy()
+        logger.info("No var filters provided. Using all features.")
+
+    data = adata_subset.layers[layer] if layer else adata_subset.X
+    if issparse(data):
+        data = data.copy()
+        if data.data.size and np.isnan(data.data).any():
+            logger.warning("NaNs detected in sparse data, filling with 0.5 before NMF.")
+            data.data = np.nan_to_num(data.data, nan=0.5)
+        if data.data.size and (data.data < 0).any():
+            logger.warning("Negative values detected in sparse data, clipping to 0 for NMF.")
+            data.data[data.data < 0] = 0
+    else:
+        if np.isnan(data).any():
+            logger.warning("NaNs detected, filling with 0.5 before NMF.")
+            data = np.nan_to_num(data, nan=0.5)
+        if (data < 0).any():
+            logger.warning("Negative values detected, clipping to 0 for NMF.")
+            data = np.clip(data, a_min=0, a_max=None)
+
+    model = NMF(
+        n_components=n_components,
+        init="nndsvda",
+        max_iter=max_iter,
+        random_state=random_state,
+    )
+    embedding = model.fit_transform(data)
+    components = model.components_.T
+
+    if subset_mask is not None:
+        components_matrix = np.zeros((adata.shape[1], components.shape[1]))
+        components_matrix[subset_mask, :] = components
+    else:
+        components_matrix = components
+
+    adata.obsm[embedding_key] = embedding
+    adata.varm[components_key] = components_matrix
+    adata.uns[uns_key] = {
+        "n_components": n_components,
+        "max_iter": max_iter,
+        "random_state": random_state,
+        "layer": layer,
+        "var_filters": list(var_filters) if var_filters else None,
+        "components_key": components_key,
+    }
+
+    logger.info(
+        "Stored: adata.obsm['%s'] and adata.varm['%s']",
+        embedding_key,
+        components_key,
+    )
+    return adata