bsvae 0.4.2__tar.gz → 0.4.4__tar.gz

{bsvae-0.4.2 → bsvae-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bsvae
-Version: 0.4.2
+Version: 0.4.4
 Summary: Gaussian Mixture VAE for multi-modal biological module discovery in omics data.
 License: GPL-3.0-only
 License-File: LICENSE

{bsvae-0.4.2 → bsvae-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bsvae"
-version = "0.4.2"  # dynamically resolved by poetry-dynamic-versioning
+version = "0.4.4"  # dynamically resolved by poetry-dynamic-versioning
 description = "Gaussian Mixture VAE for multi-modal biological module discovery in omics data."
 readme = "README.md"
 authors = ["Kynon J Benjamin <kj.benjamin90@gmail.com>"]

{bsvae-0.4.2 → bsvae-0.4.4}/src/bsvae/benchmark/harness.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import json
 import logging
+import re
 import time
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional, Sequence
@@ -35,15 +36,28 @@ logger = logging.getLogger(__name__)
 
 METHODS = ("bsvae", "wgcna")
 KNOWN_METHODS = ("bsvae", "bsvae_raw", "bsvae_coherent", "bsvae_hier", "bsvae_consensus", "bsvae_isoform", "bsvae_regressed",
+                 "bsvae_elbo_k", "bsvae_stability_k",
                  "wgcna", "wgcna_raw", "wgcna_coherent", "wgcna_signed", "wgcna_unsigned", "wgcna_regressed",
                  "usage_network", "ica", "nmf", "acorde", "twn")
+# ``bsvae_elbo_k`` / ``bsvae_stability_k``: the ``bsvae`` method with K chosen from the data (kselect.py);
+# ``wgcna_k<N>``: WGCNA at a forced resolution (spectral clustering of the TOM into N clusters), any N.
+_WGCNA_K = re.compile(r"^wgcna_k(\d+)$")
+
+
+def is_known_method(method: str) -> bool:
+    return method in KNOWN_METHODS or bool(_WGCNA_K.match(method))
 # Gene coherence is the default annotation policy whenever the scenario has transcript structure:
 # ``bsvae`` / ``wgcna`` apply it, ``bsvae_raw`` / ``wgcna_raw`` do not, ``*_coherent`` are explicit aliases.
 COHERENT_DEFAULT = {"bsvae": True, "bsvae_raw": False, "bsvae_coherent": True, "bsvae_regressed": True,
+                    "bsvae_elbo_k": True, "bsvae_stability_k": True,
                     "wgcna": True, "wgcna_raw": False, "wgcna_coherent": True, "wgcna_signed": True,
                     "wgcna_unsigned": True, "wgcna_regressed": True}
 
 
+def _coherent(method: str) -> bool:
+    return COHERENT_DEFAULT.get(method, bool(_WGCNA_K.match(method)))
+
+
 def _apply_coherence(result: MethodResult, expr: pd.DataFrame, truth: Dict, post_cfg: Optional[Dict], out_dir: Path) -> MethodResult:
     """Gene-coherence annotation policy on a label result (raw labels kept in modules_raw.csv)."""
     from bsvae.networks.postprocess import enforce_gene_coherence
@@ -204,9 +218,9 @@ def _done_keys(path: Path) -> set:
 
 
 def _check_k_mode(k_mode: str) -> None:
-    # K comes from the simulation metadata; no selection procedure is wired in.
-    if k_mode != "oracle":
-        raise ValueError(f"k_mode must be 'oracle' (got {k_mode!r}); the harness has no K-selection mode")
+    # oracle: the planted module count; elbo / stability: one K sweep per dataset (kselect.py).
+    from bsvae.benchmark.kselect import check_k_mode
+    check_k_mode(k_mode)
 
 
 def run_one(
@@ -222,8 +236,12 @@ def run_one(
     device=None,
     over_cluster_factor: Optional[float] = None,
     consensus_runs: int = 3,
+    method_timeout: Optional[int] = None,
 ) -> tuple[MethodResult, Dict]:
+    """``method_timeout``: seconds allowed to an external method process (acorde, TWN) before the row is
+    recorded as ``failed``; None keeps each runner's default (7,200 s)."""
     _check_k_mode(k_mode)
+    tmo = {"timeout_s": int(method_timeout)} if method_timeout else {}
     truth = load_truth(run_dir)
     expr = pd.read_csv(truth["expr_path"], sep="\t", index_col=0)
     expr.index = expr.index.astype(str)
@@ -239,9 +257,33 @@ def run_one(
         method_base = method[: -len("_raw")]
     else:
         method_base = method
+    from bsvae.benchmark.kselect import K_MODE_METHODS
+    fixed_k: Optional[int] = None
+    m_k = _WGCNA_K.match(method)
+    if m_k:
+        method_base, fixed_k = "wgcna", int(m_k.group(1))
+    elif method in K_MODE_METHODS:
+        method_base, k_mode = "bsvae", K_MODE_METHODS[method]
     out_dir = run_dir / "methods" / method / f"seed_{seed}"
     out_dir.mkdir(parents=True, exist_ok=True)
     n_modules = int(truth["metadata"]["summary"].get("n_modules", truth["metadata"]["parameters"].get("n_modules", 20)))
+    k_select: Dict = {}
+    if method_base in ("bsvae", "bsvae_coherent") and k_mode != "oracle":
+        from bsvae.benchmark.kselect import (K_GRID_DEFAULT, STABILITY_REPS_DEFAULT, SWEEP_EPOCHS_DEFAULT,
+                                             run_sweep, select_k)
+        allcfg = dict(bsvae_cfg or {})
+        tx_path = run_dir / "truth" / "tx2gene.tsv"
+        table = run_sweep(
+            run_dir, truth["expr_path"], tx_path if tx_path.exists() else None,
+            train_cfg={k: v for k, v in allcfg.items() if not k.startswith("_")},
+            k_grid=allcfg.get("_k_grid") or K_GRID_DEFAULT,
+            sweep_epochs=allcfg.get("_sweep_epochs") or SWEEP_EPOCHS_DEFAULT,
+            stability_reps=allcfg.get("_stability_reps") or STABILITY_REPS_DEFAULT,
+            seed=seed, no_cuda=bool(allcfg.get("no_cuda", False)),
+        )
+        k_sel, info = select_k(table, k_mode)
+        k_select = {"k_mode": k_mode, "n_modules_oracle": n_modules, "selected_k": int(k_sel), **info}
+        n_modules = int(k_sel)
 
     if method_base in ("bsvae", "bsvae_coherent"):
         cfg = dict(bsvae_cfg or {})
@@ -333,13 +375,13 @@ def run_one(
                     min_module_size=(post_cfg or {}).get("min_module_size", 10))
     elif method == "acorde":
         from bsvae.benchmark.methods import run_acorde
-        result = run_acorde(expr, workdir=out_dir, seed=seed)
+        result = run_acorde(expr, workdir=out_dir, seed=seed, **tmo)
     elif method == "twn":
         from bsvae.benchmark.methods import run_twn
         if "tx2gene" not in truth:
             result = MethodResult("twn", "skipped", message="scenario has no transcript structure")
         else:
-            result = run_twn(expr, truth["tx2gene"], workdir=out_dir)
+            result = run_twn(expr, truth["tx2gene"], workdir=out_dir, **tmo)
     elif method_base in ("wgcna", "wgcna_unsigned", "wgcna_signed", "wgcna_coherent"):
         cfg = dict(WGCNA_DEFAULTS)
         cfg.update(wgcna_cfg or {})
@@ -347,14 +389,22 @@ def run_one(
             cfg["network_type"] = "unsigned"
         elif method_base == "wgcna_signed":
             cfg["network_type"] = "signed"
+        if fixed_k is not None:
+            cfg["fixed_k"] = fixed_k
         result = run_wgcna(expr, workdir=out_dir, seed=seed, **cfg)
     else:
         raise ValueError(f"Unknown method: {method}")
 
     result.method = method
-    if COHERENT_DEFAULT.get(method, False):
+    if k_select:
+        result.params = {**result.params, **k_select}
+    if _coherent(method):
         result = _apply_coherence(result, expr, truth, post_cfg, out_dir)
     metrics = evaluate_run(truth, result) if result.status == "ok" else {}
+    if k_select:
+        metrics = {**metrics, "k_mode": k_mode, "selected_k": k_select["selected_k"], "k_elbo": k_select["k_elbo"],
+                   "k_stability": k_select.get("k_stability"), "stability_ari_at_k": k_select.get("stability_ari_at_k"),
+                   "val_elbo_range": k_select["val_elbo_range"]}
     if result.labels is not None:
         pd.DataFrame({"feature_id": result.labels.index, "module": result.labels.values}).to_csv(
             out_dir / "modules.csv", index=False,
@@ -384,6 +434,7 @@ def run_grid(
     device=None,
     over_cluster_factor: Optional[float] = None,
     consensus_runs: int = 3,
+    method_timeout: Optional[int] = None,
 ) -> pd.DataFrame:
     """Run every (scenario, rep, method, seed) and return the tidy results table."""
     _check_k_mode(k_mode)
@@ -402,9 +453,9 @@ def run_grid(
     n_reps = int(cfg.get("reps", 1) if reps is None else reps)
     seed_list = list(seeds) if seeds is not None else [0]
     methods = [m for m in methods]
-    unknown = [m for m in methods if m not in KNOWN_METHODS]
+    unknown = [m for m in methods if not is_known_method(m)]
     if unknown:
-        raise ValueError(f"Unknown method(s): {unknown}; choose from {KNOWN_METHODS}")
+        raise ValueError(f"Unknown method(s): {unknown}; choose from {KNOWN_METHODS} or wgcna_k<N>")
     if any(m.startswith("wgcna") for m in methods) and not wgcna_available():
         logger.warning("WGCNA (Rscript + WGCNA package) not available; its rows will be marked skipped")
 
@@ -427,6 +478,7 @@ def run_grid(
                         run_dir, method, seed, k_mode=k_mode, bsvae_cfg=bsvae_cfg,
                         wgcna_cfg=wgcna_cfg, post_cfg=post_cfg, postprocess=postprocess, device=device,
                         over_cluster_factor=over_cluster_factor, consensus_runs=consensus_runs,
+                        method_timeout=method_timeout,
                     )
                     row = {
                         "scenario_id": sid, "rep": rep, "method": method, "seed": seed,

bsvae-0.4.4/src/bsvae/benchmark/kselect.py

@@ -0,0 +1,125 @@
+"""
+Data-driven K for the benchmark harness (``k_mode`` ``elbo`` or ``stability``).
+
+The oracle mode hands BSVAE the planted module count. The two data-driven modes run one
+``bsvae-sweep-k`` per dataset, cached under ``<run_dir>/methods/_ksweep`` and shared by every
+method row of that dataset, which scores a K grid by held-out ELBO (gene-grouped validation
+split when the scenario has transcript structure) and, with more than one stability replicate,
+by the mean pairwise ARI of the held-out hard assignments across replicates. ``elbo`` and
+``stability`` read their K from the same table, so the two rows differ only in the final fit.
+The chosen K is then over-clustered like any other expected module count (``over_cluster_k``).
+"""
+from __future__ import annotations
+
+import fcntl
+import logging
+from pathlib import Path
+from typing import Dict, Optional, Sequence, Tuple
+
+import pandas as pd
+
+K_MODES = ("oracle", "elbo", "stability")
+K_MODE_METHODS = {"bsvae_elbo_k": "elbo", "bsvae_stability_k": "stability"}
+K_GRID_DEFAULT: Sequence[int] = (4, 6, 8, 12, 16, 24, 32)
+SWEEP_EPOCHS_DEFAULT = 60
+STABILITY_REPS_DEFAULT = 3
+SWEEP_NAME = "sweep"
+
+logger = logging.getLogger(__name__)
+
+
+def check_k_mode(k_mode: str) -> None:
+    if k_mode not in K_MODES:
+        raise ValueError(f"k_mode must be one of {K_MODES} (got {k_mode!r}); 'oracle' is the planted count")
+
+
+def sweep_results_path(run_dir: str | Path) -> Path:
+    return Path(run_dir) / "methods" / "_ksweep" / SWEEP_NAME / "sweep_k" / "sweep_results.csv"
+
+
+def run_sweep(
+    run_dir: str | Path,
+    expr_path: str | Path,
+    tx2gene_path: Optional[str | Path] = None,
+    *,
+    train_cfg: Optional[Dict] = None,
+    k_grid: Sequence[int] = K_GRID_DEFAULT,
+    sweep_epochs: int = SWEEP_EPOCHS_DEFAULT,
+    stability_reps: int = STABILITY_REPS_DEFAULT,
+    seed: int = 0,
+    no_cuda: bool = False,
+) -> Path:
+    """Run the K sweep for one dataset unless its results table already exists; returns that table.
+
+    A file lock serialises concurrent rows of the same dataset (the pack runner starts several rows
+    at once), so the sweep is trained once and the second row reads the finished table.
+    """
+    out = sweep_results_path(run_dir)
+    if out.exists():
+        return out
+    root = out.parents[2]                      # <run_dir>/methods/_ksweep
+    root.mkdir(parents=True, exist_ok=True)
+    with open(root / ".lock", "w") as fh:
+        fcntl.flock(fh, fcntl.LOCK_EX)
+        try:
+            if not out.exists():
+                _run(root, expr_path, tx2gene_path, train_cfg or {}, k_grid, sweep_epochs, stability_reps, seed, no_cuda)
+        finally:
+            fcntl.flock(fh, fcntl.LOCK_UN)
+    if not out.exists():
+        raise RuntimeError(f"K sweep finished without writing {out}")
+    return out
+
+
+def _run(root: Path, expr_path, tx2gene_path, train_cfg: Dict, k_grid, sweep_epochs, stability_reps, seed, no_cuda) -> None:
+    from bsvae.benchmark.runners import BSVAE_DEFAULTS
+    from bsvae.cli.sweep_k import main as sweep_main, parse_args as sweep_parse_args
+
+    argv = [SWEEP_NAME, "--dataset", str(expr_path), "--outdir", str(root),
+            "--k-grid", ",".join(str(int(k)) for k in k_grid),
+            "--sweep-epochs", str(int(sweep_epochs)), "--stability-reps", str(int(stability_reps)),
+            "--no-train-final", "--seed", str(int(seed)), "--log-level", "warning"]
+    if tx2gene_path is not None:
+        argv += ["--tx2gene", str(tx2gene_path)]
+    if no_cuda:
+        argv += ["--no-cuda"]
+    args = sweep_parse_args(argv)
+    cfg = dict(BSVAE_DEFAULTS)
+    cfg.update({k: v for k, v in train_cfg.items() if v is not None})
+    for k in ("epochs", "n_modules", "seed", "no_cuda", "tx2gene", "outdir", "log_level"):
+        cfg.pop(k, None)
+    unknown = [k for k in cfg if not hasattr(args, k)]
+    if unknown:
+        raise ValueError(f"Unknown BSVAE option(s) for the K sweep: {unknown}")
+    for k, v in cfg.items():
+        setattr(args, k, v)
+    logger.info("K sweep: grid %s, %d epochs, %d stability reps -> %s", list(k_grid), sweep_epochs, stability_reps, root)
+    sweep_main(args)
+
+
+def select_k(results_csv: str | Path, k_mode: str) -> Tuple[int, Dict]:
+    """K for ``k_mode`` from a sweep table, with the selections of both modes in the returned info."""
+    t = pd.read_csv(results_csv)
+    if t.empty or "val_elbo_mean" not in t:
+        raise ValueError(f"empty or malformed sweep table: {results_csv}")
+    t = t.sort_values("k").reset_index(drop=True)
+    elbo_row = t.loc[t["val_elbo_mean"].idxmin()]
+    info: Dict = {"k_grid": [int(k) for k in t["k"]], "k_elbo": int(elbo_row["k"]),
+                  "val_elbo_range": float(t["val_elbo_mean"].max() - t["val_elbo_mean"].min())}
+    stab = t["stability_ari_mean"] if "stability_ari_mean" in t else pd.Series(dtype=float)
+    if stab.notna().any():
+        s = t.sort_values(["stability_ari_mean", "val_elbo_mean"], ascending=[False, True])
+        info["k_stability"] = int(s["k"].iloc[0])
+        info["stability_ari_at_k"] = float(s["stability_ari_mean"].iloc[0])
+    if k_mode == "elbo":
+        return info["k_elbo"], info
+    if k_mode == "stability":
+        if "k_stability" not in info:
+            raise ValueError("the sweep has no stability column; rerun it with stability_reps > 1")
+        return info["k_stability"], info
+    check_k_mode(k_mode)
+    raise ValueError("select_k is for the data-driven modes; oracle K comes from the scenario")
+
+
+__all__ = ["K_MODES", "K_MODE_METHODS", "K_GRID_DEFAULT", "SWEEP_EPOCHS_DEFAULT", "STABILITY_REPS_DEFAULT",
+           "check_k_mode", "run_sweep", "select_k", "sweep_results_path"]

{bsvae-0.4.2 → bsvae-0.4.4}/src/bsvae/benchmark/runners.py

@@ -125,12 +125,21 @@ def run_wgcna(
     n_threads: int = 1,
     seed: int = 42,
     timeout_s: int = 7200,
+    fixed_k: Optional[int] = None,
 ) -> MethodResult:
-    """Run WGCNA on a features × samples matrix via ``Rscript``."""
+    """Run WGCNA on a features × samples matrix via ``Rscript``.
+
+    ``fixed_k`` replaces the dynamic tree cut by a spectral clustering of the TOM into that many
+    clusters (normalised Laplacian, k-means; no merge step, no size floor): WGCNA's network at a
+    forced resolution, single-block path only. Ward on the TOM dissimilarity (the first
+    implementation) leaves one giant cluster on gene-level data and PAM peels singletons on isoform
+    data, so neither serves as a resolution comparator across the benchmark families.
+    """
     params = dict(
         network_type=network_type, soft_power=soft_power, min_module_size=min_module_size,
         deep_split=deep_split, merge_cut_height=merge_cut_height,
         blockwise_threshold=blockwise_threshold, max_block_size=max_block_size, seed=seed,
+        fixed_k=fixed_k,
     )
     if not wgcna_available():
         logger.warning("Rscript/WGCNA not available; skipping WGCNA")
@@ -155,6 +164,8 @@ def run_wgcna(
     ]
     if soft_power is not None:
         cmd += ["--soft-power", str(int(soft_power))]
+    if fixed_k is not None:
+        cmd += ["--fixed-k", str(int(fixed_k))]
 
     t0 = time.perf_counter()
     try:

{bsvae-0.4.2 → bsvae-0.4.4}/src/bsvae/benchmark/wgcna.R

@@ -5,17 +5,38 @@
 ##   Rscript wgcna.R --expr expr.csv.gz --outdir out [--network-type signed]
 ##          [--soft-power P] [--min-module-size 10] [--deep-split 2]
 ##          [--merge-cut-height 0.25] [--blockwise-threshold 20000]
-##          [--max-block-size 5000] [--threads 1] [--seed 42]
+##          [--max-block-size 5000] [--threads 1] [--seed 42] [--fixed-k 0]
+##
+## --fixed-k K > 0 replaces the dynamic tree cut by a spectral clustering of the TOM into K clusters
+## (normalised Laplacian D^-1/2 TOM D^-1/2, its K leading eigenvectors row-normalised, k-means with 20
+## restarts; no merge step, no size floor): WGCNA's network at a forced resolution. The other fixed-K
+## rules on the same network fail on one of the benchmark families: cutting the average-linkage tree
+## or PAM on the TOM dissimilarity peel off singletons (PAM recovers the gene-level modules but gives
+## singletons on isoform data and whenever K exceeds the number of real clusters), and Ward (ward.D2) on
+## the TOM dissimilarity leaves one giant cluster on gene-level data (ARI 0.08 against 0.62 for the
+## dynamic cut at the planted K). Spectral clustering gives K balanced clusters on both.
 ##
 ## Writes modules.csv (feature_id, module, module_label; grey -> -1),
 ## eigengenes.csv (samples x modules) and params.json.
 
 suppressPackageStartupMessages(library(WGCNA))
 
+## Spectral clustering of a similarity matrix into exactly K clusters (Ng-Jordan-Weiss): the K leading
+## eigenvectors of D^-1/2 S D^-1/2, rows normalised to unit length, k-means with 20 restarts.
+spectral_fixed_k <- function(S, K, seed) {
+    d <- pmax(rowSums(S), .Machine$double.eps)
+    L <- sweep(sweep(S, 1, sqrt(d), "/"), 2, sqrt(d), "/")
+    ev <- eigen(L, symmetric = TRUE)
+    U <- ev$vectors[, seq_len(K), drop = FALSE]
+    U <- U / pmax(sqrt(rowSums(U^2)), .Machine$double.eps)
+    set.seed(seed)
+    as.integer(kmeans(U, centers = K, nstart = 20, iter.max = 200)$cluster)
+}
+
 args <- commandArgs(trailingOnly = TRUE)
 opt <- list(expr = NULL, outdir = NULL, `network-type` = "signed", `soft-power` = NA,
             `min-module-size` = 10, `deep-split` = 2, `merge-cut-height` = 0.25,
-            `blockwise-threshold` = 20000, `max-block-size` = 5000, threads = 1, seed = 42)
+            `blockwise-threshold` = 20000, `max-block-size` = 5000, threads = 1, seed = 42, `fixed-k` = 0)
 i <- 1
 while (i <= length(args)) {
     key <- sub("^--", "", args[i])
@@ -70,6 +91,9 @@ if (is.na(soft_power)) {
 min_size <- int(opt$`min-module-size`)
 deep_split <- int(opt$`deep-split`)
 cut_height <- num(opt$`merge-cut-height`)
+fixed_k <- int(opt$`fixed-k`)
+if (is.na(fixed_k)) fixed_k <- 0L
+if (fixed_k > 0 && n_feat > int(opt$`blockwise-threshold`)) stop("--fixed-k needs the single-block path (n_feat <= blockwise-threshold)")
 
 if (n_feat > int(opt$`blockwise-threshold`)) {
     log_msg("Blockwise WGCNA (maxBlockSize=%d)", int(opt$`max-block-size`))
@@ -87,6 +111,11 @@ if (n_feat > int(opt$`blockwise-threshold`)) {
     TOM <- TOMsimilarity(adj, TOMType = tom_type, verbose = 0)
     dissTOM <- 1 - TOM
     tree <- hclust(as.dist(dissTOM), method = "average")
+    if (fixed_k > 0) {
+        log_msg("Spectral clustering of the TOM into %d clusters (normalised Laplacian, k-means)", fixed_k)
+        labels <- spectral_fixed_k(TOM, min(fixed_k, n_feat), int(opt$seed))
+        MEs <- moduleEigengenes(datExpr, colors = labels)$eigengenes
+    } else {
     dyn <- cutreeDynamic(dendro = tree, distM = dissTOM, deepSplit = deep_split,
                          pamRespectsDendro = FALSE, minClusterSize = min_size, verbose = 0)
     if (length(unique(dyn[dyn != 0])) >= 2) {
@@ -97,6 +126,7 @@ if (n_feat > int(opt$`blockwise-threshold`)) {
         labels <- dyn
         MEs <- moduleEigengenes(datExpr, colors = labels)$eigengenes
     }
+    }
 }
 
 ## Relabel to 0..M-1 with grey (0) -> -1
@@ -121,8 +151,8 @@ colnames(MEs_out) <- paste0("ME", as.integer(relabel[me_names[keep]]))
 MEs_out <- cbind(data.frame(sample_id = rownames(datExpr), stringsAsFactors = FALSE), MEs_out)
 write.csv(MEs_out, file.path(opt$outdir, "eigengenes.csv"), row.names = FALSE, quote = FALSE)
 
-params <- sprintf('{"soft_power": %d, "network_type": "%s", "min_module_size": %d, "deep_split": %d, "merge_cut_height": %s, "n_modules": %d, "n_grey": %d, "blockwise": %s}',
-                  as.integer(soft_power), network_type, min_size, deep_split, format(cut_height),
+params <- sprintf('{"soft_power": %d, "network_type": "%s", "min_module_size": %d, "deep_split": %d, "merge_cut_height": %s, "fixed_k": %d, "n_modules": %d, "n_grey": %d, "blockwise": %s}',
+                  as.integer(soft_power), network_type, min_size, deep_split, format(cut_height), fixed_k,
                   length(uniq), sum(all_module < 0), tolower(as.character(n_feat > int(opt$`blockwise-threshold`))))
 writeLines(params, file.path(opt$outdir, "params.json"))
 log_msg("Done: %d modules, %d grey", length(uniq), sum(all_module < 0))

{bsvae-0.4.2 → bsvae-0.4.4}/src/bsvae/cli/benchmark.py

@@ -31,8 +31,14 @@ def build_parser() -> argparse.ArgumentParser:
                    help="Comma list of replicate indices to run (e.g. one per SLURM task); overrides --reps.")
     r.add_argument("--seeds", default="0", help="Comma list of method seeds (default: 0).")
     r.add_argument("--scenario-ids", default=None, help="Comma list to restrict scenarios.")
-    r.add_argument("--k-mode", choices=["oracle"], default="oracle",
-                   help="oracle: the true module count times --over-cluster-factor (the only mode).")
+    r.add_argument("--k-mode", choices=["oracle", "elbo", "stability"], default="oracle",
+                   help="oracle: the planted module count (times --over-cluster-factor); elbo / stability: K from one "
+                        "held-out sweep per dataset (bsvae-sweep-k), selected by validation ELBO or by assignment "
+                        "stability across replicates. The methods bsvae_elbo_k / bsvae_stability_k force their mode.")
+    r.add_argument("--k-grid", default=None, help="K grid of the sweep (comma list; default 4,6,8,12,16,24,32).")
+    r.add_argument("--sweep-epochs", type=int, default=None, help="Epochs per sweep fit (default 60).")
+    r.add_argument("--stability-reps", type=int, default=None,
+                   help="Training replicates per K in the sweep (default 3; 1 gives ELBO selection only).")
     r.add_argument("--no-resume", dest="resume", action="store_false", default=True)
     r.add_argument("--no-cuda", action="store_true", default=False)
     r.add_argument("--log-level", default="info")
@@ -65,6 +71,8 @@ def build_parser() -> argparse.ArgumentParser:
     r.add_argument("--wgcna-min-module-size", type=int, default=None)
     r.add_argument("--wgcna-merge-cut-height", type=float, default=None)
     r.add_argument("--wgcna-threads", type=int, default=None)
+    r.add_argument("--method-timeout", type=int, default=None,
+                   help="Seconds allowed to an external method process (acorde, twn) before the row fails (default 7200).")
 
     s = subs.add_parser("summarize", help="Mean ± sd per scenario and method.")
     s.add_argument("--results", required=True, help="results.tsv written by `run`.")
@@ -87,6 +95,12 @@ def handle_run(args, logger):
                             "bal_type", "normalize_input"])
     if args.no_cuda:
         bsvae_cfg["no_cuda"] = True
+    if args.k_grid:
+        bsvae_cfg["_k_grid"] = [int(x) for x in args.k_grid.split(",") if x.strip()]
+    if args.sweep_epochs is not None:
+        bsvae_cfg["_sweep_epochs"] = args.sweep_epochs
+    if args.stability_reps is not None:
+        bsvae_cfg["_stability_reps"] = args.stability_reps
     post_cfg = _opt(args, ["min_gamma", "min_module_size", "merge_cut_height", "kme_reassign"])
     wgcna_cfg = {}
     for cli_name, key in [("wgcna_network_type", "network_type"), ("wgcna_soft_power", "soft_power"),
@@ -106,6 +120,7 @@ def handle_run(args, logger):
         postprocess=not args.no_postprocess, resume=args.resume,
         scenario_ids=[s.strip() for s in args.scenario_ids.split(",")] if args.scenario_ids else None,
         over_cluster_factor=args.over_cluster_factor, consensus_runs=args.consensus_runs,
+        method_timeout=args.method_timeout,
     )
     logger.info("Benchmark complete: %d rows → %s/results.tsv", len(df), args.outdir)
 

{bsvae-0.4.2 → bsvae-0.4.4}/src/bsvae/networks/targets.py

@@ -676,7 +676,10 @@ def combine_target_tests(
         null_like = zmat[(zmat.max(axis=1) < sps.norm.isf(0.05)) | zmat.isna().any(axis=1)]
         src = null_like if len(null_like) >= 20 else zmat
         rho = src.corr(min_periods=5).fillna(0.0).clip(lower=0.0)
-        np.fill_diagonal(rho.values, 1.0)
+        # write the unit diagonal into a copy: under pandas copy-on-write (the pandas 3 default) .values is read-only
+        r = rho.to_numpy(dtype=float, copy=True)
+        np.fill_diagonal(r, 1.0)
+        rho = pd.DataFrame(r, index=rho.index, columns=rho.columns)
     rows = []
     for tx, g in long.groupby(level=0):
         p = np.clip(g["p_value"].values.astype(float), 1e-300, 1.0)

{bsvae-0.4.2 → bsvae-0.4.4}/src/bsvae/utils/isoform.py

@@ -166,6 +166,7 @@ def cross_fit_calls(
     min_kme: float = 0.3,
     alpha: Optional[float] = 0.05,
     require_module_agreement: bool = False,
+    null: str = "analytic",
     **kwargs,
 ) -> pd.DataFrame:
     """
@@ -192,7 +193,11 @@ def cross_fit_calls(
     one), ``label_discordant`` (the transcript's raw label differs from its
     gene's in any fold), per-fold ``kme_fold<f>`` columns and ``called``.
     ``require_module_agreement`` additionally demands ``modules_agree`` for a
-    call (an annotation by default).
+    call (an annotation by default).  ``null="stratum"`` replaces the
+    Bonferroni p of the combined statistic with the empirical null fitted per
+    selection stratum (:func:`stratum_empirical_null` on |z|; scales in
+    ``out.attrs["null_scale"]``); the per-fold scans always use the analytic
+    form since only their kME enter the combination.
     """
     from scipy import stats as sps
 
@@ -239,7 +244,13 @@ def cross_fit_calls(
     rows = np.arange(len(idx))
     z_best = Zc[rows, best_j]
     n_mod = max(1, len(cols))
-    p = np.minimum(2.0 * sps.norm.sf(np.abs(z_best)) * n_mod, 1.0)
+    null_scale: Dict[object, float] = {}
+    if null == "analytic":
+        p = np.minimum(2.0 * sps.norm.sf(np.abs(z_best)) * n_mod, 1.0)
+    elif null == "stratum":
+        p, null_scale = stratum_empirical_null(np.abs(z_best), discordant, n_mod)
+    else:
+        raise ValueError("null must be 'analytic' or 'stratum'")
     q = _bh_adjust(p)
     kme = KW[rows, best_j] / np.maximum(NW[rows, best_j], 1.0)
     agree = np.all(np.vstack(fold_best) == best_j[None, :], axis=0)
@@ -268,6 +279,7 @@ def cross_fit_calls(
         "called": called,
     }, index=idx)
     out.index.name = "feature_id"
+    out.attrs["null_scale"] = null_scale
     logger.info("Cross-fitted isoform-specific calls over %d folds: %d of %d transcripts called",
                 len(folds), int(called.sum()), len(out))
     return out
@@ -286,6 +298,59 @@ def _bh_adjust(p: np.ndarray) -> np.ndarray:
     return np.minimum(q, 1.0)
 
 
+def stratum_empirical_null(stat: np.ndarray, strata: np.ndarray, n_modules: "int | np.ndarray",
+                           min_size: int = 50, levels: Tuple[float, ...] = (0.6, 0.8)) -> Tuple[np.ndarray, Dict[object, float]]:
+    """
+    Empirical null of a best-of-``n_modules`` statistic, fitted per selection stratum.
+
+    ``stat`` is the absolute statistic of each transcript's best module (the
+    |t| of the best residual kME, or the combined |z| of the cross-fit),
+    ``strata`` the stratum each transcript was selected in (the callers pass
+    ``label_discordant``: whether the transcript's own label is the module it
+    is tested against).  Under the null ``stat / sigma_s`` is taken to be the
+    maximum of ``n_modules`` independent |N(0, 1)|, with CDF
+    ``(2 Phi(x) - 1) ** m``; ``sigma_s`` is fitted per stratum by least squares
+    on the ``levels`` quantiles of ``stat`` (default the 60th and 80th
+    percentiles).  The lower quantiles of a best-of-m statistic over
+    correlated modules sit below the independent-module form, so anchoring the
+    fit there (25th/50th) shrinks ``sigma_s`` and inflates the tail; anchors
+    at the 90th percentile or above absorb real divergence once it exceeds
+    about a tenth of a stratum.  The 60th/80th pair keeps the concordant
+    stratum near the analytic calibration and tolerates up to ~15 % of
+    real signal in the upper tail.  A stratum with fewer than
+    ``min_size`` transcripts keeps ``sigma_s = 1``.  At ``sigma_s = 1`` the
+    p-value is Sidak's ``1 - (1 - p_best) ** m`` rather than the Bonferroni
+    ``min(1, m p_best)`` of the analytic null.
+
+    Returns the p-values and the fitted scale per stratum.
+    """
+    from scipy import stats as sps
+
+    a = np.abs(np.asarray(stat, dtype=np.float64))
+    strata = np.asarray(strata)
+    m = np.maximum(np.broadcast_to(np.asarray(n_modules, dtype=np.float64), a.shape), 1.0)
+    levels = np.asarray(levels, dtype=np.float64)
+    p = np.ones_like(a)
+    scales: Dict[object, float] = {}
+    for s in pd.unique(strata):
+        idx = np.where(strata == s)[0]
+        sigma = 1.0
+        if idx.size >= int(min_size):
+            ms = float(np.median(m[idx]))
+            x = sps.norm.ppf((levels ** (1.0 / ms) + 1.0) / 2.0)      # null quantiles at unit scale
+            q = np.quantile(a[idx], levels)
+            sigma = float((q * x).sum() / (x * x).sum())
+            if not np.isfinite(sigma) or sigma <= 0:
+                sigma = 1.0
+        scales[s.item() if hasattr(s, "item") else s] = sigma
+        sf = np.minimum(sps.norm.sf(a[idx] / sigma), 0.5 - 1e-12)
+        p[idx] = -np.expm1(m[idx] * np.log1p(-2.0 * sf))
+    return np.clip(p, 0.0, 1.0), scales
+
+
+__all__.append("stratum_empirical_null")
+
+
 def _residualize_eigengenes(eigengenes: pd.DataFrame, column: int) -> pd.DataFrame:
     """Regress every eigengene on ``eigengenes[:, column]`` and return the residuals."""
     E = eigengenes.values.astype(np.float64)
@@ -317,6 +382,7 @@ def isoform_specific_calls(
     return_kme: bool = False,
     covariates: Optional[pd.DataFrame] = None,
     max_target_corr: Optional[float] = None,
+    null: str = "analytic",
 ) -> "pd.DataFrame | Tuple[pd.DataFrame, pd.DataFrame]":
     """
     Call isoforms whose *gene-programme-removed* profile belongs to a module
@@ -359,6 +425,16 @@ def isoform_specific_calls(
     0.3 with the residualised eigengenes keeps precision above the previous
     0.4 default on the isoform simulations.
 
+    ``null`` chooses the reference distribution of the best kME.
+    ``"analytic"`` (default) is the t-test above, Bonferroni-corrected over
+    modules.  ``"stratum"`` fits an empirical null per selection stratum
+    (``label_discordant`` True / False) with :func:`stratum_empirical_null`:
+    the |t| of the best module is scaled by a stratum-specific factor fitted
+    on the lower half of the stratum's distribution, so transcripts whose
+    own label is the tested module are compared with the distribution that
+    selection actually gives them.  The fitted scales are returned in
+    ``out.attrs["null_scale"]``.
+
     Every multi-isoform transcript is tested, whatever its own module label:
     the residual test is the primary evidence for isoform-specific
     membership, and ``label_discordant`` (the transcript's own label differs
@@ -413,7 +489,8 @@ def isoform_specific_calls(
             E = _residualize_eigengenes(eigengenes, cols.index(int(gm))) if gm in cols else eigengenes
             kme[rows_idx] = compute_kme(resid.iloc[rows_idx], E, list(resid.index[rows_idx])).values
     else:
-        kme = compute_kme(resid, eigengenes, list(resid.index)).values
+        # a writable copy: the leave-one-out pass below updates kme in place (read-only view under copy-on-write)
+        kme = compute_kme(resid, eigengenes, list(resid.index)).to_numpy(dtype=np.float64, copy=True)
     mode = {True: "exact", False: "off"}.get(loo_eigengene, loo_eigengene)
     if mode not in ("exact", "approx", "off"):
         raise ValueError("loo_eigengene must be True/False, 'exact' or 'approx'")
@@ -488,17 +565,24 @@ def isoform_specific_calls(
     n = expr.shape[1]
     r2 = np.minimum(best ** 2, 1.0 - 1e-12)
     tval = np.abs(best) * np.sqrt(max(n - 2, 1) / (1.0 - r2))
-    p = np.minimum(2.0 * sps.t.sf(tval, max(n - 2, 1)) * len(cols), 1.0)
+    own = lab.reindex(resid.index).fillna(-1).astype(int).values
+    discordant = (own >= 0) & (gm_per_tx >= 0) & (own != gm_per_tx)
+    null_scale: Dict[object, float] = {}
+    if null == "analytic":
+        p = np.minimum(2.0 * sps.t.sf(tval, max(n - 2, 1)) * len(cols), 1.0)
+    elif null == "stratum":
+        p, null_scale = stratum_empirical_null(tval, discordant, len(cols))
+    else:
+        raise ValueError("null must be 'analytic' or 'stratum'")
     q = _bh_adjust(p)
     called = np.abs(best) >= float(min_kme)
     if alpha is not None:
         called &= q < float(alpha)
-    own = lab.reindex(resid.index).fillna(-1).astype(int).values
     out = pd.DataFrame({
         "gene_id": stats["gene_id"].values,
         "gene_module": gm_per_tx,
         "isoform_module": own,
-        "label_discordant": (own >= 0) & (gm_per_tx >= 0) & (own != gm_per_tx),
+        "label_discordant": discordant,
         "best_other_module": [cols[j] for j in best_j],
         "kme_residual": best,
         "p_value": p,
@@ -509,6 +593,7 @@ def isoform_specific_calls(
         "called": called,
     }, index=resid.index)
     out.index.name = "feature_id"
+    out.attrs["null_scale"] = null_scale
     logger.info("Isoform-specific calls: %d of %d multi-isoform transcripts (kME ≥ %.2f%s)",
                 int(out["called"].sum()), len(out), min_kme, f", FDR < {alpha}" if alpha is not None else "")
     if return_kme: