stably 0.3.0__py3-none-any.whl

stably/__init__.py

@@ -0,0 +1,37 @@
+"""
+True Shah & Samworth (2013) ElasticNet stability selection package.
+
+Uses the r-concave PFER bound (equation 8) for threshold derivation,
+which is the main theoretical contribution of the paper over Meinshausen
+& Buhlmann (2010). Complementary pairs subsampling with data-adaptive
+threshold computed from the D function (Appendix A.4).
+
+Operates on protein-level DIA-NN matrices (pg_matrix). Peptide-level input
+is rejected explicitly — see io.load_data and the package README for the
+rationale.
+"""
+
+__version__ = "0.3.0"
+
+from .config import Config
+from .preprocessing import Preprocessor
+from .io import load_data, save_results
+from .visualization import create_visualizations
+
+from .rconcave import compute_D, compute_rconcave_threshold, print_threshold_comparison
+from .feature_selection import stability_selection_elasticnet, calculate_cohens_d
+from .workflows import full_dataset_stability_elasticnet
+
+__all__ = [
+    'Config',
+    'Preprocessor',
+    'load_data',
+    'save_results',
+    'create_visualizations',
+    'compute_D',
+    'compute_rconcave_threshold',
+    'print_threshold_comparison',
+    'stability_selection_elasticnet',
+    'calculate_cohens_d',
+    'full_dataset_stability_elasticnet',
+]

stably/__main__.py

@@ -0,0 +1,131 @@
+"""
+Command-line entry point for stably.
+
+Uses the r-concave bound (equation 8, Shah & Samworth 2013) for threshold
+derivation — the main theoretical contribution of S&S over Meinshausen &
+Buhlmann (2010).
+
+Usage:
+    python -m stably --config config_stably.yaml
+"""
+
+import argparse
+import sys
+import warnings
+
+import numpy as np
+
+# Force UTF-8 output so unicode glyphs (θ, π, π̂) print correctly on Windows
+# consoles whose default codepage is cp1252.
+if hasattr(sys.stdout, 'reconfigure'):
+    sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+if hasattr(sys.stderr, 'reconfigure'):
+    sys.stderr.reconfigure(encoding='utf-8', errors='replace')
+
+from stably import (
+    Config,
+    load_data,
+    save_results,
+    create_visualizations,
+    full_dataset_stability_elasticnet,
+)
+
+# Silence noisy non-actionable warnings only. ConvergenceWarning is captured
+# per-iteration inside the stability selection loop and surfaced in the run
+# manifest, so we deliberately do not blanket-filter warnings here.
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+warnings.filterwarnings('ignore', category=FutureWarning)
+
+
+def main():
+    """Run the complete biomarker discovery pipeline."""
+    parser = argparse.ArgumentParser(
+        description='Biomarker Discovery: true S&S (2013) r-concave ElasticNet stability selection'
+    )
+    parser.add_argument(
+        '--config',
+        type=str,
+        required=True,
+        help='Path to YAML configuration file'
+    )
+    args = parser.parse_args()
+
+    print(f"Loading configuration from: {args.config}")
+    config = Config.from_yaml(args.config)
+
+    np.random.seed(config.RANDOM_STATE)
+
+    print("=" * 60)
+    print("TRUE S&S (2013) r-CONCAVE ELASTICNET STABILITY SELECTION")
+    print("=" * 60)
+
+    # Load data
+    print("\nLoading data...")
+    X_raw, y, sample_ids, protein_metadata = load_data(config)
+    print(f"Loaded {X_raw.shape[0]} samples and {X_raw.shape[1]} features.")
+    print(f"Class distribution: {np.sum(y==0)} {config.CONTROL_NAME}, {np.sum(y==1)} {config.CASE_NAME}.")
+
+    k = config.MAX_CANDIDATES
+    B = config.STABILITY_ITERATIONS // 2
+    pfer = config.STABILITY_PFER
+    print(f"\nMAX_CANDIDATES = {k}, L1_RATIO = {config.L1_RATIO}")
+    print(f"B = {B} complementary pairs ({config.STABILITY_ITERATIONS} subsamples)")
+    print(f"PFER budget: {pfer:.1f} expected false positives")
+    print(f"NOTE: θ = q/p will be computed after preprocessing (p is data-dependent)")
+
+    if B > 50:
+        print(f"\n  WARNING: S&S Section 3.4.1 recommends B ≤ 50 for the r-concavity")
+        print(f"  assumption to hold.  Current B = {B}.  Consider reducing")
+        print(f"  stability_iterations to 100 (= 2 × 50).")
+
+    results = full_dataset_stability_elasticnet(X_raw, y, config)
+
+    if results is None:
+        print("\nAnalysis failed — no stable features found.")
+        return None
+
+    print(f"\n{'=' * 60}")
+    print("RESULTS")
+    print(f"{'=' * 60}")
+
+    feat_stab = results['feature_stability']
+    ss_info = results.get('stability_selection', {})
+    threshold_info = ss_info.get('threshold_info', {})
+
+    def get_gene_name(feature_idx):
+        if feature_idx in protein_metadata.index:
+            gene = protein_metadata.loc[feature_idx, 'Genes']
+            return f"{gene} (index {feature_idx})"
+        return str(feature_idx)
+
+    threshold = ss_info.get('threshold', float('nan'))
+    c_ref = ss_info.get('C_ref', float('nan'))
+    sel_probs = feat_stab.get('selection_probabilities', {})
+    stable_list = feat_stab.get('stable_features', [])
+    sorted_stable = sorted(stable_list, key=lambda f: sel_probs.get(f, 0), reverse=True)
+
+    print(f"\nStable features (sorted by selection probability):")
+    print(f"  r-concave threshold: π = {threshold:.4f}")
+    if threshold_info:
+        print(f"  M&B threshold:       π = {threshold_info.get('mb_threshold', 'N/A'):.4f}")
+        print(f"  Threshold reduction: Δπ = {threshold_info.get('threshold_reduction', 0):.4f}")
+    print(f"  C_ref = {c_ref:.2e}")
+    print(f"  Total stable: {len(sorted_stable)}")
+
+    for i, feat in enumerate(sorted_stable[:20], 1):
+        print(f"    {i}. {get_gene_name(feat)}  π̂={sel_probs.get(feat, 0):.3f}")
+
+    print(f"\n{'=' * 60}")
+    print("ANALYSIS COMPLETE")
+    print(f"{'=' * 60}")
+
+    results['permutation_test'] = None
+
+    save_results(results, protein_metadata, config)
+    create_visualizations(results, protein_metadata, config)
+
+    return results
+
+
+if __name__ == "__main__":
+    main()

stably/config.py

@@ -0,0 +1,87 @@
+"""Configuration management for the true S&S ElasticNet stability selection pipeline."""
+
+import yaml
+from pathlib import Path
+from typing import Optional
+
+
+class Config:
+    """
+    Configuration manager that loads settings from a YAML file.
+
+    Usage:
+        config = Config.from_yaml('config.yaml')
+        print(config.DATA_FILE)
+    """
+
+    def __init__(self, config_dict: dict):
+        """Initialize Config from a dictionary."""
+        # Data files
+        self.DATA_FILE = config_dict['data_file']
+        self.LABEL_FILE = config_dict['label_file']
+
+        # Sample information
+        self.CASE_NAME = config_dict['case_name']
+        self.CONTROL_NAME = config_dict['control_name']
+        self.GROUP_COLUMN = config_dict['group_column']
+        self.SAMPLE_ID_COLUMN = config_dict['sample_id_column']
+
+        # Random seed
+        self.RANDOM_STATE = config_dict['random_state']
+
+        # Preprocessing parameters
+        self.MAX_MISSING = config_dict['max_missing']
+        self.LOG_TRANSFORM = config_dict['log_transform']
+        self.CORRELATION_THRESHOLD = config_dict['correlation_threshold']
+        self.IMPUTATION_STRATEGY = config_dict['imputation_strategy']
+        self.KNN_NEIGHBORS = config_dict.get('knn_neighbors', 5)
+        self.MIN_PROTEOTYPIC_PEPTIDES = config_dict.get('min_proteotypic_peptides', 2)
+
+        # Upstream DIA-NN log (for database version capture, HC-INTER-03).
+        # Optional: when None, load_data will try to auto-discover a *.log.txt
+        # next to the data file or in a sibling PDC_outputs_*/ directory.
+        self.DIANN_LOG_FILE = config_dict.get('diann_log_file', None)
+
+        # Machine learning
+        self.MAX_ITERATIONS = config_dict['max_iterations']
+
+        # ElasticNet mixing parameter
+        self.L1_RATIO = config_dict.get('l1_ratio', 0.5)
+
+        # Stability selection
+        self.MAX_CANDIDATES = config_dict['max_candidates']
+        self.STABILITY_ITERATIONS = config_dict['stability_iterations']
+        self.STABILITY_PFER = float(config_dict['stability_pfer'])
+        self.N_JOBS = config_dict['n_jobs']
+
+        # Output
+        self.OUTPUT_DIR = config_dict.get('output_dir', 'biomarker_results')
+
+    @classmethod
+    def from_yaml(cls, yaml_path: str) -> 'Config':
+        """
+        Load configuration from a YAML file.
+
+        Parameters
+        ----------
+        yaml_path : str
+            Path to the YAML configuration file
+
+        Returns
+        -------
+        Config
+            Configuration object
+        """
+        yaml_path = Path(yaml_path)
+        if not yaml_path.exists():
+            raise FileNotFoundError(f"Configuration file not found: {yaml_path}")
+
+        with open(yaml_path, 'r') as f:
+            config_dict = yaml.safe_load(f)
+
+        return cls(config_dict)
+
+    @classmethod
+    def from_dict(cls, config_dict: dict) -> 'Config':
+        """Create Config from a dictionary (useful for testing)."""
+        return cls(config_dict)

stably/diann_log.py

@@ -0,0 +1,141 @@
+"""Parse a DIA-NN log file to capture the upstream provenance needed for
+HC-INTER-03 (peptide/protein identifiers referenced to a specific database
+version) and HC-FDR-06 (MBR-transferred vs MS2-confirmed IDs).
+
+The parser is intentionally permissive: if fields cannot be located it returns
+None for those fields rather than raising, so a partial manifest is still
+better than no manifest.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Optional
+
+
+def find_diann_log(data_file: str | Path) -> Optional[Path]:
+    """
+    Heuristic search for a DIA-NN log near `data_file`.
+
+    Looks in:
+      1. The directory of `data_file` itself.
+      2. Immediate subdirectories (one level deep).
+      3. Sibling directories (one level up, one level down).
+
+    Returns the first match or None.
+    """
+    data_path = Path(data_file).resolve()
+    candidates = []
+
+    if data_path.parent.exists():
+        candidates.extend(sorted(data_path.parent.glob("*.log.txt")))
+        for child in sorted(data_path.parent.iterdir()):
+            if child.is_dir():
+                candidates.extend(sorted(child.glob("*.log.txt")))
+
+    parent = data_path.parent.parent
+    if parent.exists():
+        candidates.extend(sorted(parent.glob("*.log.txt")))
+
+    # De-duplicate while preserving order
+    seen = set()
+    for path in candidates:
+        resolved = path.resolve()
+        if resolved in seen:
+            continue
+        seen.add(resolved)
+        if resolved.is_file() and _looks_like_diann_log(resolved):
+            return resolved
+    return None
+
+
+def _looks_like_diann_log(path: Path) -> bool:
+    try:
+        with open(path, 'r', encoding='utf-8', errors='replace') as f:
+            head = f.read(512)
+    except OSError:
+        return False
+    return head.lstrip().startswith("DIA-NN")
+
+
+_FASTA_RE = re.compile(r"--fasta\s+(\S+)")
+_LIB_RE = re.compile(r"--lib\s+(\S+)")
+_QVALUE_RE = re.compile(r"--qvalue\s+(\S+)")
+_THREADS_RE = re.compile(r"--threads\s+(\d+)")
+_REANALYSE_RE = re.compile(r"--reanalyse(?!\w)")
+_MATRICES_RE = re.compile(r"--matrices(?!\w)")
+_MBR_LINE_RE = re.compile(r"MBR\s+enabled", re.IGNORECASE)
+_VERSION_RE = re.compile(r"^DIA-NN\s+(\S+)(?:\s+(\S+))?", re.MULTILINE)
+_COMPILED_RE = re.compile(r"^Compiled on\s+(.+)$", re.MULTILINE)
+_FASTA_DATE_TAIL_RE = re.compile(r"_([A-Z][a-z]{2}\d{2})\.", re.IGNORECASE)
+
+
+def parse_diann_log(path: str | Path, head_chars: int = 8192) -> dict:
+    """
+    Parse the header region of a DIA-NN log.
+
+    Only the first `head_chars` are read, which is enough for every field of
+    interest on a standard DIA-NN log. Returns a dict with keys:
+        diann_version, diann_edition, compile_date, command_line,
+        fasta_path, fasta_name, fasta_release_tag,
+        library_path, qvalue, mbr_enabled, reanalyse,
+        matrices, threads, log_file.
+    Missing fields are None.
+    """
+    path = Path(path)
+    with open(path, 'r', encoding='utf-8', errors='replace') as f:
+        head = f.read(head_chars)
+
+    version_match = _VERSION_RE.search(head)
+    diann_version = version_match.group(1) if version_match else None
+    diann_edition = version_match.group(2) if version_match and version_match.group(2) else None
+
+    compiled_match = _COMPILED_RE.search(head)
+    compile_date = compiled_match.group(1).strip() if compiled_match else None
+
+    command_line = _extract_command_line(head)
+
+    fasta_match = _FASTA_RE.search(head)
+    fasta_path = fasta_match.group(1) if fasta_match else None
+    fasta_name = Path(fasta_path).name if fasta_path else None
+    fasta_tag_match = _FASTA_DATE_TAIL_RE.search(fasta_name or "")
+    fasta_release_tag = fasta_tag_match.group(1) if fasta_tag_match else None
+
+    lib_match = _LIB_RE.search(head)
+    library_path = lib_match.group(1) if lib_match else None
+
+    qvalue_match = _QVALUE_RE.search(head)
+    qvalue = float(qvalue_match.group(1)) if qvalue_match else None
+
+    threads_match = _THREADS_RE.search(head)
+    threads = int(threads_match.group(1)) if threads_match else None
+
+    reanalyse = bool(_REANALYSE_RE.search(head))
+    matrices = bool(_MATRICES_RE.search(head))
+    mbr_enabled = bool(_MBR_LINE_RE.search(head))
+
+    return {
+        'diann_version': diann_version,
+        'diann_edition': diann_edition,
+        'compile_date': compile_date,
+        'command_line': command_line,
+        'fasta_path': fasta_path,
+        'fasta_name': fasta_name,
+        'fasta_release_tag': fasta_release_tag,
+        'library_path': library_path,
+        'qvalue': qvalue,
+        'mbr_enabled': mbr_enabled,
+        'reanalyse': reanalyse,
+        'matrices': matrices,
+        'threads': threads,
+        'log_file': str(path),
+    }
+
+
+def _extract_command_line(head: str) -> Optional[str]:
+    """The DIA-NN command line is the first line containing --fasta or --lib."""
+    for line in head.splitlines():
+        if '--fasta' in line or '--lib' in line:
+            return line.strip()
+    return None

stably/feature_selection.py

@@ -0,0 +1,217 @@
+"""Feature selection using the true Shah & Samworth (2013) r-concave bound.
+
+Uses the r-concave PFER bound (equation 8) instead of the Meinshausen &
+Buhlmann worst-case bound for threshold computation.
+"""
+
+import warnings
+
+import numpy as np
+import pandas as pd
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import StratifiedShuffleSplit
+from sklearn.exceptions import ConvergenceWarning
+from joblib import Parallel, delayed
+from typing import List, Dict, Tuple
+
+from .rconcave import compute_rconcave_threshold, print_threshold_comparison
+
+
+# Design note: C_ref is calibrated once on the FULL training set and reused
+# across every complementary-pairs subsample. This is the Shah & Samworth
+# (2013) design — regularisation is treated as a fixed hyperparameter of the
+# stability procedure, not something re-tuned per subsample. It is not a
+# cross-validation leakage bug: stability selection does not evaluate model
+# performance on held-out data; it counts how often each feature is selected
+# across subsamples at a fixed regularisation strength.
+def _find_elasticnet_C_for_q(X, y, q, random_state, l1_ratio, max_iter=1000):
+    """Largest ElasticNet C (in a log-spaced sweep) that keeps <= q non-zero coefs."""
+    Cs = np.logspace(-4, 2, 50)
+    C_ref = Cs[0]
+    for C in Cs:
+        model = LogisticRegression(
+            penalty='elasticnet',
+            C=C,
+            l1_ratio=l1_ratio,
+            solver='saga',
+            random_state=random_state,
+            max_iter=max_iter,
+        )
+        model.fit(X, y)
+        n_nonzero = int(np.sum(model.coef_[0] != 0))
+        if n_nonzero <= q:
+            C_ref = C
+    return C_ref
+
+
+def _stability_iteration_elasticnet_path(
+    X, y, q, iteration_idx, random_seed, C_ref, l1_ratio, max_iter
+):
+    """Single complementary-pairs iteration. Returns (selected_features, converged)."""
+    # Complementary pairs: (2i, 2i+1) share the split seed
+    splitter = StratifiedShuffleSplit(
+        n_splits=1,
+        train_size=0.5,
+        random_state=random_seed + (iteration_idx // 2),
+    )
+    train_indices, test_indices = next(splitter.split(X, y))
+    subsample_indices = train_indices if iteration_idx % 2 == 0 else test_indices
+
+    X_sub = X.iloc[subsample_indices]
+    y_sub = y[subsample_indices]
+
+    model = LogisticRegression(
+        penalty='elasticnet',
+        C=C_ref,
+        l1_ratio=l1_ratio,
+        solver='saga',
+        random_state=random_seed + iteration_idx,
+        max_iter=max_iter,
+    )
+
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always", ConvergenceWarning)
+        model.fit(X_sub, y_sub)
+        converged = not any(issubclass(w.category, ConvergenceWarning) for w in caught)
+
+    coefs = np.abs(model.coef_[0])
+    nonzero_idx = np.where(coefs > 1e-6)[0]
+    if len(nonzero_idx) == 0:
+        return [], converged
+    top_q_idx = nonzero_idx[np.argsort(coefs[nonzero_idx])[::-1][:q]]
+    return X.columns[top_q_idx].tolist(), converged
+
+
+def stability_selection_elasticnet(
+    X, y, config, q=None, n_iterations=None, n_jobs=None
+) -> Tuple[List[str], float, Dict[str, float], float, dict]:
+    """
+    Shah & Samworth (2013) stability selection with the true r-concave bound.
+
+    Returns
+    -------
+    stable_features : list of str
+    threshold : float (r-concave PFER-derived selection probability)
+    selection_probs : dict[str, float]
+    C_ref : float
+    threshold_info : dict with rconcave/MB comparison and convergence stats
+    """
+    q = q or config.MAX_CANDIDATES
+    n_iterations = n_iterations or config.STABILITY_ITERATIONS
+    n_jobs = n_jobs or config.N_JOBS
+    l1_ratio = config.L1_RATIO
+
+    p = X.shape[1]
+    B = n_iterations // 2
+    pfer = config.STABILITY_PFER
+
+    # Compute the r-concave threshold (true S&S bound)
+    print(f"    Computing r-concave threshold (q={q}, p={p}, B={B}, PFER={pfer:.1f})...")
+    threshold, bound_at_tau, mb_threshold = compute_rconcave_threshold(q, p, B, pfer)
+    threshold = min(threshold, 1.0)
+
+    print_threshold_comparison(q, p, B, pfer)
+
+    threshold_info = {
+        'rconcave_threshold': threshold,
+        'mb_threshold': mb_threshold,
+        'threshold_reduction': mb_threshold - threshold,
+        'pfer_bound': bound_at_tau,
+        'pfer_target': pfer,
+        'theta': q / p,
+        'B': B,
+    }
+
+    # Calibrate C from the regularisation path (once, on the full data; see
+    # design note at the top of this module).
+    print(f"    Calibrating ElasticNet regularisation from path (l1_ratio={l1_ratio})...")
+    C_ref = _find_elasticnet_C_for_q(X, y, q, config.RANDOM_STATE, l1_ratio, config.MAX_ITERATIONS)
+    print(f"    C_ref={C_ref:.2e}  |  r-concave threshold: π={threshold:.4f}  |  PFER≤{pfer:.1f}")
+    print(f"    Running {n_iterations} complementary-pairs iterations (n_jobs={n_jobs})...")
+
+    iteration_results = Parallel(n_jobs=n_jobs, verbose=0)(
+        delayed(_stability_iteration_elasticnet_path)(
+            X, y, q, i, config.RANDOM_STATE, C_ref, l1_ratio, config.MAX_ITERATIONS
+        )
+        for i in range(n_iterations)
+    )
+
+    all_selected = [features for features, _ in iteration_results]
+    n_non_converged = sum(1 for _, converged in iteration_results if not converged)
+    threshold_info['n_iterations'] = n_iterations
+    threshold_info['n_non_converged'] = n_non_converged
+    threshold_info['non_convergence_rate'] = n_non_converged / n_iterations
+
+    if n_non_converged / n_iterations > 0.05:
+        print(
+            f"    WARNING: {n_non_converged}/{n_iterations} "
+            f"({100 * n_non_converged / n_iterations:.1f}%) iterations did not converge. "
+            f"Consider increasing config.MAX_ITERATIONS."
+        )
+    elif n_non_converged > 0:
+        print(f"    Convergence: {n_iterations - n_non_converged}/{n_iterations} iterations")
+
+    # Empirical selection probabilities
+    feature_counts: Dict[str, int] = {}
+    for selected in all_selected:
+        for feature in selected:
+            feature_counts[feature] = feature_counts.get(feature, 0) + 1
+
+    selection_probs = {f: c / n_iterations for f, c in feature_counts.items()}
+
+    stable_features = [f for f, prob in selection_probs.items() if prob >= threshold]
+
+    print(f"    Completed: {len(stable_features)} stable features (π̂ ≥ {threshold:.4f})")
+    return stable_features, threshold, selection_probs, C_ref, threshold_info
+
+
+def calculate_cohens_d(X, y, features):
+    """
+    Cohen's d and per-group sample sizes for each feature between case and control.
+
+    Returns
+    -------
+    dict[str, dict]
+        Per-feature entries with keys:
+            cohens_d   — signed (group1 - group0) / pooled_std
+            n_case     — number of non-NaN samples with y == 1
+            n_control  — number of non-NaN samples with y == 0
+            direction  — 'up_case', 'up_control', or 'zero'
+    """
+    results = {}
+    for feature in features:
+        group0 = X.loc[y == 0, feature].dropna()
+        group1 = X.loc[y == 1, feature].dropna()
+        n0, n1 = len(group0), len(group1)
+
+        if n0 < 2 or n1 < 2:
+            results[feature] = {
+                'cohens_d': float('nan'),
+                'n_case': n1,
+                'n_control': n0,
+                'direction': 'undetermined',
+            }
+            continue
+
+        pooled_std = np.sqrt(
+            ((n0 - 1) * group0.std() ** 2 + (n1 - 1) * group1.std() ** 2) / (n0 + n1 - 2)
+        )
+        if pooled_std > 0:
+            d = (group1.mean() - group0.mean()) / pooled_std
+        else:
+            d = 0.0
+
+        if d > 0:
+            direction = 'up_case'
+        elif d < 0:
+            direction = 'up_control'
+        else:
+            direction = 'zero'
+
+        results[feature] = {
+            'cohens_d': float(d),
+            'n_case': n1,
+            'n_control': n0,
+            'direction': direction,
+        }
+    return results