pythonflex 0.3.3__py3-none-any.whl → 0.4__py3-none-any.whl

pythonflex/examples/runtime/runtime_benchmark_repeated.py

@@ -0,0 +1,347 @@
+"""Run five no-mPR runtime benchmarks for each bundled gold standard.
+
+Run from any directory with:
+    python path/to/src/pythonflex/examples/runtime_benchmark_repeated.py
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import argparse
+from datetime import datetime
+from pathlib import Path
+from time import perf_counter
+from typing import Any, Callable
+
+import pandas as pd
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[3]
+SRC_ROOT = PROJECT_ROOT / "src"
+if str(SRC_ROOT) not in sys.path:
+    sys.path.insert(0, str(SRC_ROOT))
+
+# Plot generation is benchmarked and saved without opening interactive windows.
+os.environ.setdefault("MPLBACKEND", "Agg")
+
+import pythonflex as flex  # noqa: E402
+
+
+GENE_EFFECT_PATH = Path(
+    "C:/Users/yd/Desktop/projects/_datasets/depmap/25Q2/gene_effect.csv"
+)
+RUN_COUNT = 5
+GOLD_STANDARDS = ("CORUM", "PATHWAY", "GOBP")
+DEFAULT_BENCHMARK_ROOT = (
+    PROJECT_ROOT
+    / "output"
+    / f"runtime_benchmark_{RUN_COUNT}_runs_no_mpr_{datetime.now():%Y%m%d_%H%M%S}"
+)
+BENCHMARK_ROOT = DEFAULT_BENCHMARK_ROOT
+
+
+def build_config(gold_standard: str, output_folder: Path) -> dict[str, Any]:
+    return {
+        "min_genes_in_complex": 2,
+        "min_genes_per_complex_analysis": 3,
+        "output_folder": str(output_folder),
+        "gold_standard": gold_standard,
+        "color_map": "RdYlBu",
+        "jaccard": True,
+        "analysis_genes": "common",
+        "plotting": {
+            "save_plot": True,
+            "show_plot": False,
+            "output_type": "png",
+        },
+        "preprocessing": {
+            "fill_na": True,
+            "normalize": False,
+        },
+        "corr_function": "numpy",
+        "logging": {
+            "visible_levels": ["DONE", "INFO", "WARNING"],
+        },
+    }
+
+
+def timed_call(
+    timings: list[dict[str, Any]],
+    gold_standard: str,
+    repetition: int,
+    step: str,
+    operation: Callable[[], Any],
+) -> Any:
+    start = perf_counter()
+    result = operation()
+    timings.append(
+        {
+            "gold_standard": gold_standard,
+            "repetition": repetition,
+            "step": step,
+            "seconds": perf_counter() - start,
+        }
+    )
+    return result
+
+
+def run_repetition(
+    gold_standard: str,
+    repetition: int,
+) -> list[dict[str, Any]]:
+    output_folder = BENCHMARK_ROOT / gold_standard / f"run_{repetition:02d}"
+    timings: list[dict[str, Any]] = []
+    workflow_start = perf_counter()
+
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "initialize",
+        lambda: flex.initialize(build_config(gold_standard, output_folder)),
+    )
+    gene_effect = timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "read_gene_effect",
+        lambda: pd.read_csv(GENE_EFFECT_PATH, index_col=0),
+    )
+    inputs = {
+        "All screens": {
+            "path": gene_effect,
+            "sort": "high",
+            "color": "#000000",
+        },
+    }
+    data, _ = timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "load_datasets",
+        lambda: flex.load_datasets(inputs),
+    )
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "load_gold_standard",
+        flex.load_gold_standard,
+    )
+
+    name, dataset = next(iter(data.items()))
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "pra",
+        lambda: flex.pra(name, dataset, is_corr=False),
+    )
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "pra_percomplex",
+        lambda: flex.pra_percomplex(name, dataset, is_corr=False),
+    )
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "complex_contributions",
+        lambda: flex.complex_contributions(name),
+    )
+
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "plot_precision_recall_curve",
+        flex.plot_precision_recall_curve,
+    )
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "plot_auc_scores",
+        flex.plot_auc_scores,
+    )
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "plot_significant_complexes",
+        flex.plot_significant_complexes,
+    )
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "plot_percomplex_scatter",
+        lambda: flex.plot_percomplex_scatter(n_top=20),
+    )
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "plot_percomplex_scatter_bysize",
+        flex.plot_percomplex_scatter_bysize,
+    )
+    timed_call(
+        timings,
+        gold_standard,
+        repetition,
+        "plot_complex_contributions",
+        flex.plot_complex_contributions,
+    )
+
+    timings.append(
+        {
+            "gold_standard": gold_standard,
+            "repetition": repetition,
+            "step": "total_runtime",
+            "seconds": perf_counter() - workflow_start,
+        }
+    )
+    output_folder.mkdir(parents=True, exist_ok=True)
+    pd.DataFrame(timings).to_csv(
+        output_folder / "benchmark_results.csv",
+        index=False,
+    )
+    return timings
+
+
+def write_reports(timings: list[dict[str, Any]]) -> None:
+    raw = pd.DataFrame(timings)
+    raw.to_csv(BENCHMARK_ROOT / "benchmark_results_all_runs.csv", index=False)
+
+    summary = (
+        raw.groupby(["gold_standard", "step"], as_index=False)["seconds"]
+        .agg(
+            repetitions="count",
+            mean_seconds="mean",
+            std_seconds="std",
+            min_seconds="min",
+            max_seconds="max",
+        )
+    )
+    summary.to_csv(BENCHMARK_ROOT / "benchmark_summary_mean_std.csv", index=False)
+
+    total_rows = raw[raw["step"] == "total_runtime"]
+    if not total_rows.empty:
+        per_repetition_total = (
+            total_rows.groupby("repetition", as_index=False)["seconds"]
+            .sum()
+            .rename(columns={"seconds": "all_gold_standards_seconds"})
+        )
+        per_repetition_total.to_csv(
+            BENCHMARK_ROOT / "benchmark_total_by_repetition.csv",
+            index=False,
+        )
+
+
+def parse_gold_standards(value: str) -> list[str]:
+    selected = [item.strip().upper() for item in value.split(",") if item.strip()]
+    invalid = [item for item in selected if item not in GOLD_STANDARDS]
+    if invalid:
+        raise ValueError(
+            f"Invalid gold standard(s): {invalid}. Choose from {GOLD_STANDARDS}."
+        )
+    return selected
+
+
+def load_existing_timings() -> list[dict[str, Any]]:
+    raw_path = BENCHMARK_ROOT / "benchmark_results_all_runs.csv"
+    if not raw_path.exists():
+        return []
+    return pd.read_csv(raw_path).to_dict("records")
+
+
+def has_completed_run(
+    timings: list[dict[str, Any]],
+    gold_standard: str,
+    repetition: int,
+) -> bool:
+    return any(
+        row.get("gold_standard") == gold_standard
+        and int(row.get("repetition", -1)) == repetition
+        and row.get("step") == "total_runtime"
+        for row in timings
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run repeated no-mPR pythonFLEX runtime benchmarks. "
+            "Use --output-root to resume or add runs to an existing output folder."
+        )
+    )
+    parser.add_argument(
+        "--output-root",
+        type=Path,
+        default=None,
+        help="Existing or new benchmark root. Defaults to a new timestamped folder.",
+    )
+    parser.add_argument(
+        "--gold-standards",
+        default=",".join(GOLD_STANDARDS),
+        help="Comma-separated subset of CORUM, PATHWAY, GOBP.",
+    )
+    parser.add_argument(
+        "--start-repetition",
+        type=int,
+        default=1,
+        help="First repetition number to run.",
+    )
+    parser.add_argument(
+        "--end-repetition",
+        type=int,
+        default=RUN_COUNT,
+        help="Last repetition number to run.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    if not GENE_EFFECT_PATH.exists():
+        raise FileNotFoundError(f"Input dataset was not found: {GENE_EFFECT_PATH}")
+
+    args = parse_args()
+    selected_standards = parse_gold_standards(args.gold_standards)
+    if args.start_repetition < 1 or args.end_repetition < args.start_repetition:
+        raise ValueError("Invalid repetition range.")
+    if args.end_repetition > RUN_COUNT:
+        raise ValueError(f"end-repetition cannot exceed RUN_COUNT={RUN_COUNT}.")
+
+    global BENCHMARK_ROOT
+    if args.output_root is not None:
+        BENCHMARK_ROOT = args.output_root.resolve()
+
+    BENCHMARK_ROOT.mkdir(parents=True, exist_ok=args.output_root is not None)
+    all_timings = load_existing_timings()
+    print(f"Benchmark output folder: {BENCHMARK_ROOT}")
+    print("mPR preparation is excluded from this repeated benchmark.")
+
+    for gold_standard in selected_standards:
+        for repetition in range(args.start_repetition, args.end_repetition + 1):
+            if has_completed_run(all_timings, gold_standard, repetition):
+                print(
+                    f"Skipping completed no-mPR benchmark: {gold_standard} "
+                    f"repetition {repetition}/{RUN_COUNT}"
+                )
+                continue
+            print(
+                f"Running no-mPR benchmark: {gold_standard} "
+                f"repetition {repetition}/{RUN_COUNT}"
+            )
+            all_timings.extend(run_repetition(gold_standard, repetition))
+            write_reports(all_timings)
+
+    summary_path = BENCHMARK_ROOT / "benchmark_summary_mean_std.csv"
+    print(f"Benchmark summary saved to: {summary_path}")
+
+
+if __name__ == "__main__":
+    main()