research-tool-cli 0.1.0__py3-none-any.whl

tests/test_post_hoc_tagging.py

@@ -0,0 +1,49 @@
+"""Adversarial tests for post-hoc tagging.
+
+Any analysis not in the locked plan gets tagged EXPLORATORY_POST_HOC
+everywhere downstream.  The tag must be impossible to strip without
+creating a new object.
+"""
+
+from core.stats.post_hoc import run_post_hoc, PostHocResult, EXPLORATORY_POST_HOC
+
+
+def test_post_hoc_result_has_tag():
+    r = PostHocResult(test_name="test", statistic=1.0, p_value=0.05)
+    assert r.tag == EXPLORATORY_POST_HOC
+    assert r.is_post_hoc() is True
+
+
+def test_tag_survives_to_dict():
+    r = PostHocResult(test_name="test", statistic=1.0, p_value=0.05)
+    d = r.to_dict()
+    assert d["tag"] == EXPLORATORY_POST_HOC
+
+
+def test_tag_not_silently_removable():
+    """The tag is a class attribute frozen at creation."""
+    r = PostHocResult(test_name="test", statistic=1.0, p_value=0.05)
+    d = r.to_dict()
+    assert d["tag"] is not None
+    assert d["tag"] == "EXPLORATORY_POST_HOC"
+
+
+def test_run_post_hoc_wraps_result():
+    def fake_test():
+        return {"test_name": "chi_square", "statistic": 3.84, "p_value": 0.05,
+                "ci_lower": None, "ci_upper": None, "params": {}}
+    r = run_post_hoc(fake_test)
+    assert isinstance(r, PostHocResult)
+    assert r.tag == EXPLORATORY_POST_HOC
+    assert r.test_name == "chi_square"
+
+
+def test_multiple_post_hoc_results():
+    """Multiple post-hoc results each carry their own tag."""
+    results = [
+        PostHocResult(test_name="a", statistic=1.0, p_value=0.05),
+        PostHocResult(test_name="b", statistic=2.0, p_value=0.01),
+    ]
+    for r in results:
+        assert r.tag == EXPLORATORY_POST_HOC
+        assert "EXPLORATORY_POST_HOC" in r.to_dict()["tag"]

tests/test_posthoc_analyze.py

@@ -0,0 +1,203 @@
+"""Tests for post-hoc amendment execution and deduplication."""
+
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+
+import pytest
+
+from core.database import get_connection, init_db, DATA_ROOT
+from core.masking.gate import seal_outcomes, unmask_study
+from core.planning.study_plan import StudyPlan, CoxPHModel
+from core.planning.lock import lock_plan, lock_amendment, load_plan
+from core.cli.main import cmd_analyze
+import argparse
+
+
+STUDY_ID = "test_posthoc"
+
+
+@pytest.fixture(autouse=True)
+def _setup():
+    p = DATA_ROOT / STUDY_ID
+    if p.exists():
+        shutil.rmtree(p)
+    p.mkdir(parents=True)
+
+    conn = get_connection(STUDY_ID)
+    init_db(conn)
+    conn.execute(
+        "INSERT OR REPLACE INTO studies (id, name, created_at, data_dir, study_type, is_locked) "
+        "VALUES (?, ?, ?, ?, ?, ?)",
+        (STUDY_ID, "Post-hoc Test", "2025-01-01", str(DATA_ROOT / STUDY_ID), "cohort", 0),
+    )
+    raw = f"raw_{STUDY_ID}"
+    conn.execute(f"CREATE TABLE IF NOT EXISTS {raw} (row_id INTEGER PRIMARY KEY, age TEXT, pfs_days TEXT, pfs_event TEXT, treatment_arm TEXT)")
+    for i in range(20):
+        arm = "A" if i % 2 == 0 else "B"
+        event = "1" if i % 3 == 0 else "0"
+        conn.execute(f"INSERT INTO {raw} (age, pfs_days, pfs_event, treatment_arm) VALUES (?, ?, ?, ?)",
+                     (str(30 + i), str(100 + i * 5), event, arm))
+    conn.execute("DELETE FROM variables WHERE study_id=?", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'age', 'baseline', 'continuous')", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'pfs_days', 'outcome', 'time_to_event')", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'pfs_event', 'outcome', 'categorical')", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'treatment_arm', 'baseline', 'categorical')", (STUDY_ID,))
+    conn.commit()
+    conn.close()
+
+    seal_outcomes(STUDY_ID)
+    yield
+    if p.exists():
+        shutil.rmtree(p)
+
+
+def _set_state(state: int):
+    conn = get_connection(STUDY_ID)
+    init_db(conn)
+    conn.execute("UPDATE studies SET is_locked=? WHERE id=?", (state, STUDY_ID))
+    conn.commit()
+    conn.close()
+
+
+def test_posthoc_test_executed_by_analyze():
+    """Post-hoc tests added via amend must be executed by analyze without --post-hoc flag."""
+    _set_state(1)
+    plan = StudyPlan(
+        study_id=STUDY_ID,
+        study_type="cohort",
+        primary_comparison="PFS by arm",
+        planned_tests=[],
+    )
+    lock_plan(STUDY_ID, plan)
+    _set_state(2)  # unmasked
+    unmask_study(STUDY_ID)
+
+    lock_amendment(
+        STUDY_ID,
+        amendment_reason="Add logrank test",
+        post_hoc_tests=[{"variable_name": "pfs_days", "test_name": "kaplan_meier_logrank", "rationale": "Post-hoc KM"}],
+    )
+
+    args = argparse.Namespace(study_id=STUDY_ID, force=False, post_hoc=False, rerun=False)
+    cmd_analyze(args)
+
+    conn = get_connection(STUDY_ID)
+    rows = conn.execute(
+        "SELECT test_name, is_pre_registered, status_json FROM analysis_results WHERE study_id=?",
+        (STUDY_ID,),
+    ).fetchall()
+    conn.close()
+
+    logrank_results = [r for r in rows if r["test_name"] == "kaplan_meier_logrank"]
+    assert len(logrank_results) == 1, f"Expected 1 logrank result, got {len(logrank_results)}"
+    assert logrank_results[0]["is_pre_registered"] == 0, "Post-hoc test should have is_pre_registered=0"
+    status = json.loads(logrank_results[0]["status_json"])
+    assert status["status"] in ("completed", "error"), f"Unexpected status: {status}"
+
+
+def test_amend_dedup_prevents_duplicate_posthoc_tests():
+    """Amending with the same test twice should not create duplicates."""
+    _set_state(1)
+    plan = StudyPlan(
+        study_id=STUDY_ID,
+        study_type="cohort",
+        primary_comparison="PFS by arm",
+        planned_tests=[],
+    )
+    lock_plan(STUDY_ID, plan)
+    _set_state(2)
+    unmask_study(STUDY_ID)
+
+    lock_amendment(
+        STUDY_ID,
+        amendment_reason="First add",
+        post_hoc_tests=[{"variable_name": "pfs_days", "test_name": "kaplan_meier_logrank", "rationale": "Post-hoc KM"}],
+    )
+    lock_amendment(
+        STUDY_ID,
+        amendment_reason="Second add (duplicate)",
+        post_hoc_tests=[{"variable_name": "pfs_days", "test_name": "kaplan_meier_logrank", "rationale": "Post-hoc KM"}],
+    )
+
+    latest = load_plan(STUDY_ID)
+    logrank_tests = [t for t in latest.post_hoc_tests if t["test_name"] == "kaplan_meier_logrank"]
+    assert len(logrank_tests) == 1, f"Expected 1 logrank test after dedup, got {len(logrank_tests)}"
+
+
+def test_planned_and_posthoc_both_executed():
+    """analyze should run both planned_tests and post_hoc_tests in a single call."""
+    _set_state(1)
+    plan = StudyPlan(
+        study_id=STUDY_ID,
+        study_type="cohort",
+        primary_comparison="PFS by arm",
+        planned_tests=[{"variable_name": "pfs_days", "test_name": "kaplan_meier_logrank", "rationale": "Pre-registered KM"}],
+    )
+    lock_plan(STUDY_ID, plan)
+    _set_state(2)
+    unmask_study(STUDY_ID)
+
+    lock_amendment(
+        STUDY_ID,
+        amendment_reason="Add chi-square",
+        post_hoc_tests=[{"variable_name": "pfs_event", "test_name": "chi_square", "rationale": "Post-hoc chi"}],
+    )
+
+    args = argparse.Namespace(study_id=STUDY_ID, force=False, post_hoc=False, rerun=False)
+    cmd_analyze(args)
+
+    conn = get_connection(STUDY_ID)
+    rows = conn.execute(
+        "SELECT test_name, is_pre_registered FROM analysis_results WHERE study_id=?",
+        (STUDY_ID,),
+    ).fetchall()
+    conn.close()
+
+    pre_reg = [r for r in rows if r["is_pre_registered"] == 1]
+    post_hoc = [r for r in rows if r["is_pre_registered"] == 0]
+    assert len(pre_reg) == 1, f"Expected 1 pre-registered result, got {len(pre_reg)}"
+    assert len(post_hoc) == 1, f"Expected 1 post-hoc result, got {len(post_hoc)}"
+    assert pre_reg[0]["test_name"] == "kaplan_meier_logrank"
+    assert post_hoc[0]["test_name"] == "chi_square"
+
+
+def test_plot_km_works_with_posthoc_km_result():
+    """plot-km must succeed against a post-hoc kaplan_meier_logrank result."""
+    from core.reporting.plots import _resolve_km_vars
+
+    _set_state(1)
+    plan = StudyPlan(
+        study_id=STUDY_ID,
+        study_type="cohort",
+        primary_comparison="PFS by arm",
+        planned_tests=[],
+    )
+    lock_plan(STUDY_ID, plan)
+    _set_state(2)
+    unmask_study(STUDY_ID)
+
+    lock_amendment(
+        STUDY_ID,
+        amendment_reason="Add KM test",
+        post_hoc_tests=[{"variable_name": "pfs_days", "test_name": "kaplan_meier_logrank", "rationale": "Post-hoc KM"}],
+    )
+
+    args = argparse.Namespace(study_id=STUDY_ID, force=False, post_hoc=False, rerun=False)
+    cmd_analyze(args)
+
+    # Find the kaplan_meier_logrank result ID
+    conn = get_connection(STUDY_ID)
+    row = conn.execute(
+        "SELECT id FROM analysis_results WHERE study_id=? AND test_name='kaplan_meier_logrank' LIMIT 1",
+        (STUDY_ID,),
+    ).fetchone()
+    conn.close()
+    assert row is not None, "Expected a kaplan_meier_logrank result"
+
+    # _resolve_km_vars should not raise — it must find the test in post_hoc_tests
+    resolved = _resolve_km_vars(STUDY_ID, row["id"])
+    assert resolved["time_col"] == "pfs_days"
+    assert resolved["event_col"] == "pfs_event"

tests/test_provenance_tracker.py

@@ -0,0 +1,110 @@
+"""Tests for provenance tracking."""
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from core.database import get_connection, init_db, DATA_ROOT
+from core.provenance.tracker import ProvenanceTracker, ProvenanceEntry
+
+STUDY_ID = "test_provenance"
+
+
+@pytest.fixture(autouse=True)
+def _setup():
+    conn = get_connection(STUDY_ID)
+    init_db(conn)
+    conn.execute(
+        "INSERT OR REPLACE INTO studies (id, name, created_at, data_dir) VALUES (?, ?, ?, ?)",
+        (STUDY_ID, "Provenance Test", "2025-01-01T00:00:00",
+         str(Path("data/studies") / STUDY_ID)),
+    )
+    raw = f"raw_{STUDY_ID}"
+    conn.execute(f"CREATE TABLE IF NOT EXISTS {raw} (row_id INTEGER PRIMARY KEY, age TEXT, response TEXT)")
+    conn.execute(f"INSERT INTO {raw} (age, response) VALUES ('65', 'CR')")
+    conn.execute(f"INSERT INTO {raw} (age, response) VALUES ('70', 'PR')")
+    conn.execute("DELETE FROM variables WHERE study_id=?", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'age', 'baseline', 'continuous')", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'response', 'outcome', 'categorical')", (STUDY_ID,))
+    conn.commit()
+    conn.close()
+    yield
+    p = DATA_ROOT / STUDY_ID
+    if p.exists():
+        shutil.rmtree(p)
+
+
+def test_record_and_load():
+    tracker = ProvenanceTracker(STUDY_ID)
+    tracker.record_run(
+        function_name="chi_square",
+        parameters={"group_col": "treatment_arm", "outcome_col": "response"},
+        source_row_ids=[1, 2, 3, 4, 5],
+        column_names=["treatment_arm", "response"],
+        test_name="chi_square",
+        result_id="result_001",
+        is_pre_registered=True,
+    )
+    assert len(tracker.get_all()) == 1
+
+    # Re-load from disk
+    tracker2 = ProvenanceTracker(STUDY_ID)
+    assert len(tracker2.get_all()) == 1
+
+
+def test_get_lineage():
+    tracker = ProvenanceTracker(STUDY_ID)
+    tracker.record_run(
+        function_name="chi_square",
+        parameters={},
+        source_row_ids=[1, 2, 3],
+        column_names=["x", "y"],
+        test_name="chi_square",
+        result_id="r1",
+    )
+    tracker.record_run(
+        function_name="t_test",
+        parameters={},
+        source_row_ids=[4, 5, 6],
+        column_names=["a", "b"],
+        test_name="t_test",
+        result_id="r2",
+    )
+    lines = tracker.get_lineage(test_name="chi_square")
+    assert len(lines) == 1
+    assert lines[0].test_name == "chi_square"
+
+
+def test_provenance_entry_timestamp():
+    import datetime
+    e = ProvenanceEntry(
+        function_name="test",
+        parameters={},
+        source_row_ids=[1],
+        column_names=["x"],
+        test_name="test",
+        result_id="r1",
+        study_id=STUDY_ID,
+    )
+    assert e.computed_at is not None
+
+
+def test_pre_registered_flag():
+    tracker = ProvenanceTracker(STUDY_ID)
+    tracker.record_run(
+        function_name="t_test",
+        parameters={},
+        source_row_ids=[],
+        column_names=[],
+        test_name="t_test",
+        result_id="r_post_hoc",
+        is_pre_registered=False,
+    )
+    entries = tracker.get_all()
+    assert entries[0].is_pre_registered is False
+
+
+def test_empty_tracker():
+    tracker = ProvenanceTracker("nonexistent_study")
+    assert tracker.get_all() == []

tests/test_roadmap_features.py

@@ -0,0 +1,148 @@
+"""Regression tests for classification overrides, Cox checks, and appendix export."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+from pathlib import Path
+
+import pytest
+
+from core.cli.main import cmd_export, cmd_plan
+from core.database import DATA_ROOT, get_connection, init_db
+from core.planning.lock import lock_plan
+from core.planning.study_plan import StudyPlan
+from core.planning.test_selector import check_assumptions
+
+
+STUDY_ID = "test_roadmap_features"
+
+
+@pytest.fixture(autouse=True)
+def _setup():
+    study_path = DATA_ROOT / STUDY_ID
+    if study_path.exists():
+        shutil.rmtree(study_path)
+    study_path.mkdir(parents=True)
+    conn = get_connection(STUDY_ID)
+    init_db(conn)
+    conn.execute(
+        "INSERT INTO studies (id, name, created_at, data_dir, study_type) VALUES (?, ?, ?, ?, ?)",
+        (STUDY_ID, "Roadmap", "2025-01-01", str(study_path), "cohort"),
+    )
+    conn.execute(
+        "CREATE TABLE raw_test_roadmap_features "
+        "(row_id INTEGER PRIMARY KEY, treatment_arm TEXT, response TEXT, "
+        "prior_lines TEXT, pfs_days TEXT, pfs_event TEXT)"
+    )
+    conn.executemany(
+        "INSERT INTO raw_test_roadmap_features "
+        "(treatment_arm, response, prior_lines, pfs_days, pfs_event) VALUES (?, ?, ?, ?, ?)",
+        [("A", "CR", "2", "10", "1"), ("A", "PR", "3", "20", "1"),
+         ("B", "SD", "1", "10", "0"), ("B", "PD", "2", "20", "0")],
+    )
+    conn.executemany(
+        "INSERT INTO variables (id, study_id, column_name, role, data_type) VALUES (?, ?, ?, ?, ?)",
+        [(1, STUDY_ID, "response", "outcome", "categorical"),
+         (2, STUDY_ID, "prior_lines", "baseline", "continuous"),
+         (3, STUDY_ID, "pfs_days", "outcome", "time_to_event"),
+         (4, STUDY_ID, "treatment_arm", "baseline", "categorical")],
+    )
+    conn.commit()
+    conn.close()
+    yield
+    if study_path.exists():
+        shutil.rmtree(study_path)
+
+
+def _plan_args(**kwargs):
+    values = dict(
+        study_id=STUDY_ID,
+        outcome_var_ids="1",
+        tests=["response:chi_square:"],
+        covariates="",
+        study_type="cohort",
+        comparison="response by treatment arm",
+        overrides=[],
+    )
+    values.update(kwargs)
+    return argparse.Namespace(**values)
+
+
+def test_plan_persists_valid_role_override():
+    cmd_plan(_plan_args(overrides=["id=2:role=covariate"]))
+
+    plan = json.loads((DATA_ROOT / STUDY_ID / "study_plan.provisional.json").read_text())
+    assert plan["role_overrides"] == {"2": "covariate"}
+    assert plan["audit"]["role_overrides"] == [{"variable_id": 2, "role": "covariate"}]
+
+
+@pytest.mark.parametrize("override", ["id=99:role=covariate", "id=2:role=banana", "2:role=covariate"])
+def test_plan_rejects_invalid_override(override, capsys):
+    with pytest.raises(SystemExit):
+        cmd_plan(_plan_args(overrides=[override]))
+    assert "override" in capsys.readouterr().err.lower()
+
+
+def test_plan_rejects_override_after_lock(capsys):
+    lock_plan(STUDY_ID, StudyPlan(study_id=STUDY_ID, study_type="cohort"))
+
+    with pytest.raises(SystemExit):
+        cmd_plan(_plan_args(overrides=["id=2:role=covariate"]))
+    assert "locked" in capsys.readouterr().err.lower()
+
+
+def test_cox_assumption_check_warns_on_marginal_event_rate_difference():
+    conn = get_connection(STUDY_ID)
+    masked = f"raw_masked_{STUDY_ID}"
+    conn.execute(f"CREATE TABLE {masked} (row_id INTEGER PRIMARY KEY, pfs_days TEXT, pfs_event TEXT)")
+    conn.execute(
+        f"INSERT INTO {masked} SELECT row_id, pfs_days, pfs_event FROM raw_{STUDY_ID}"
+    )
+    conn.commit()
+    conn.close()
+
+    warnings = check_assumptions(
+        STUDY_ID,
+        [{"variable_name": "pfs_days", "test_name": "cox_proportional_hazards",
+          "n_covariates": 3}],
+    )
+    # 4 rows in test data, 2 events → EPV = 2/4 = 0.5 → warn
+    assert any("EPV" in warning for warning in warnings)
+    assert all("A" not in warning.split(":")[-1] for warning in warnings)
+
+
+def test_plan_persists_cox_warning_for_analysis_enforcement():
+    conn = get_connection(STUDY_ID)
+    masked = f"raw_masked_{STUDY_ID}"
+    conn.execute(f"CREATE TABLE {masked} (row_id INTEGER PRIMARY KEY, pfs_days TEXT, pfs_event TEXT)")
+    conn.execute(f"INSERT INTO {masked} SELECT row_id, pfs_days, pfs_event FROM raw_{STUDY_ID}")
+    conn.commit()
+    conn.close()
+
+    cmd_plan(_plan_args(
+        outcome_var_ids="3",
+        tests=["pfs_days:cox_proportional_hazards:adjusted survival"],
+    ))
+    plan = json.loads((DATA_ROOT / STUDY_ID / "study_plan.provisional.json").read_text())
+    assert "pfs_days" in plan["warnings"]
+
+
+def test_appendix_export_contains_table1_uro_warnings_and_strobe():
+    lock_plan(STUDY_ID, StudyPlan(study_id=STUDY_ID, study_type="cohort"))
+    args = argparse.Namespace(
+        study_id=STUDY_ID,
+        mode="supplementary",
+        study_plan_version=None,
+        format="appendix",
+    )
+    cmd_export(args)
+
+    appendix = DATA_ROOT / STUDY_ID / "study_result.v1.appendix.md"
+    assert appendix.exists()
+    text = appendix.read_text()
+    assert "Table 1" in text
+    assert "URO" in text
+    assert "STROBE" in text
+    assert "Warnings" in text

tests/test_stats_descriptive.py

@@ -0,0 +1,57 @@
+"""Tests for descriptive statistics (Table 1)."""
+
+from __future__ import annotations
+import shutil
+from pathlib import Path
+
+import pytest
+
+from core.database import get_connection, init_db, DATA_ROOT
+from core.stats.descriptive import generate_table1
+
+STUDY_ID = "test_descriptive"
+
+
+@pytest.fixture(autouse=True)
+def _setup():
+    conn = get_connection(STUDY_ID)
+    init_db(conn)
+    conn.execute(
+        "INSERT OR REPLACE INTO studies (id, name, created_at, data_dir) VALUES (?, ?, ?, ?)",
+        (STUDY_ID, "Descriptive Test", "2025-01-01T00:00:00",
+         str(Path("data/studies") / STUDY_ID)),
+    )
+    raw = f"raw_{STUDY_ID}"
+    conn.execute(f"CREATE TABLE IF NOT EXISTS {raw} (row_id INTEGER PRIMARY KEY, age TEXT, sex TEXT, iss_stage TEXT)")
+    for i in range(10):
+        conn.execute(f"INSERT INTO {raw} (age, sex, iss_stage) VALUES (?, ?, ?)",
+                     (str(50 + i * 3), ["M", "F"][i % 2], ["I", "II", "III"][i % 3]))
+    conn.execute("DELETE FROM variables WHERE study_id=?", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'age', 'baseline', 'continuous')", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'sex', 'baseline', 'categorical')", (STUDY_ID,))
+    conn.execute("INSERT INTO variables (study_id, column_name, role, data_type) VALUES (?, 'iss_stage', 'baseline', 'categorical')", (STUDY_ID,))
+    conn.commit()
+    conn.close()
+    yield
+    p = DATA_ROOT / STUDY_ID
+    if p.exists():
+        shutil.rmtree(p)
+
+
+def test_table1_returns_dataframe():
+    tbl = generate_table1(STUDY_ID)
+    assert tbl is not None
+    # Should have rows for each variable
+    assert len(tbl) > 0
+
+
+def test_table1_includes_age():
+    tbl = generate_table1(STUDY_ID)
+    # tableone labels rows with variable names
+    age_rows = tbl.filter(like="age", axis=0)
+    assert len(age_rows) > 0
+
+
+def test_table1_grouped():
+    tbl = generate_table1(STUDY_ID, groupby="sex")
+    assert tbl is not None