research-tool-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/backend/_bootstrap.py +15 -0
- app/backend/exports/verification_script.py +19 -0
- app/backend/main.py +59 -0
- app/backend/routers/__init__.py +1 -0
- app/backend/routers/execution.py +478 -0
- app/backend/routers/ingestion.py +233 -0
- app/backend/routers/planning.py +265 -0
- app/backend/routers/reporting.py +531 -0
- app/backend/state.py +44 -0
- core/__init__.py +0 -0
- core/cli/__init__.py +0 -0
- core/cli/main.py +1705 -0
- core/database.py +62 -0
- core/ingestion/__init__.py +0 -0
- core/ingestion/csv_loader.py +191 -0
- core/ingestion/variable_classifier.py +171 -0
- core/masking/__init__.py +0 -0
- core/masking/gate.py +128 -0
- core/models.py +138 -0
- core/planning/__init__.py +0 -0
- core/planning/diagnostics.py +89 -0
- core/planning/lock.py +232 -0
- core/planning/study_plan.py +73 -0
- core/planning/test_selector.py +518 -0
- core/provenance/__init__.py +0 -0
- core/provenance/hashing.py +38 -0
- core/provenance/tracker.py +105 -0
- core/reporting/__init__.py +62 -0
- core/reporting/appendix.py +58 -0
- core/reporting/bundle.py +378 -0
- core/reporting/excel_export.py +683 -0
- core/reporting/flowchart/__init__.py +20 -0
- core/reporting/flowchart/flowchart.py +511 -0
- core/reporting/forensics.py +592 -0
- core/reporting/forest_plot.py +614 -0
- core/reporting/lineage.py +562 -0
- core/reporting/manuscript_draft.py +726 -0
- core/reporting/plots.py +568 -0
- core/reporting/strobe_checklist.py +460 -0
- core/stats/__init__.py +0 -0
- core/stats/descriptive.py +104 -0
- core/stats/inferential.py +540 -0
- core/stats/multiple_comparisons.py +62 -0
- core/stats/post_hoc.py +62 -0
- exports/verification_script.py +19 -0
- research_tool_cli-0.1.0.dist-info/METADATA +16 -0
- research_tool_cli-0.1.0.dist-info/RECORD +93 -0
- research_tool_cli-0.1.0.dist-info/WHEEL +5 -0
- research_tool_cli-0.1.0.dist-info/entry_points.txt +2 -0
- research_tool_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- research_tool_cli-0.1.0.dist-info/top_level.txt +4 -0
- tests/__init__.py +0 -0
- tests/conftest.py +45 -0
- tests/test_amendments.py +296 -0
- tests/test_analyze_batch_robustness.py +162 -0
- tests/test_app_statistical_reporting.py +383 -0
- tests/test_benchmark_21.py +556 -0
- tests/test_bundle.py +277 -0
- tests/test_cox_ph_plan.py +498 -0
- tests/test_csv_loader.py +368 -0
- tests/test_end_to_end.py +302 -0
- tests/test_excel_export.py +164 -0
- tests/test_flowchart.py +244 -0
- tests/test_forensics.py +305 -0
- tests/test_forest_plot.py +374 -0
- tests/test_from_json.py +176 -0
- tests/test_latest_plan_version.py +164 -0
- tests/test_lineage.py +329 -0
- tests/test_lock_immutability.py +133 -0
- tests/test_m1_fk_violation.py +85 -0
- tests/test_m2_data_hash_consistency.py +40 -0
- tests/test_m3_correction_timing.py +59 -0
- tests/test_m4_dedup_field.py +59 -0
- tests/test_m5_excel_hash_scope.py +45 -0
- tests/test_m6_fisher_exact_naming.py +44 -0
- tests/test_m7_duplicate_study_plan.py +55 -0
- tests/test_m8_filter_superseded.py +58 -0
- tests/test_m9_table1_groupby.py +56 -0
- tests/test_manuscript_draft.py +289 -0
- tests/test_masking_gate.py +196 -0
- tests/test_masking_migration.py +111 -0
- tests/test_multiple_comparisons.py +56 -0
- tests/test_plan_validation.py +289 -0
- tests/test_plots.py +394 -0
- tests/test_post_hoc_tagging.py +49 -0
- tests/test_posthoc_analyze.py +203 -0
- tests/test_provenance_tracker.py +110 -0
- tests/test_roadmap_features.py +148 -0
- tests/test_stats_descriptive.py +57 -0
- tests/test_stats_inferential.py +380 -0
- tests/test_strobe_checklist.py +172 -0
- tests/test_test_selector.py +350 -0
- tests/test_variable_classifier.py +124 -0
tests/test_forensics.py
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""Tests for forensics anomaly detection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import shutil
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
|
|
11
|
+
from core.database import DATA_ROOT, get_connection, init_db
|
|
12
|
+
from core.reporting.forensics import (
|
|
13
|
+
_find_time_pairs,
|
|
14
|
+
_is_benford_eligible,
|
|
15
|
+
_first_digit_dist,
|
|
16
|
+
_get_numeric_values,
|
|
17
|
+
_check_impossible_timelines,
|
|
18
|
+
run_forensics,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# ── Helpers ──────────────────────────────────────────────────────────────────
|
|
23
|
+
|
|
24
|
+
def _make_study(study_id: str, rows: list[dict], col_info: dict | None = None):
|
|
25
|
+
"""Create a study with a raw table and variable metadata."""
|
|
26
|
+
study_dir = DATA_ROOT / study_id
|
|
27
|
+
if study_dir.exists():
|
|
28
|
+
shutil.rmtree(study_dir)
|
|
29
|
+
|
|
30
|
+
conn = get_connection(study_id)
|
|
31
|
+
init_db(conn)
|
|
32
|
+
|
|
33
|
+
conn.execute(
|
|
34
|
+
"""INSERT INTO studies (id, name, created_at, data_dir, is_locked, unmasked_at)
|
|
35
|
+
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
36
|
+
(study_id, "test", "2026-01-01T00:00:00+00:00", str(study_dir), 2,
|
|
37
|
+
"2026-07-01T00:00:00+00:00"),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
# Create raw table
|
|
41
|
+
if rows:
|
|
42
|
+
cols = list(rows[0].keys())
|
|
43
|
+
col_defs = ", ".join(f'"{c}" TEXT' for c in cols)
|
|
44
|
+
conn.execute(f"""
|
|
45
|
+
CREATE TABLE IF NOT EXISTS raw_{study_id} (
|
|
46
|
+
"row_id" INTEGER PRIMARY KEY,
|
|
47
|
+
{col_defs}
|
|
48
|
+
)
|
|
49
|
+
""")
|
|
50
|
+
col_list = ", ".join(f'"{c}"' for c in cols)
|
|
51
|
+
for r in rows:
|
|
52
|
+
placeholders = ", ".join("?" for _ in cols)
|
|
53
|
+
conn.execute(
|
|
54
|
+
f"INSERT INTO raw_{study_id} ({col_list}) "
|
|
55
|
+
f"VALUES ({placeholders})",
|
|
56
|
+
[str(r[c]) for c in cols],
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
# Variable metadata
|
|
60
|
+
if col_info:
|
|
61
|
+
for col_name, info in col_info.items():
|
|
62
|
+
conn.execute(
|
|
63
|
+
"""INSERT INTO variables (study_id, column_name, role, data_type, is_masked)
|
|
64
|
+
VALUES (?, ?, ?, ?, ?)""",
|
|
65
|
+
(study_id, col_name, info.get("role", "baseline"),
|
|
66
|
+
info.get("data_type", "continuous"), info.get("is_masked", 0)),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
conn.commit()
|
|
70
|
+
conn.close()
|
|
71
|
+
return study_dir
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# ── Tests ────────────────────────────────────────────────────────────────────
|
|
75
|
+
|
|
76
|
+
class TestImpossibleTimelines:
|
|
77
|
+
"""Tier 1: PFS > OS detection."""
|
|
78
|
+
|
|
79
|
+
def test_catches_pfs_exceeds_os(self):
|
|
80
|
+
"""A deliberately broken row where PFS > OS must be flagged."""
|
|
81
|
+
study_id = "test_forensics_timeline_break"
|
|
82
|
+
col_info = {
|
|
83
|
+
"pfs_days": {"role": "outcome", "data_type": "time_to_event"},
|
|
84
|
+
"os_days": {"role": "outcome", "data_type": "time_to_event"},
|
|
85
|
+
}
|
|
86
|
+
rows = [
|
|
87
|
+
{"pfs_days": "100", "os_days": "500"},
|
|
88
|
+
{"pfs_days": "600", "os_days": "200"},
|
|
89
|
+
{"pfs_days": "300", "os_days": "400"},
|
|
90
|
+
]
|
|
91
|
+
_make_study(study_id, rows, col_info)
|
|
92
|
+
|
|
93
|
+
conn = get_connection(study_id)
|
|
94
|
+
lines: list[str] = []
|
|
95
|
+
_check_impossible_timelines(
|
|
96
|
+
conn, f"raw_{study_id}", f"raw_masked_{study_id}",
|
|
97
|
+
col_info, 3, True, lines,
|
|
98
|
+
)
|
|
99
|
+
conn.close()
|
|
100
|
+
|
|
101
|
+
combined = "\n".join(lines)
|
|
102
|
+
assert "⚠" in combined, "Should flag PFS > OS"
|
|
103
|
+
assert "pfs_days=600" in combined or "600" in combined
|
|
104
|
+
assert "os_days=200" in combined or "200" in combined
|
|
105
|
+
|
|
106
|
+
shutil.rmtree(DATA_ROOT / study_id, ignore_errors=True)
|
|
107
|
+
|
|
108
|
+
def test_no_false_positive_when_valid(self):
|
|
109
|
+
"""All PFS ≤ OS must not raise flags."""
|
|
110
|
+
study_id = "test_forensics_timeline_ok"
|
|
111
|
+
col_info = {
|
|
112
|
+
"pfs_days": {"role": "outcome", "data_type": "time_to_event"},
|
|
113
|
+
"os_days": {"role": "outcome", "data_type": "time_to_event"},
|
|
114
|
+
}
|
|
115
|
+
rows = [
|
|
116
|
+
{"pfs_days": "100", "os_days": "500"},
|
|
117
|
+
{"pfs_days": "200", "os_days": "300"},
|
|
118
|
+
{"pfs_days": "50", "os_days": "600"},
|
|
119
|
+
]
|
|
120
|
+
_make_study(study_id, rows, col_info)
|
|
121
|
+
|
|
122
|
+
conn = get_connection(study_id)
|
|
123
|
+
lines: list[str] = []
|
|
124
|
+
_check_impossible_timelines(
|
|
125
|
+
conn, f"raw_{study_id}", f"raw_masked_{study_id}",
|
|
126
|
+
col_info, 3, True, lines,
|
|
127
|
+
)
|
|
128
|
+
conn.close()
|
|
129
|
+
|
|
130
|
+
combined = "\n".join(lines)
|
|
131
|
+
assert "✓" in combined
|
|
132
|
+
assert "⚠" not in combined
|
|
133
|
+
|
|
134
|
+
shutil.rmtree(DATA_ROOT / study_id, ignore_errors=True)
|
|
135
|
+
|
|
136
|
+
def test_no_time_pair_returns_early(self):
|
|
137
|
+
"""No time_to_event columns must produce a clean skip."""
|
|
138
|
+
col_info = {
|
|
139
|
+
"age": {"role": "baseline", "data_type": "continuous"},
|
|
140
|
+
"sex": {"role": "baseline", "data_type": "categorical"},
|
|
141
|
+
}
|
|
142
|
+
pairs = _find_time_pairs(col_info)
|
|
143
|
+
assert pairs == []
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class TestBenfordEligibility:
|
|
147
|
+
"""Tier 2: Benford's Law eligibility gate."""
|
|
148
|
+
|
|
149
|
+
def test_eligible_multiple_orders(self):
|
|
150
|
+
"""Values spanning >2 orders with sufficient N are eligible."""
|
|
151
|
+
vals = [10 ** (i / 10) * 100 for i in range(60)] # 100 to ~1M
|
|
152
|
+
eligible, reason = _is_benford_eligible(vals, len(vals))
|
|
153
|
+
assert eligible, f"Should be eligible: {reason}"
|
|
154
|
+
|
|
155
|
+
def test_refuses_bounded_column(self):
|
|
156
|
+
"""Bounded narrow-range column (<2 orders) must be rejected."""
|
|
157
|
+
vals = [float(i) for i in range(40, 81)] # 40–80, <2 orders
|
|
158
|
+
eligible, reason = _is_benford_eligible(vals, len(vals))
|
|
159
|
+
assert not eligible, f"Should be rejected: {reason}"
|
|
160
|
+
assert "100" in reason or "order" in reason
|
|
161
|
+
|
|
162
|
+
def test_refuses_small_n(self):
|
|
163
|
+
"""N < 30 must be rejected."""
|
|
164
|
+
vals = [10.0, 50.0, 200.0, 800.0, 3000.0]
|
|
165
|
+
eligible, reason = _is_benford_eligible(vals, len(vals))
|
|
166
|
+
assert not eligible
|
|
167
|
+
assert "Sample too small" in reason
|
|
168
|
+
|
|
169
|
+
def test_refuses_negative_values(self):
|
|
170
|
+
"""Non-positive values must be rejected."""
|
|
171
|
+
vals = [1.0, 10.0, 100.0, -5.0, 1000.0] * 10 # N=50, wide range
|
|
172
|
+
eligible, reason = _is_benford_eligible(vals, len(vals))
|
|
173
|
+
assert not eligible
|
|
174
|
+
assert "non-positive" in reason
|
|
175
|
+
|
|
176
|
+
def test_refuses_too_few_unique(self):
|
|
177
|
+
"""Too few unique values must be rejected."""
|
|
178
|
+
vals = [10.0] * 5 + [100.0] * 5 + [200.0] * 5 + [500.0] * 5 + [1000.0] * 5
|
|
179
|
+
eligible, reason = _is_benford_eligible(vals, len(vals))
|
|
180
|
+
# N=25 is already too small, so this will fail the N check
|
|
181
|
+
assert not eligible
|
|
182
|
+
|
|
183
|
+
def test_marginal_n_below_50_caveated(self):
|
|
184
|
+
"""30 ≤ N < 50 should be eligible but caveated."""
|
|
185
|
+
vals = [10 ** (i / 10) * 10 for i in range(40)] # 10–10^4, 40 vals
|
|
186
|
+
eligible, reason = _is_benford_eligible(vals, len(vals))
|
|
187
|
+
assert eligible, f"Should be eligible: {reason}"
|
|
188
|
+
assert "Marginal" in reason
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class TestFullReportOnSynthetic:
|
|
192
|
+
"""Full report against synthetic_21.csv constraints."""
|
|
193
|
+
|
|
194
|
+
def test_synthetic_21_caveats_benford(self):
|
|
195
|
+
"""synthetic_21.csv has N=21 — Benford section must caveat sample size."""
|
|
196
|
+
# Build the synthetic_21 schema
|
|
197
|
+
col_info = {
|
|
198
|
+
"patient_id": {"role": "baseline", "data_type": "categorical"},
|
|
199
|
+
"age": {"role": "baseline", "data_type": "continuous"},
|
|
200
|
+
"sex": {"role": "baseline", "data_type": "categorical"},
|
|
201
|
+
"iss_stage": {"role": "baseline", "data_type": "categorical"},
|
|
202
|
+
"prior_lines": {"role": "baseline", "data_type": "continuous"},
|
|
203
|
+
"high_risk_fish": {"role": "baseline", "data_type": "categorical"},
|
|
204
|
+
"treatment_arm": {"role": "baseline", "data_type": "categorical"},
|
|
205
|
+
"response_category": {"role": "outcome", "data_type": "categorical"},
|
|
206
|
+
"pfs_days": {"role": "outcome", "data_type": "time_to_event"},
|
|
207
|
+
"pfs_event": {"role": "outcome", "data_type": "categorical"},
|
|
208
|
+
"os_days": {"role": "outcome", "data_type": "time_to_event"},
|
|
209
|
+
"os_event": {"role": "outcome", "data_type": "categorical"},
|
|
210
|
+
}
|
|
211
|
+
rows = [
|
|
212
|
+
{"patient_id": f"SUBJ_{i:03d}", "age": str(40 + (i * 2) % 44),
|
|
213
|
+
"sex": "M" if i % 2 == 0 else "F",
|
|
214
|
+
"iss_stage": ["I", "II", "III"][i % 3],
|
|
215
|
+
"prior_lines": str(i % 5),
|
|
216
|
+
"high_risk_fish": "yes" if i % 3 == 0 else "no",
|
|
217
|
+
"treatment_arm": "A" if i % 2 == 0 else "B",
|
|
218
|
+
"response_category": ["CR", "PR", "SD", "PD"][i % 4],
|
|
219
|
+
"pfs_days": str(30 + i * 30),
|
|
220
|
+
"pfs_event": "1" if i % 3 != 0 else "0",
|
|
221
|
+
"os_days": str(30 + i * 60),
|
|
222
|
+
"os_event": "1" if i % 2 == 0 else "0"}
|
|
223
|
+
for i in range(21)
|
|
224
|
+
]
|
|
225
|
+
study_id = "test_forensics_synth21"
|
|
226
|
+
_make_study(study_id, rows, col_info)
|
|
227
|
+
|
|
228
|
+
report = run_forensics(study_id)
|
|
229
|
+
assert report.exists()
|
|
230
|
+
text = report.read_text()
|
|
231
|
+
|
|
232
|
+
# Benford column should say "too small"
|
|
233
|
+
assert "Sample too small" in text
|
|
234
|
+
# Should still produce a complete report
|
|
235
|
+
assert "Forensics Report" in text
|
|
236
|
+
assert "Tier 1" in text
|
|
237
|
+
assert "Tier 2" in text
|
|
238
|
+
assert "## Summary" in text or "Summary of Checks" in text
|
|
239
|
+
|
|
240
|
+
shutil.rmtree(DATA_ROOT / study_id, ignore_errors=True)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class TestOutOfRange:
|
|
244
|
+
"""Tier 1: out-of-range value detection."""
|
|
245
|
+
|
|
246
|
+
def test_age_out_of_range(self):
|
|
247
|
+
"""Age > 120 must be flagged."""
|
|
248
|
+
study_id = "test_forensics_oor"
|
|
249
|
+
col_info = {
|
|
250
|
+
"age": {"role": "baseline", "data_type": "continuous"},
|
|
251
|
+
}
|
|
252
|
+
rows = [{"age": str(v)} for v in [45, 130, 80, -5, 55]]
|
|
253
|
+
_make_study(study_id, rows, col_info)
|
|
254
|
+
|
|
255
|
+
report = run_forensics(study_id)
|
|
256
|
+
text = report.read_text()
|
|
257
|
+
assert "⚠" in text
|
|
258
|
+
assert "age" in text
|
|
259
|
+
assert "130" in text or "130" in text
|
|
260
|
+
assert "-5" in text
|
|
261
|
+
|
|
262
|
+
shutil.rmtree(DATA_ROOT / study_id, ignore_errors=True)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
class TestDuplicates:
|
|
266
|
+
"""Tier 1: duplicate detection."""
|
|
267
|
+
|
|
268
|
+
def test_exact_duplicate_flagged(self):
|
|
269
|
+
study_id = "test_forensics_dup_exact"
|
|
270
|
+
col_info = {
|
|
271
|
+
"patient_id": {"role": "baseline", "data_type": "categorical"},
|
|
272
|
+
"age": {"role": "baseline", "data_type": "continuous"},
|
|
273
|
+
"sex": {"role": "baseline", "data_type": "categorical"},
|
|
274
|
+
}
|
|
275
|
+
rows = [
|
|
276
|
+
{"patient_id": "A", "age": "50", "sex": "M"},
|
|
277
|
+
{"patient_id": "A", "age": "50", "sex": "M"},
|
|
278
|
+
]
|
|
279
|
+
_make_study(study_id, rows, col_info)
|
|
280
|
+
|
|
281
|
+
report = run_forensics(study_id)
|
|
282
|
+
text = report.read_text()
|
|
283
|
+
assert "⚠" in text
|
|
284
|
+
assert "exact duplicate" in text
|
|
285
|
+
|
|
286
|
+
shutil.rmtree(DATA_ROOT / study_id, ignore_errors=True)
|
|
287
|
+
|
|
288
|
+
def test_near_duplicate_conflict_flagged(self):
|
|
289
|
+
study_id = "test_forensics_dup_near"
|
|
290
|
+
col_info = {
|
|
291
|
+
"age": {"role": "baseline", "data_type": "continuous"},
|
|
292
|
+
"sex": {"role": "baseline", "data_type": "categorical"},
|
|
293
|
+
}
|
|
294
|
+
rows = [
|
|
295
|
+
{"patient_id": "A", "age": "50", "sex": "M"},
|
|
296
|
+
{"patient_id": "A", "age": "51", "sex": "M"},
|
|
297
|
+
]
|
|
298
|
+
_make_study(study_id, rows, col_info)
|
|
299
|
+
|
|
300
|
+
report = run_forensics(study_id)
|
|
301
|
+
text = report.read_text()
|
|
302
|
+
assert "conflicting" in text
|
|
303
|
+
assert "age" in text
|
|
304
|
+
|
|
305
|
+
shutil.rmtree(DATA_ROOT / study_id, ignore_errors=True)
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import shutil
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from core.database import DATA_ROOT, get_connection, init_db
|
|
8
|
+
from core.reporting.forest_plot import (
|
|
9
|
+
CovariateRow,
|
|
10
|
+
ForestPlotData,
|
|
11
|
+
load_forest_data,
|
|
12
|
+
render_svg,
|
|
13
|
+
render_ascii,
|
|
14
|
+
_extract_epv,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# ── Fixtures ─────────────────────────────────────────────────────────────────
|
|
19
|
+
|
|
20
|
+
def _make_study(study_id: str, covariates: list[dict],
|
|
21
|
+
concordance: float = 0.72, lr_p: float = 0.01,
|
|
22
|
+
n: int = 200, epv_text: str | None = None):
|
|
23
|
+
study_dir = DATA_ROOT / study_id
|
|
24
|
+
if study_dir.exists():
|
|
25
|
+
shutil.rmtree(study_dir)
|
|
26
|
+
study_dir.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
|
|
28
|
+
conn = get_connection(study_id)
|
|
29
|
+
init_db(conn)
|
|
30
|
+
conn.execute(
|
|
31
|
+
"""INSERT INTO studies (id, name, created_at, data_dir, is_locked, unmasked_at)
|
|
32
|
+
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
33
|
+
(study_id, "test", "2026-01-01T00:00:00+00:00", str(study_dir), 2, "2026-07-01T00:00:00+00:00"),
|
|
34
|
+
)
|
|
35
|
+
conn.execute(
|
|
36
|
+
"""INSERT INTO analysis_results
|
|
37
|
+
(study_id, test_name, computed_at, concordance_index, lr_test_p,
|
|
38
|
+
is_pre_registered, study_plan_version, status_json,
|
|
39
|
+
sample_counts_json, provenance_json)
|
|
40
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
41
|
+
(study_id, "cox_ph_model", "2026-07-01T00:00:00+00:00",
|
|
42
|
+
concordance, lr_p, 1, 1,
|
|
43
|
+
'{"status": "completed"}',
|
|
44
|
+
f'{{"n_total": {n}, "n_analyzed": {n}, "n_excluded": 0}}',
|
|
45
|
+
'{}'),
|
|
46
|
+
)
|
|
47
|
+
result_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
|
|
48
|
+
|
|
49
|
+
for i, c in enumerate(covariates):
|
|
50
|
+
conn.execute(
|
|
51
|
+
"""INSERT INTO analysis_covariate_results
|
|
52
|
+
(result_id, covariate, hr, ci_lower, ci_upper, wald_p, coef, se, z,
|
|
53
|
+
reference_level, tested_level)
|
|
54
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
55
|
+
(result_id, c["covariate"], c["hr"], c["ci_lower"], c["ci_upper"],
|
|
56
|
+
c["wald_p"], c.get("coef"), c.get("se"), c.get("z"),
|
|
57
|
+
c.get("reference_level"), c.get("tested_level")),
|
|
58
|
+
)
|
|
59
|
+
conn.commit()
|
|
60
|
+
conn.close()
|
|
61
|
+
|
|
62
|
+
# Write plan with EPV warning
|
|
63
|
+
plan = {}
|
|
64
|
+
if epv_text:
|
|
65
|
+
plan["warnings"] = {"cox_model": epv_text}
|
|
66
|
+
import json
|
|
67
|
+
(study_dir / "study_plan.v1.locked.json").write_text(json.dumps(plan))
|
|
68
|
+
|
|
69
|
+
return study_id, result_id
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# ── Tests ────────────────────────────────────────────────────────────────────
|
|
73
|
+
|
|
74
|
+
class TestExtractEPV:
|
|
75
|
+
def test_extracts_epv_from_warning(self):
|
|
76
|
+
epv, warn = _extract_epv('{"test": "EPV=2.5"}')
|
|
77
|
+
assert epv == 2.5
|
|
78
|
+
assert warn is True
|
|
79
|
+
|
|
80
|
+
def test_high_epv_no_warning(self):
|
|
81
|
+
epv, warn = _extract_epv('{"test": "EPV=25.0"}')
|
|
82
|
+
assert epv == 25.0
|
|
83
|
+
assert warn is False
|
|
84
|
+
|
|
85
|
+
def test_no_epv_string(self):
|
|
86
|
+
epv, warn = _extract_epv('{"test": "no epv here"}')
|
|
87
|
+
assert epv is None
|
|
88
|
+
assert warn is False
|
|
89
|
+
|
|
90
|
+
def test_none_input(self):
|
|
91
|
+
epv, warn = _extract_epv(None)
|
|
92
|
+
assert epv is None
|
|
93
|
+
assert warn is False
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class TestCovariateRowProperties:
|
|
97
|
+
def test_ci_crosses_one_detection(self):
|
|
98
|
+
def make(hr, lo, hi, p):
|
|
99
|
+
return CovariateRow("x", hr, lo, hi, p, coef=None, se=None, z=None)
|
|
100
|
+
assert make(1.5, 0.8, 2.2, 0.1).ci_crosses_one is True # crosses 1
|
|
101
|
+
assert make(2.5, 1.2, 4.0, 0.01).ci_crosses_one is False # entirely above 1
|
|
102
|
+
assert make(0.5, 0.3, 0.9, 0.02).ci_crosses_one is False # entirely below 1
|
|
103
|
+
assert make(0.5, 0.3, 1.0, 0.01).ci_crosses_one is True # upper bound = 1
|
|
104
|
+
assert make(1.0, 0.8, 1.2, 0.05).ci_crosses_one is True # HR exactly 1
|
|
105
|
+
assert make(0.8, 0.6, 1.2, 0.1).ci_crosses_one is True # crosses 1
|
|
106
|
+
assert make(0.4, 0.3, 0.5, 0.001).ci_crosses_one is False # entirely below 1
|
|
107
|
+
|
|
108
|
+
def test_unstable_flag(self):
|
|
109
|
+
c = CovariateRow("x", 0, 0, 0, 1.0, coef=None, se=None, z=None, unstable=True)
|
|
110
|
+
assert c.unstable is True
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class TestAgainstRealStudy:
|
|
114
|
+
"""Test against 4-covariate myeloma cohort study with EPV=2.5."""
|
|
115
|
+
|
|
116
|
+
def setup_method(self):
|
|
117
|
+
self.sid = "test_forest_real_study"
|
|
118
|
+
covariates = [
|
|
119
|
+
{"covariate": "treatment_arm", "hr": 1.7029, "ci_lower": 0.2402,
|
|
120
|
+
"ci_upper": 12.0733, "wald_p": 0.5943, "reference_level": "A", "tested_level": "B"},
|
|
121
|
+
{"covariate": "age", "hr": 0.9934, "ci_lower": 0.9199,
|
|
122
|
+
"ci_upper": 1.0727, "wald_p": 0.8656},
|
|
123
|
+
{"covariate": "high_risk_fish", "hr": 0.3455, "ci_lower": 0.0606,
|
|
124
|
+
"ci_upper": 1.9711, "wald_p": 0.2316},
|
|
125
|
+
{"covariate": "prior_lines", "hr": 0.8521, "ci_lower": 0.5412,
|
|
126
|
+
"ci_upper": 1.3415, "wald_p": 0.4870},
|
|
127
|
+
]
|
|
128
|
+
_make_study(
|
|
129
|
+
self.sid, covariates, n=21,
|
|
130
|
+
epv_text="cox_ph_model pfs_model: 10 events across 4 predictor(s) (EPV=2.5). Cox models are unreliable below 10 events per predictor."
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
def teardown_method(self):
|
|
134
|
+
shutil.rmtree(DATA_ROOT / self.sid, ignore_errors=True)
|
|
135
|
+
|
|
136
|
+
def test_loads_four_covariates(self):
|
|
137
|
+
"""Must load exactly 4 covariates with correct display labels."""
|
|
138
|
+
data = load_forest_data(self.sid)
|
|
139
|
+
assert len(data.covariates) == 4
|
|
140
|
+
labels = [c.display_label for c in data.covariates]
|
|
141
|
+
assert any("Treatment Group" in l for l in labels)
|
|
142
|
+
assert any("Age, years" in l for l in labels)
|
|
143
|
+
assert any("High-Risk Cytogenetics" in l for l in labels)
|
|
144
|
+
assert any("Prior Lines of Therapy" in l for l in labels)
|
|
145
|
+
|
|
146
|
+
def test_values_match_db(self):
|
|
147
|
+
"""HR/CI/p values must match DB exactly — pull and compare."""
|
|
148
|
+
data = load_forest_data(self.sid)
|
|
149
|
+
conn = get_connection(self.sid)
|
|
150
|
+
rows = conn.execute(
|
|
151
|
+
"SELECT covariate, hr, ci_lower, ci_upper, wald_p, reference_level, tested_level "
|
|
152
|
+
"FROM analysis_covariate_results "
|
|
153
|
+
"WHERE result_id=(SELECT id FROM analysis_results "
|
|
154
|
+
" WHERE study_id=? AND test_name='cox_ph_model' "
|
|
155
|
+
" AND id NOT IN (SELECT COALESCE(superseded_previous_result_id, -1) "
|
|
156
|
+
" FROM analysis_results WHERE study_id=? AND test_name='cox_ph_model' "
|
|
157
|
+
" AND superseded_previous_result_id IS NOT NULL) "
|
|
158
|
+
" ORDER BY id DESC LIMIT 1)",
|
|
159
|
+
(self.sid,) * 2,
|
|
160
|
+
).fetchall()
|
|
161
|
+
conn.close()
|
|
162
|
+
db_map = {r["covariate"]: r for r in rows}
|
|
163
|
+
for c in data.covariates:
|
|
164
|
+
db = db_map[c.covariate]
|
|
165
|
+
assert c.hr == db["hr"]
|
|
166
|
+
assert c.ci_lower == db["ci_lower"]
|
|
167
|
+
assert c.ci_upper == db["ci_upper"]
|
|
168
|
+
assert c.wald_p == db["wald_p"]
|
|
169
|
+
assert c.reference_level == db["reference_level"]
|
|
170
|
+
assert c.tested_level == db["tested_level"]
|
|
171
|
+
|
|
172
|
+
def test_epv_caveat_present(self):
|
|
173
|
+
"""Study has EPV=2.5 — report must carry the caveat."""
|
|
174
|
+
data = load_forest_data(self.sid)
|
|
175
|
+
assert data.epv_warning is True
|
|
176
|
+
assert data.epv is not None
|
|
177
|
+
assert data.epv < 10
|
|
178
|
+
|
|
179
|
+
def test_all_cis_cross_one(self):
|
|
180
|
+
"""All 4 covariates in this study have CI crossing 1."""
|
|
181
|
+
data = load_forest_data(self.sid)
|
|
182
|
+
for c in data.covariates:
|
|
183
|
+
assert c.ci_crosses_one, f"{c.covariate} CI does not cross 1"
|
|
184
|
+
|
|
185
|
+
def test_svg_renders_with_epv_caveat(self):
|
|
186
|
+
"""SVG output must contain the EPV caveat text."""
|
|
187
|
+
data = load_forest_data(self.sid)
|
|
188
|
+
out = "/tmp/test_forest_real.svg"
|
|
189
|
+
render_svg(data, out)
|
|
190
|
+
svg = Path(out).read_text()
|
|
191
|
+
assert "EPV=2.5" in svg
|
|
192
|
+
assert "Caution" in svg
|
|
193
|
+
assert "unstable" in svg
|
|
194
|
+
Path(out).unlink(missing_ok=True)
|
|
195
|
+
|
|
196
|
+
def test_ascii_renders_with_epv_caveat(self):
|
|
197
|
+
"""ASCII output must contain the EPV caveat text."""
|
|
198
|
+
data = load_forest_data(self.sid)
|
|
199
|
+
text = render_ascii(data)
|
|
200
|
+
assert "EPV=2.5" in text
|
|
201
|
+
assert "Caution" in text
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
class TestHighEPVNoCaveat:
|
|
205
|
+
"""Constructed study with high EPV — caveat must NOT appear."""
|
|
206
|
+
|
|
207
|
+
def setup_method(self):
|
|
208
|
+
self.sid = "test_forest_high_epv"
|
|
209
|
+
covariates = [
|
|
210
|
+
{"covariate": "treatment_arm", "hr": 2.0, "ci_lower": 1.2,
|
|
211
|
+
"ci_upper": 3.3, "wald_p": 0.008, "coef": 0.69, "se": 0.26, "z": 2.65,
|
|
212
|
+
"reference_level": "A", "tested_level": "B"},
|
|
213
|
+
{"covariate": "age", "hr": 1.05, "ci_lower": 1.01,
|
|
214
|
+
"ci_upper": 1.09, "wald_p": 0.015, "coef": 0.05, "se": 0.02, "z": 2.43},
|
|
215
|
+
]
|
|
216
|
+
_make_study(self.sid, covariates, n=500,
|
|
217
|
+
epv_text="EPV=50.0 — adequate")
|
|
218
|
+
self.data = load_forest_data(self.sid)
|
|
219
|
+
|
|
220
|
+
def teardown_method(self):
|
|
221
|
+
shutil.rmtree(DATA_ROOT / self.sid, ignore_errors=True)
|
|
222
|
+
|
|
223
|
+
def test_epv_warning_false(self):
|
|
224
|
+
assert self.data.epv_warning is False
|
|
225
|
+
assert self.data.epv == 50.0
|
|
226
|
+
|
|
227
|
+
def test_svg_no_caveat(self):
|
|
228
|
+
out = "/tmp/test_forest_high_epv.svg"
|
|
229
|
+
render_svg(self.data, out)
|
|
230
|
+
svg = Path(out).read_text()
|
|
231
|
+
assert "Caution" not in svg
|
|
232
|
+
Path(out).unlink(missing_ok=True)
|
|
233
|
+
|
|
234
|
+
def test_ascii_no_caveat(self):
|
|
235
|
+
text = render_ascii(self.data)
|
|
236
|
+
assert "Caution" not in text
|
|
237
|
+
|
|
238
|
+
def test_some_cis_dont_cross_one(self):
|
|
239
|
+
assert self.data.covariates[0].ci_crosses_one is False
|
|
240
|
+
assert self.data.covariates[1].ci_crosses_one is False
|
|
241
|
+
|
|
242
|
+
def test_concordance_loaded(self):
|
|
243
|
+
assert self.data.concordance_index == 0.72
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class TestUnstableCovariate:
|
|
247
|
+
"""Row with inf/nan HR should render as 'did not converge'."""
|
|
248
|
+
|
|
249
|
+
def setup_method(self):
|
|
250
|
+
self.sid = "test_forest_unstable"
|
|
251
|
+
covariates = [
|
|
252
|
+
{"covariate": "stable_var", "hr": 1.5, "ci_lower": 0.8,
|
|
253
|
+
"ci_upper": 2.8, "wald_p": 0.2, "coef": 0.4, "se": 0.3, "z": 1.33,
|
|
254
|
+
"reference_level": "A", "tested_level": "B"},
|
|
255
|
+
{"covariate": "unstable_var", "hr": float("inf"), "ci_lower": 0.5,
|
|
256
|
+
"ci_upper": float("inf"), "wald_p": 0.5, "coef": None, "se": None, "z": None,
|
|
257
|
+
"reference_level": "A", "tested_level": "B"},
|
|
258
|
+
]
|
|
259
|
+
_make_study(self.sid, covariates, epv_text="EPV=25.0")
|
|
260
|
+
self.data = load_forest_data(self.sid)
|
|
261
|
+
|
|
262
|
+
def teardown_method(self):
|
|
263
|
+
shutil.rmtree(DATA_ROOT / self.sid, ignore_errors=True)
|
|
264
|
+
|
|
265
|
+
def test_unstable_flagged(self):
|
|
266
|
+
assert self.data.covariates[1].unstable is True
|
|
267
|
+
|
|
268
|
+
def test_stable_not_flagged(self):
|
|
269
|
+
assert self.data.covariates[0].unstable is False
|
|
270
|
+
|
|
271
|
+
def test_svg_shows_unstable_text(self):
|
|
272
|
+
out = "/tmp/test_forest_unstable.svg"
|
|
273
|
+
render_svg(self.data, out)
|
|
274
|
+
svg = Path(out).read_text()
|
|
275
|
+
assert "did not converge" in svg.lower()
|
|
276
|
+
Path(out).unlink(missing_ok=True)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
class TestWideHRAlignment:
|
|
281
|
+
"""When HR or CI bounds are ≥10.00, columns must stay aligned."""
|
|
282
|
+
|
|
283
|
+
def setup_method(self):
|
|
284
|
+
self.sid = "test_forest_wide_hr"
|
|
285
|
+
covariates = [
|
|
286
|
+
{
|
|
287
|
+
"covariate": "treatment_arm",
|
|
288
|
+
"hr": 12.45, "ci_lower": 1.02, "ci_upper": 145.30,
|
|
289
|
+
"wald_p": 0.041, "coef": 2.52, "se": 1.20, "z": 2.10,
|
|
290
|
+
"reference_level": "A", "tested_level": "B",
|
|
291
|
+
},
|
|
292
|
+
{
|
|
293
|
+
"covariate": "age",
|
|
294
|
+
"hr": 1.01, "ci_lower": 0.98, "ci_upper": 1.04,
|
|
295
|
+
"wald_p": 0.33, "coef": 0.01, "se": 0.01, "z": 0.98,
|
|
296
|
+
},
|
|
297
|
+
]
|
|
298
|
+
_make_study(self.sid, covariates, n=500, epv_text="EPV=50.0")
|
|
299
|
+
self.data = load_forest_data(self.sid)
|
|
300
|
+
|
|
301
|
+
def teardown_method(self):
|
|
302
|
+
shutil.rmtree(DATA_ROOT / self.sid, ignore_errors=True)
|
|
303
|
+
|
|
304
|
+
def test_columns_aligned(self):
|
|
305
|
+
"""CI brackets must start at the same character position on every row."""
|
|
306
|
+
text = render_ascii(self.data)
|
|
307
|
+
lines = text.splitlines()
|
|
308
|
+
# Skip header row which contains "aHR [95% CI]"
|
|
309
|
+
data_lines = [l for l in lines if "[" in l and "Covariate" not in l]
|
|
310
|
+
assert len(data_lines) == 2
|
|
311
|
+
ci_starts = [l.index("[") for l in data_lines]
|
|
312
|
+
assert ci_starts[0] == ci_starts[1], \
|
|
313
|
+
f"CI brackets misaligned: {ci_starts}"
|
|
314
|
+
|
|
315
|
+
def test_header_data_column_alignment(self):
|
|
316
|
+
"""Header columns (aHR, p-value) must align exactly with data row columns."""
|
|
317
|
+
text = render_ascii(self.data)
|
|
318
|
+
lines = text.splitlines()
|
|
319
|
+
# Find header row (has "aHR" and "p-value")
|
|
320
|
+
header = next(l for l in lines if "aHR" in l and "p-value" in l)
|
|
321
|
+
# Find first data row (has "p=" and CI bracket)
|
|
322
|
+
data_row = next(l for l in lines if "p=" in l and "[" in l)
|
|
323
|
+
# Check alignment of key column positions
|
|
324
|
+
# HR column: header "aHR" right-edge aligns with data HR numeric right-edge (both in 7-char field)
|
|
325
|
+
# CI bracket: header "[95% CI]" left-edge aligns with data "[" left-edge
|
|
326
|
+
assert header.index("[95% CI]") == data_row.index("[")
|
|
327
|
+
# p-value header aligns with p= field
|
|
328
|
+
assert header.index("p-value") == data_row.index("p=")
|
|
329
|
+
|
|
330
|
+
def test_ref_line_at_computed_position(self):
|
|
331
|
+
"""The │ reference line must sit at exactly the character index _pos(1.0) computes."""
|
|
332
|
+
text = render_ascii(self.data)
|
|
333
|
+
lines = text.splitlines()
|
|
334
|
+
# Find the axis tick line (has │ at tick positions including value 1.0).
|
|
335
|
+
# Locate the │ that sits at the 1.0 tick position by searching the scale header structure.
|
|
336
|
+
# The second │ after name_w chars marks the ref position.
|
|
337
|
+
tick_line = [l for l in lines if "│" in l and "─" in l and "Forest" not in l
|
|
338
|
+
and "C-index" not in l and "EPV" not in l and "⚠" not in l
|
|
339
|
+
and "**" not in l and not l.startswith("─")][0]
|
|
340
|
+
# The plot area starts after label column + separator │, at index name_w + 1.
|
|
341
|
+
# The ref │ is somewhere in the tick line — find it as the ─┬ at the tick for 1.0,
|
|
342
|
+
# then locate the │ immediately after it.
|
|
343
|
+
# Tick marker for 1.0: the tick text "1" follows a "┬" character placed at the ref x.
|
|
344
|
+
# In the scale header: "...┬" then below "...│" — the │ under each ┬ is at the same index.
|
|
345
|
+
# So find the │ that's below "┬" in the scale header — that's the ref pos.
|
|
346
|
+
# Easier: find the separator line ──┬── (which has ┬ at each tick position).
|
|
347
|
+
# Actually, the simplest: compute lo/hi like render_ascii does, derive ref,
|
|
348
|
+
# then assert │ in a data row is at name_w + 1 + ref.
|
|
349
|
+
all_vals = [1.0]
|
|
350
|
+
for c in self.data.covariates:
|
|
351
|
+
if not c.unstable and c.hr > 0 and c.ci_lower > 0 and c.ci_upper > 0:
|
|
352
|
+
all_vals.extend([c.hr, c.ci_lower, c.ci_upper])
|
|
353
|
+
lo = min(all_vals) / 1.3
|
|
354
|
+
hi = max(all_vals) * 1.3
|
|
355
|
+
if lo <= 0:
|
|
356
|
+
lo = 0.1
|
|
357
|
+
plot_w = 30
|
|
358
|
+
ref = int((math.log(1.0) - math.log(lo)) / (math.log(hi) - math.log(lo)) * plot_w)
|
|
359
|
+
for l in lines:
|
|
360
|
+
if "[" in l and "p=" in l: # data row
|
|
361
|
+
label_part = l.split("│")[0]
|
|
362
|
+
# │ in data row is at index len(label_part)
|
|
363
|
+
assert l.index("│", len(label_part)) == len(label_part)
|
|
364
|
+
# Within the plot, ref is at index ref (0-based in 30-char buf).
|
|
365
|
+
# So │ appears at len(label_part) + 1 (│ sep) + ref within plot... no.
|
|
366
|
+
# Actually │ IS at len(label_part) — it's the label separator at position name_w+0.
|
|
367
|
+
# The ref line │ inside the plot area is at a later index.
|
|
368
|
+
# data_line format: {label}│{plot_buf} {hr} {ci} {p}
|
|
369
|
+
# The second │ in the line (reference line inside plot) is what we want.
|
|
370
|
+
rest = l[len(label_part) + 1:] # after first │
|
|
371
|
+
ref_char_idx = rest.index("│")
|
|
372
|
+
# ref_char_idx in rest should equal ref (plot_buf index)
|
|
373
|
+
assert ref_char_idx == ref, \
|
|
374
|
+
f"ref │ at plot index {ref_char_idx}, expected {ref} (lo={lo:.2f}, hi={hi:.2f})"
|