research-tool-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/backend/_bootstrap.py +15 -0
- app/backend/exports/verification_script.py +19 -0
- app/backend/main.py +59 -0
- app/backend/routers/__init__.py +1 -0
- app/backend/routers/execution.py +478 -0
- app/backend/routers/ingestion.py +233 -0
- app/backend/routers/planning.py +265 -0
- app/backend/routers/reporting.py +531 -0
- app/backend/state.py +44 -0
- core/__init__.py +0 -0
- core/cli/__init__.py +0 -0
- core/cli/main.py +1705 -0
- core/database.py +62 -0
- core/ingestion/__init__.py +0 -0
- core/ingestion/csv_loader.py +191 -0
- core/ingestion/variable_classifier.py +171 -0
- core/masking/__init__.py +0 -0
- core/masking/gate.py +128 -0
- core/models.py +138 -0
- core/planning/__init__.py +0 -0
- core/planning/diagnostics.py +89 -0
- core/planning/lock.py +232 -0
- core/planning/study_plan.py +73 -0
- core/planning/test_selector.py +518 -0
- core/provenance/__init__.py +0 -0
- core/provenance/hashing.py +38 -0
- core/provenance/tracker.py +105 -0
- core/reporting/__init__.py +62 -0
- core/reporting/appendix.py +58 -0
- core/reporting/bundle.py +378 -0
- core/reporting/excel_export.py +683 -0
- core/reporting/flowchart/__init__.py +20 -0
- core/reporting/flowchart/flowchart.py +511 -0
- core/reporting/forensics.py +592 -0
- core/reporting/forest_plot.py +614 -0
- core/reporting/lineage.py +562 -0
- core/reporting/manuscript_draft.py +726 -0
- core/reporting/plots.py +568 -0
- core/reporting/strobe_checklist.py +460 -0
- core/stats/__init__.py +0 -0
- core/stats/descriptive.py +104 -0
- core/stats/inferential.py +540 -0
- core/stats/multiple_comparisons.py +62 -0
- core/stats/post_hoc.py +62 -0
- exports/verification_script.py +19 -0
- research_tool_cli-0.1.0.dist-info/METADATA +16 -0
- research_tool_cli-0.1.0.dist-info/RECORD +93 -0
- research_tool_cli-0.1.0.dist-info/WHEEL +5 -0
- research_tool_cli-0.1.0.dist-info/entry_points.txt +2 -0
- research_tool_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- research_tool_cli-0.1.0.dist-info/top_level.txt +4 -0
- tests/__init__.py +0 -0
- tests/conftest.py +45 -0
- tests/test_amendments.py +296 -0
- tests/test_analyze_batch_robustness.py +162 -0
- tests/test_app_statistical_reporting.py +383 -0
- tests/test_benchmark_21.py +556 -0
- tests/test_bundle.py +277 -0
- tests/test_cox_ph_plan.py +498 -0
- tests/test_csv_loader.py +368 -0
- tests/test_end_to_end.py +302 -0
- tests/test_excel_export.py +164 -0
- tests/test_flowchart.py +244 -0
- tests/test_forensics.py +305 -0
- tests/test_forest_plot.py +374 -0
- tests/test_from_json.py +176 -0
- tests/test_latest_plan_version.py +164 -0
- tests/test_lineage.py +329 -0
- tests/test_lock_immutability.py +133 -0
- tests/test_m1_fk_violation.py +85 -0
- tests/test_m2_data_hash_consistency.py +40 -0
- tests/test_m3_correction_timing.py +59 -0
- tests/test_m4_dedup_field.py +59 -0
- tests/test_m5_excel_hash_scope.py +45 -0
- tests/test_m6_fisher_exact_naming.py +44 -0
- tests/test_m7_duplicate_study_plan.py +55 -0
- tests/test_m8_filter_superseded.py +58 -0
- tests/test_m9_table1_groupby.py +56 -0
- tests/test_manuscript_draft.py +289 -0
- tests/test_masking_gate.py +196 -0
- tests/test_masking_migration.py +111 -0
- tests/test_multiple_comparisons.py +56 -0
- tests/test_plan_validation.py +289 -0
- tests/test_plots.py +394 -0
- tests/test_post_hoc_tagging.py +49 -0
- tests/test_posthoc_analyze.py +203 -0
- tests/test_provenance_tracker.py +110 -0
- tests/test_roadmap_features.py +148 -0
- tests/test_stats_descriptive.py +57 -0
- tests/test_stats_inferential.py +380 -0
- tests/test_strobe_checklist.py +172 -0
- tests/test_test_selector.py +350 -0
- tests/test_variable_classifier.py +124 -0
core/cli/main.py
ADDED
|
@@ -0,0 +1,1705 @@
|
|
|
1
|
+
"""CLI entrypoint — argparse dispatch.
|
|
2
|
+
|
|
3
|
+
Commands (Phase 1):
|
|
4
|
+
new-study create a new study
|
|
5
|
+
ingest load CSV/Excel
|
|
6
|
+
classify-variables tag columns with role and data type
|
|
7
|
+
explore-baseline peek at baseline data (outcomes masked)
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
import sys
|
|
15
|
+
import uuid
|
|
16
|
+
from datetime import datetime, timezone
|
|
17
|
+
|
|
18
|
+
from core.database import get_connection, init_db, study_dir, DATA_ROOT
|
|
19
|
+
from core.ingestion.csv_loader import load_file
|
|
20
|
+
from core.ingestion.variable_classifier import classify_variables_interactive, _classify_batch
|
|
21
|
+
from core.masking.gate import seal_outcomes, is_masked
|
|
22
|
+
from core.planning.study_plan import StudyPlan
|
|
23
|
+
from core.planning.lock import lock_plan, lock_amendment, load_plan, unmask_study
|
|
24
|
+
from core.planning.test_selector import check_assumptions
|
|
25
|
+
from core.stats.descriptive import generate_table1
|
|
26
|
+
from core.stats.inferential import run_test
|
|
27
|
+
from core.reporting.strobe_checklist import generate_report
|
|
28
|
+
from core.reporting.manuscript_draft import write_draft
|
|
29
|
+
from core.reporting.flowchart import render_flowchart
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _model_field(model, key: str, default=""):
|
|
33
|
+
"""Read field from dict or dataclass."""
|
|
34
|
+
if isinstance(model, dict):
|
|
35
|
+
return model.get(key, default)
|
|
36
|
+
return getattr(model, key, default)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def cmd_new_study(args: argparse.Namespace) -> None:
|
|
40
|
+
study_id = uuid.uuid4().hex
|
|
41
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
42
|
+
conn = get_connection(study_id)
|
|
43
|
+
init_db(conn)
|
|
44
|
+
conn.execute(
|
|
45
|
+
"INSERT INTO studies (id, name, created_at, data_dir) VALUES (?, ?, ?, ?)",
|
|
46
|
+
(study_id, args.name, now, str(study_dir(study_id))),
|
|
47
|
+
)
|
|
48
|
+
conn.commit()
|
|
49
|
+
conn.close()
|
|
50
|
+
# Create study dir
|
|
51
|
+
study_dir(study_id).mkdir(parents=True, exist_ok=True)
|
|
52
|
+
print(study_id)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def cmd_ingest(args: argparse.Namespace) -> None:
|
|
56
|
+
na_vals = None
|
|
57
|
+
if getattr(args, "na_values", None):
|
|
58
|
+
na_vals = [v.strip() for v in args.na_values.split(",")]
|
|
59
|
+
print(f"Treating values as missing: {na_vals}")
|
|
60
|
+
force = getattr(args, "force_reingest", False)
|
|
61
|
+
columns = load_file(args.study_id, args.file, na_values=na_vals, force=force)
|
|
62
|
+
print(f"Ingested {len(columns)} columns: {', '.join(columns)}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def cmd_classify_variables(args: argparse.Namespace) -> None:
|
|
66
|
+
"""Auto-classify and write to DB. In v1 this uses heuristics."""
|
|
67
|
+
conn = get_connection(args.study_id)
|
|
68
|
+
init_db(conn)
|
|
69
|
+
raw_table = f"raw_{args.study_id}"
|
|
70
|
+
cur = conn.execute(f"PRAGMA table_info({raw_table})")
|
|
71
|
+
cols_info = cur.fetchall()
|
|
72
|
+
columns = [r["name"] for r in cols_info if r["name"] != "row_id"]
|
|
73
|
+
conn.close()
|
|
74
|
+
|
|
75
|
+
suggestions = classify_variables_interactive(args.study_id, columns)
|
|
76
|
+
_classify_batch(args.study_id, suggestions)
|
|
77
|
+
|
|
78
|
+
# After classification, seal outcome values into shadow table
|
|
79
|
+
seal_outcomes(args.study_id)
|
|
80
|
+
|
|
81
|
+
# Query back to get DB-assigned IDs
|
|
82
|
+
conn = get_connection(args.study_id)
|
|
83
|
+
rows = conn.execute(
|
|
84
|
+
"SELECT id, column_name, role, data_type FROM variables WHERE study_id=? ORDER BY id",
|
|
85
|
+
(args.study_id,),
|
|
86
|
+
).fetchall()
|
|
87
|
+
conn.close()
|
|
88
|
+
for r in rows:
|
|
89
|
+
print(f" {r['id']:<4} {r['column_name']:<30} → {r['role']:<10} {r['data_type']}")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def cmd_explore_baseline(args: argparse.Namespace) -> None:
|
|
93
|
+
"""Show summary of baseline variables only (outcomes masked — physically NULL)."""
|
|
94
|
+
conn = get_connection(args.study_id)
|
|
95
|
+
cur = conn.execute("SELECT column_name, role, data_type FROM variables WHERE study_id=?", (args.study_id,))
|
|
96
|
+
vars_info = cur.fetchall()
|
|
97
|
+
|
|
98
|
+
baseline_cols = [r["column_name"] for r in vars_info if r["role"] == "baseline"]
|
|
99
|
+
if not baseline_cols:
|
|
100
|
+
print("No baseline variables classified yet.")
|
|
101
|
+
conn.close()
|
|
102
|
+
return
|
|
103
|
+
|
|
104
|
+
col_list = ", ".join(baseline_cols)
|
|
105
|
+
raw_table = f"raw_{args.study_id}"
|
|
106
|
+
cur2 = conn.execute(f"SELECT {col_list} FROM {raw_table}")
|
|
107
|
+
rows = cur2.fetchall()
|
|
108
|
+
conn.close()
|
|
109
|
+
print(f"Baseline variables ({len(baseline_cols)}): {', '.join(baseline_cols)}")
|
|
110
|
+
print(f" {len(rows)} rows available")
|
|
111
|
+
if args.head:
|
|
112
|
+
for r in rows[:args.head]:
|
|
113
|
+
r = {k: v for k, v in dict(r).items() if k in baseline_cols}
|
|
114
|
+
print(r)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def cmd_list_variables(args: argparse.Namespace) -> None:
|
|
118
|
+
"""List all classified variables with their DB IDs."""
|
|
119
|
+
conn = get_connection(args.study_id)
|
|
120
|
+
rows = conn.execute(
|
|
121
|
+
"SELECT id, column_name, role, data_type FROM variables WHERE study_id=? ORDER BY id",
|
|
122
|
+
(args.study_id,),
|
|
123
|
+
).fetchall()
|
|
124
|
+
conn.close()
|
|
125
|
+
if not rows:
|
|
126
|
+
print(f"No variables classified for study '{args.study_id}'. Run 'classify-variables' first.")
|
|
127
|
+
return
|
|
128
|
+
print(f" {'ID':<4} {'Column':<30} {'Role':<12} {'Type'}")
|
|
129
|
+
print(f" {'──':<4} {'──────':<30} {'────':<12} ────")
|
|
130
|
+
for r in rows:
|
|
131
|
+
print(f" {r['id']:<4} {r['column_name']:<30} {r['role']:<12} {r['data_type']}")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def cmd_plan(args: argparse.Namespace) -> None:
|
|
135
|
+
"""Declare a study plan."""
|
|
136
|
+
|
|
137
|
+
# Load from JSON if --from-json is provided
|
|
138
|
+
json_path = getattr(args, "from_json", None)
|
|
139
|
+
if json_path:
|
|
140
|
+
from pathlib import Path
|
|
141
|
+
import json as _json
|
|
142
|
+
data = _json.loads(Path(json_path).read_text())
|
|
143
|
+
# Override args with JSON values (CLI flags still win if provided)
|
|
144
|
+
if "comparison" in data:
|
|
145
|
+
args.comparison = data["comparison"]
|
|
146
|
+
if "outcome_var_ids" in data:
|
|
147
|
+
v = data["outcome_var_ids"]
|
|
148
|
+
args.outcome_var_ids = ",".join(str(x) for x in v) if isinstance(v, list) else v
|
|
149
|
+
if "study_type" in data:
|
|
150
|
+
args.study_type = data["study_type"]
|
|
151
|
+
if "tests" in data:
|
|
152
|
+
args.tests = data["tests"]
|
|
153
|
+
if "covariates" in data:
|
|
154
|
+
v = data["covariates"]
|
|
155
|
+
args.covariates = ",".join(str(x) for x in v) if isinstance(v, list) else v
|
|
156
|
+
if "cox_ph_models" in data:
|
|
157
|
+
args.cox_ph_models = data["cox_ph_models"]
|
|
158
|
+
if "interaction_terms" in data:
|
|
159
|
+
args.interaction_terms = data["interaction_terms"]
|
|
160
|
+
if "matching_criteria" in data:
|
|
161
|
+
v = data["matching_criteria"]
|
|
162
|
+
args.matching_criteria = ",".join(str(x) for x in v) if isinstance(v, list) else v
|
|
163
|
+
if "overrides" in data:
|
|
164
|
+
args.overrides = data["overrides"]
|
|
165
|
+
|
|
166
|
+
outcome_ids = [int(x) for x in args.outcome_var_ids.split(",")]
|
|
167
|
+
tests = []
|
|
168
|
+
if args.tests:
|
|
169
|
+
for t in args.tests:
|
|
170
|
+
if isinstance(t, dict):
|
|
171
|
+
# Native JSON dict (from --from-json)
|
|
172
|
+
tests.append({
|
|
173
|
+
"variable_name": t.get("variable_name", ""),
|
|
174
|
+
"test_name": t.get("test_name", ""),
|
|
175
|
+
"rationale": t.get("rationale", ""),
|
|
176
|
+
})
|
|
177
|
+
else:
|
|
178
|
+
parts = t.split(":", 2)
|
|
179
|
+
# Format: variable_name:test_name:[rationale]
|
|
180
|
+
var_name = parts[0] if len(parts) > 0 else ""
|
|
181
|
+
test_name = parts[1] if len(parts) > 1 else ""
|
|
182
|
+
rationale = parts[2] if len(parts) > 2 else ""
|
|
183
|
+
tests.append({
|
|
184
|
+
"variable_name": var_name,
|
|
185
|
+
"test_name": test_name,
|
|
186
|
+
"rationale": rationale,
|
|
187
|
+
})
|
|
188
|
+
covariates = [int(x) for x in args.covariates.split(",")] if args.covariates else []
|
|
189
|
+
matching_criteria = [int(x) for x in args.matching_criteria.split(",")] if getattr(args, "matching_criteria", None) else []
|
|
190
|
+
|
|
191
|
+
# Parse Cox PH models
|
|
192
|
+
cox_ph_models = []
|
|
193
|
+
if getattr(args, "cox_ph_models", None):
|
|
194
|
+
for m in args.cox_ph_models:
|
|
195
|
+
if isinstance(m, dict):
|
|
196
|
+
# Native JSON dict (from --from-json)
|
|
197
|
+
cox_ph_models.append({
|
|
198
|
+
"model_name": m.get("model_name", ""),
|
|
199
|
+
"survival_time_col": m.get("survival_time_col", ""),
|
|
200
|
+
"event_col": m.get("event_col", ""),
|
|
201
|
+
"primary_treatment_col": m.get("primary_treatment_col", ""),
|
|
202
|
+
"covariate_cols": m.get("covariate_cols", []),
|
|
203
|
+
"rationale": m.get("rationale", ""),
|
|
204
|
+
"interaction_terms": m.get("interaction_terms", []),
|
|
205
|
+
})
|
|
206
|
+
else:
|
|
207
|
+
# Format: model_name:survival_time_col:event_col:primary_treatment_col[:covariate_cols[:rationale]]
|
|
208
|
+
parts = m.split(":")
|
|
209
|
+
if len(parts) < 4:
|
|
210
|
+
print(f"Error: invalid Cox PH model format '{m}'. Use model_name:survival_time_col:event_col:primary_treatment_col[:covariate_cols[:rationale]]", file=sys.stderr)
|
|
211
|
+
sys.exit(1)
|
|
212
|
+
model_name = parts[0]
|
|
213
|
+
survival_time_col = parts[1]
|
|
214
|
+
event_col = parts[2]
|
|
215
|
+
primary_treatment_col = parts[3]
|
|
216
|
+
if len(parts) >= 6:
|
|
217
|
+
covariate_cols = [c.strip() for c in parts[4].split(",")] if parts[4] else []
|
|
218
|
+
rationale = parts[5]
|
|
219
|
+
elif len(parts) == 5:
|
|
220
|
+
covariate_cols = [c.strip() for c in parts[4].split(",")] if parts[4] else []
|
|
221
|
+
rationale = ""
|
|
222
|
+
else:
|
|
223
|
+
covariate_cols = []
|
|
224
|
+
rationale = ""
|
|
225
|
+
cox_ph_models.append({
|
|
226
|
+
"model_name": model_name,
|
|
227
|
+
"survival_time_col": survival_time_col,
|
|
228
|
+
"event_col": event_col,
|
|
229
|
+
"primary_treatment_col": primary_treatment_col,
|
|
230
|
+
"covariate_cols": covariate_cols,
|
|
231
|
+
"rationale": rationale,
|
|
232
|
+
"interaction_terms": [],
|
|
233
|
+
})
|
|
234
|
+
|
|
235
|
+
# Parse interaction terms and attach to the named model
|
|
236
|
+
if getattr(args, "interaction_terms", None):
|
|
237
|
+
for spec in args.interaction_terms:
|
|
238
|
+
if isinstance(spec, dict):
|
|
239
|
+
model_name = spec.get("model_name", "")
|
|
240
|
+
var_a = spec.get("var_a", "")
|
|
241
|
+
var_b = spec.get("var_b", "")
|
|
242
|
+
else:
|
|
243
|
+
parts = spec.split(":")
|
|
244
|
+
if len(parts) != 3:
|
|
245
|
+
print(f"Error: invalid interaction term format '{spec}'. Use model_name:var_a:var_b", file=sys.stderr)
|
|
246
|
+
sys.exit(1)
|
|
247
|
+
model_name, var_a, var_b = parts
|
|
248
|
+
found = False
|
|
249
|
+
for m in cox_ph_models:
|
|
250
|
+
if m["model_name"] == model_name:
|
|
251
|
+
m["interaction_terms"].append([var_a, var_b])
|
|
252
|
+
found = True
|
|
253
|
+
break
|
|
254
|
+
if not found:
|
|
255
|
+
print(f"Error: interaction term '{spec}' references model '{model_name}' which was not declared with --cox-ph-models", file=sys.stderr)
|
|
256
|
+
sys.exit(1)
|
|
257
|
+
|
|
258
|
+
# Role overrides are plan metadata. They do not modify the ingested rows
|
|
259
|
+
# or erase the classifier's original suggestion.
|
|
260
|
+
overrides: dict[int, str] = {}
|
|
261
|
+
override_audit: list[dict] = []
|
|
262
|
+
for raw_override in getattr(args, "overrides", []) or []:
|
|
263
|
+
parts = raw_override.split(":")
|
|
264
|
+
if len(parts) != 2 or not parts[0].startswith("id=") or not parts[1].startswith("role="):
|
|
265
|
+
print(f"Error: invalid override '{raw_override}'. Use id=<variable_id>:role=<role>.", file=sys.stderr)
|
|
266
|
+
sys.exit(1)
|
|
267
|
+
try:
|
|
268
|
+
variable_id = int(parts[0][len("id="):])
|
|
269
|
+
except ValueError:
|
|
270
|
+
print(f"Error: invalid override variable ID in '{raw_override}'.", file=sys.stderr)
|
|
271
|
+
sys.exit(1)
|
|
272
|
+
role = parts[1][len("role="):]
|
|
273
|
+
if role not in {"baseline", "outcome", "covariate"}:
|
|
274
|
+
print(f"Error: invalid override role '{role}'. Choose baseline, outcome, or covariate.", file=sys.stderr)
|
|
275
|
+
sys.exit(1)
|
|
276
|
+
if variable_id in overrides:
|
|
277
|
+
print(f"Error: duplicate override for variable ID {variable_id}.", file=sys.stderr)
|
|
278
|
+
sys.exit(1)
|
|
279
|
+
overrides[variable_id] = role
|
|
280
|
+
override_audit.append({"variable_id": variable_id, "role": role})
|
|
281
|
+
|
|
282
|
+
if overrides and list(DATA_ROOT.glob(f"{args.study_id}/study_plan.v*.locked.json")):
|
|
283
|
+
print("Error: cannot apply role overrides after the study plan is locked. Create a new plan version before locking.", file=sys.stderr)
|
|
284
|
+
sys.exit(1)
|
|
285
|
+
|
|
286
|
+
# ── Validate classified variables ─────────────────────────────────────
|
|
287
|
+
conn = get_connection(args.study_id)
|
|
288
|
+
cur = conn.execute(
|
|
289
|
+
"SELECT id, column_name, role FROM variables WHERE study_id=?",
|
|
290
|
+
(args.study_id,),
|
|
291
|
+
)
|
|
292
|
+
classified = {r["id"]: {"name": r["column_name"], "role": r["role"]} for r in cur.fetchall()}
|
|
293
|
+
conn.close()
|
|
294
|
+
|
|
295
|
+
if not classified:
|
|
296
|
+
print(
|
|
297
|
+
f"Error: no variables classified for study '{args.study_id}'. "
|
|
298
|
+
f"Run 'research-tool classify-variables {args.study_id}' after ingesting data first.",
|
|
299
|
+
file=sys.stderr,
|
|
300
|
+
)
|
|
301
|
+
sys.exit(1)
|
|
302
|
+
|
|
303
|
+
unknown_overrides = [str(i) for i in overrides if i not in classified]
|
|
304
|
+
if unknown_overrides:
|
|
305
|
+
print(f"Error: override variable ID(s) {', '.join(unknown_overrides)} not found among classified variables.", file=sys.stderr)
|
|
306
|
+
sys.exit(1)
|
|
307
|
+
for variable_id, role in overrides.items():
|
|
308
|
+
classified[variable_id]["role"] = role
|
|
309
|
+
|
|
310
|
+
# outcome-var-ids
|
|
311
|
+
missing_outcome = [str(i) for i in outcome_ids if i not in classified]
|
|
312
|
+
if missing_outcome:
|
|
313
|
+
print(
|
|
314
|
+
f"Error: outcome variable ID(s) {', '.join(missing_outcome)} not found "
|
|
315
|
+
f"among classified variables. Run 'research-tool classify-variables' first.",
|
|
316
|
+
file=sys.stderr,
|
|
317
|
+
)
|
|
318
|
+
sys.exit(1)
|
|
319
|
+
|
|
320
|
+
wrong_role = [str(i) for i in outcome_ids if classified[i]["role"] != "outcome"]
|
|
321
|
+
if wrong_role:
|
|
322
|
+
print(
|
|
323
|
+
f"Error: variable ID(s) {', '.join(wrong_role)} "
|
|
324
|
+
f"({'/'.join(classified[int(i)]['name'] for i in wrong_role)}) "
|
|
325
|
+
f"is classified as '{classified[int(wrong_role[0])]['role']}', not 'outcome'. "
|
|
326
|
+
f"Use a variable classified as outcome, or reclassify with "
|
|
327
|
+
f"'research-tool classify-variables {args.study_id}'.",
|
|
328
|
+
file=sys.stderr,
|
|
329
|
+
)
|
|
330
|
+
sys.exit(1)
|
|
331
|
+
|
|
332
|
+
# covariates
|
|
333
|
+
missing_cov = [str(i) for i in covariates if i not in classified]
|
|
334
|
+
if missing_cov:
|
|
335
|
+
print(
|
|
336
|
+
f"Error: covariate variable ID(s) {', '.join(missing_cov)} not found "
|
|
337
|
+
f"among classified variables.",
|
|
338
|
+
file=sys.stderr,
|
|
339
|
+
)
|
|
340
|
+
sys.exit(1)
|
|
341
|
+
|
|
342
|
+
# Validate Cox PH model columns exist and are classified correctly
|
|
343
|
+
if cox_ph_models:
|
|
344
|
+
conn = get_connection(args.study_id)
|
|
345
|
+
cur = conn.execute(
|
|
346
|
+
"SELECT column_name, role, data_type FROM variables WHERE study_id=?",
|
|
347
|
+
(args.study_id,),
|
|
348
|
+
)
|
|
349
|
+
var_catalog = {r["column_name"]: {"role": r["role"], "data_type": r["data_type"]} for r in cur.fetchall()}
|
|
350
|
+
conn.close()
|
|
351
|
+
|
|
352
|
+
for model in cox_ph_models:
|
|
353
|
+
model_name = model["model_name"]
|
|
354
|
+
survival_time_col = model["survival_time_col"]
|
|
355
|
+
event_col = model["event_col"]
|
|
356
|
+
primary_treatment_col = model["primary_treatment_col"]
|
|
357
|
+
covariate_cols = model["covariate_cols"]
|
|
358
|
+
|
|
359
|
+
# Check survival_time_col
|
|
360
|
+
if survival_time_col not in var_catalog:
|
|
361
|
+
print(f"Error: Cox PH model '{model_name}': survival_time_col '{survival_time_col}' not found among classified variables.", file=sys.stderr)
|
|
362
|
+
sys.exit(1)
|
|
363
|
+
if var_catalog[survival_time_col]["data_type"] != "time_to_event":
|
|
364
|
+
print(f"Error: Cox PH model '{model_name}': survival_time_col '{survival_time_col}' is not classified as time_to_event (got {var_catalog[survival_time_col]['data_type']}).", file=sys.stderr)
|
|
365
|
+
sys.exit(1)
|
|
366
|
+
|
|
367
|
+
# Check event_col
|
|
368
|
+
if event_col not in var_catalog:
|
|
369
|
+
print(f"Error: Cox PH model '{model_name}': event_col '{event_col}' not found among classified variables.", file=sys.stderr)
|
|
370
|
+
sys.exit(1)
|
|
371
|
+
if var_catalog[event_col]["role"] != "outcome":
|
|
372
|
+
print(f"Error: Cox PH model '{model_name}': event_col '{event_col}' must be classified as outcome.", file=sys.stderr)
|
|
373
|
+
sys.exit(1)
|
|
374
|
+
|
|
375
|
+
# Check primary_treatment_col
|
|
376
|
+
if primary_treatment_col not in var_catalog:
|
|
377
|
+
print(f"Error: Cox PH model '{model_name}': primary_treatment_col '{primary_treatment_col}' not found among classified variables.", file=sys.stderr)
|
|
378
|
+
sys.exit(1)
|
|
379
|
+
|
|
380
|
+
# Check covariate_cols
|
|
381
|
+
for cov in covariate_cols:
|
|
382
|
+
if cov not in var_catalog:
|
|
383
|
+
print(f"Error: Cox PH model '{model_name}': covariate '{cov}' not found among classified variables.", file=sys.stderr)
|
|
384
|
+
sys.exit(1)
|
|
385
|
+
|
|
386
|
+
# Check interaction term columns
|
|
387
|
+
for pair in model.get("interaction_terms", []):
|
|
388
|
+
for var in pair:
|
|
389
|
+
if var not in var_catalog:
|
|
390
|
+
print(f"Error: Cox PH model '{model_name}': interaction term variable '{var}' not found among classified variables.", file=sys.stderr)
|
|
391
|
+
sys.exit(1)
|
|
392
|
+
|
|
393
|
+
# Validate event_col is binary (0/1)
|
|
394
|
+
conn2 = get_connection(args.study_id)
|
|
395
|
+
raw_table = f"raw_{args.study_id}"
|
|
396
|
+
cur2 = conn2.execute(f'SELECT DISTINCT "{event_col}" AS val FROM {raw_table} WHERE "{event_col}" IS NOT NULL')
|
|
397
|
+
event_vals = [str(r["val"]) for r in cur2.fetchall() if r["val"] is not None and str(r["val"]).strip() != ""]
|
|
398
|
+
non_binary = [v for v in event_vals if v not in ("0", "1")]
|
|
399
|
+
if non_binary:
|
|
400
|
+
print(f"Error: Cox PH model '{model_name}': event_col '{event_col}' must be binary (0/1), found values: {non_binary}", file=sys.stderr)
|
|
401
|
+
conn2.close()
|
|
402
|
+
sys.exit(1)
|
|
403
|
+
# Validate survival_time_col is numeric and non-negative
|
|
404
|
+
cur2 = conn2.execute(f'SELECT "{survival_time_col}" AS val FROM {raw_table} WHERE "{survival_time_col}" IS NOT NULL')
|
|
405
|
+
for r in cur2.fetchall():
|
|
406
|
+
v = r["val"]
|
|
407
|
+
if v is None or str(v).strip() == "":
|
|
408
|
+
continue
|
|
409
|
+
try:
|
|
410
|
+
fv = float(v)
|
|
411
|
+
if not math.isfinite(fv):
|
|
412
|
+
print(f"Error: Cox PH model '{model_name}': survival_time_col '{survival_time_col}' has non-finite value '{v}'", file=sys.stderr)
|
|
413
|
+
conn2.close()
|
|
414
|
+
sys.exit(1)
|
|
415
|
+
if fv < 0:
|
|
416
|
+
print(f"Error: Cox PH model '{model_name}': survival_time_col '{survival_time_col}' has negative value ({v})", file=sys.stderr)
|
|
417
|
+
conn2.close()
|
|
418
|
+
sys.exit(1)
|
|
419
|
+
except (ValueError, TypeError):
|
|
420
|
+
print(f"Error: Cox PH model '{model_name}': survival_time_col '{survival_time_col}' is not numeric (found '{v}')", file=sys.stderr)
|
|
421
|
+
conn2.close()
|
|
422
|
+
sys.exit(1)
|
|
423
|
+
conn2.close()
|
|
424
|
+
|
|
425
|
+
# Check assumptions before building plan
|
|
426
|
+
# Enrich tests with covariate count for Cox EPV check
|
|
427
|
+
n_covariates = len(covariates)
|
|
428
|
+
for t in tests:
|
|
429
|
+
t["n_covariates"] = n_covariates
|
|
430
|
+
|
|
431
|
+
# Collect all warnings including Cox PH model warnings
|
|
432
|
+
all_warnings = check_assumptions(args.study_id, tests)
|
|
433
|
+
|
|
434
|
+
# Add Cox PH model warnings
|
|
435
|
+
from core.planning.test_selector import check_cox_ph_model_assumptions
|
|
436
|
+
cox_model_warnings = check_cox_ph_model_assumptions(args.study_id, cox_ph_models)
|
|
437
|
+
all_warnings.extend(cox_model_warnings)
|
|
438
|
+
|
|
439
|
+
for w in all_warnings:
|
|
440
|
+
print(w, file=sys.stderr)
|
|
441
|
+
|
|
442
|
+
# Warn if a matching criterion overlaps the comparison variable
|
|
443
|
+
if matching_criteria:
|
|
444
|
+
comp_lower = args.comparison.lower().replace("_", " ")
|
|
445
|
+
for vid, info in classified.items():
|
|
446
|
+
col_name = info["name"]
|
|
447
|
+
col_normalized = col_name.lower().replace("_", " ")
|
|
448
|
+
if col_normalized in comp_lower and vid in matching_criteria:
|
|
449
|
+
print(
|
|
450
|
+
f"Warning: variable '{col_name}' is declared as both a "
|
|
451
|
+
f"matching criterion and part of the primary comparison. "
|
|
452
|
+
f"Matching on the comparison variable can obscure the effect "
|
|
453
|
+
f"being studied — confirm this is intentional.",
|
|
454
|
+
file=sys.stderr,
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
# Map warnings to test names for enforcement at analyze time
|
|
458
|
+
warning_map: dict[str, str] = {}
|
|
459
|
+
for test in tests:
|
|
460
|
+
tn = test.get("test_name", "")
|
|
461
|
+
var = test.get("variable_name", "")
|
|
462
|
+
for w in all_warnings:
|
|
463
|
+
if f"{tn} on '{var}'" in w:
|
|
464
|
+
warning_map[var] = w
|
|
465
|
+
|
|
466
|
+
# Also map Cox PH model warnings
|
|
467
|
+
for model in cox_ph_models:
|
|
468
|
+
model_name = model["model_name"]
|
|
469
|
+
for w in cox_model_warnings:
|
|
470
|
+
if model_name in w:
|
|
471
|
+
warning_map[f"cox_ph_model:{model_name}"] = w
|
|
472
|
+
|
|
473
|
+
# Extract primary treatment column from Cox PH models if available
|
|
474
|
+
primary_treatment_col = ""
|
|
475
|
+
if cox_ph_models:
|
|
476
|
+
first_model = cox_ph_models[0]
|
|
477
|
+
primary_treatment_col = first_model.get("primary_treatment_col", "")
|
|
478
|
+
|
|
479
|
+
plan = StudyPlan(
|
|
480
|
+
study_id=args.study_id,
|
|
481
|
+
study_type=args.study_type,
|
|
482
|
+
primary_comparison=args.comparison,
|
|
483
|
+
primary_treatment_col=primary_treatment_col,
|
|
484
|
+
primary_outcome_variable_ids=outcome_ids,
|
|
485
|
+
planned_tests=tests,
|
|
486
|
+
covariates=covariates,
|
|
487
|
+
matching_criteria=matching_criteria,
|
|
488
|
+
warnings=warning_map,
|
|
489
|
+
role_overrides=overrides,
|
|
490
|
+
audit={"role_overrides": override_audit},
|
|
491
|
+
cox_ph_models=cox_ph_models,
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
# Write provisional plan
|
|
495
|
+
mcd = DATA_ROOT / args.study_id
|
|
496
|
+
mcd.mkdir(parents=True, exist_ok=True)
|
|
497
|
+
prov_path = mcd / "study_plan.provisional.json"
|
|
498
|
+
import json
|
|
499
|
+
prov_path.write_text(json.dumps(plan.to_dict(), indent=2))
|
|
500
|
+
print(f"Provisional plan saved to {prov_path}")
|
|
501
|
+
print(f"Run 'research-tool lock {args.study_id}' to lock it (immutable).")
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def cmd_lock(args: argparse.Namespace) -> None:
|
|
505
|
+
"""Lock the study plan — writes immutable versioned JSON snapshot."""
|
|
506
|
+
prov_path = DATA_ROOT / args.study_id / "study_plan.provisional.json"
|
|
507
|
+
if not prov_path.exists():
|
|
508
|
+
print("No provisional plan found. Run 'research-tool plan' first.",
|
|
509
|
+
file=sys.stderr)
|
|
510
|
+
sys.exit(1)
|
|
511
|
+
|
|
512
|
+
# ── Block locking on duplicate patient IDs unless explicitly allowed ──
|
|
513
|
+
strict = getattr(args, "strict_ids", False)
|
|
514
|
+
allow_dupes = getattr(args, "allow_duplicate_ids", False)
|
|
515
|
+
if strict or not allow_dupes:
|
|
516
|
+
from core.ingestion.csv_loader import find_duplicate_patient_ids
|
|
517
|
+
dupes = find_duplicate_patient_ids(args.study_id)
|
|
518
|
+
if dupes:
|
|
519
|
+
dupe_desc = ", ".join("'" + str(pid) + "' (" + str(n) + "x)" for pid, n in dupes)
|
|
520
|
+
print(
|
|
521
|
+
f"Error: duplicate patient identifiers found — {dupe_desc}. "
|
|
522
|
+
f"The study cannot be locked because duplicate IDs violate the "
|
|
523
|
+
f"independence assumption of all planned tests. "
|
|
524
|
+
f"Either:\n"
|
|
525
|
+
f" (a) remove or deduplicate the offending rows and re-ingest, or\n"
|
|
526
|
+
f" (b) pass --allow-duplicate-ids to lock if this is a "
|
|
527
|
+
f"genuinely longitudinal/repeated-measures design.",
|
|
528
|
+
file=sys.stderr,
|
|
529
|
+
)
|
|
530
|
+
sys.exit(1)
|
|
531
|
+
|
|
532
|
+
import json
|
|
533
|
+
plan_data = json.loads(prov_path.read_text())
|
|
534
|
+
plan = StudyPlan.from_dict(plan_data)
|
|
535
|
+
path = lock_plan(args.study_id, plan)
|
|
536
|
+
print(f"Plan locked: {path}")
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def cmd_amend(args: argparse.Namespace) -> None:
|
|
540
|
+
"""Amend a locked study plan (pre-unmask or post-hoc)."""
|
|
541
|
+
reason = getattr(args, "reason", "")
|
|
542
|
+
if not reason:
|
|
543
|
+
print("Error: --reason is required for any amendment.", file=sys.stderr)
|
|
544
|
+
sys.exit(1)
|
|
545
|
+
|
|
546
|
+
is_post_hoc = getattr(args, "post_hoc", False)
|
|
547
|
+
|
|
548
|
+
# Parse tests from --test flags
|
|
549
|
+
tests = []
|
|
550
|
+
raw_tests = getattr(args, "tests", []) or []
|
|
551
|
+
for t in raw_tests:
|
|
552
|
+
parts = t.split(":", 2)
|
|
553
|
+
var_name = parts[0] if len(parts) > 0 else ""
|
|
554
|
+
test_name = parts[1] if len(parts) > 1 else ""
|
|
555
|
+
rationale = parts[2] if len(parts) > 2 else ""
|
|
556
|
+
tests.append({"variable_name": var_name, "test_name": test_name, "rationale": rationale})
|
|
557
|
+
|
|
558
|
+
try:
|
|
559
|
+
if is_post_hoc:
|
|
560
|
+
path = lock_amendment(
|
|
561
|
+
args.study_id,
|
|
562
|
+
amendment_reason=reason,
|
|
563
|
+
post_hoc_tests=tests,
|
|
564
|
+
)
|
|
565
|
+
print(f"Post-hoc amendment saved as version {path.stem.split('.v')[1]}: {reason}")
|
|
566
|
+
else:
|
|
567
|
+
path = lock_amendment(
|
|
568
|
+
args.study_id,
|
|
569
|
+
amendment_reason=reason,
|
|
570
|
+
planned_tests=tests,
|
|
571
|
+
)
|
|
572
|
+
print(f"Amendment saved as version {path.stem.split('.v')[1]}: {reason}")
|
|
573
|
+
except (ValueError, RuntimeError) as e:
|
|
574
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
575
|
+
sys.exit(1)
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def cmd_unmask(args: argparse.Namespace) -> None:
|
|
579
|
+
"""Unmask outcome data (irreversible). Gates on pre-unmask diagnostics."""
|
|
580
|
+
# Load plan to check diagnostic results
|
|
581
|
+
from core.planning.lock import load_plan
|
|
582
|
+
from core.database import get_connection
|
|
583
|
+
from datetime import datetime, timezone
|
|
584
|
+
import json
|
|
585
|
+
|
|
586
|
+
try:
|
|
587
|
+
plan = load_plan(args.study_id)
|
|
588
|
+
except (FileNotFoundError, ValueError) as e:
|
|
589
|
+
print(f"Error loading plan: {e}", file=sys.stderr)
|
|
590
|
+
print("Unmask requires a locked plan. Run 'research-tool lock' first.",
|
|
591
|
+
file=sys.stderr)
|
|
592
|
+
sys.exit(1)
|
|
593
|
+
|
|
594
|
+
diag_results = getattr(plan, "diagnostic_results", [])
|
|
595
|
+
pre_unmask = [d for d in diag_results if d.get("stage") == "pre_unmask"]
|
|
596
|
+
|
|
597
|
+
forceable_fails = [d for d in pre_unmask if d.get("status") == "fail" and d.get("forceable") == True]
|
|
598
|
+
non_forceable_fails = [d for d in pre_unmask if d.get("status") == "fail" and d.get("forceable") != True]
|
|
599
|
+
|
|
600
|
+
# Tier 1: non-forceable fails → always block, no flag bypasses
|
|
601
|
+
if non_forceable_fails:
|
|
602
|
+
print(
|
|
603
|
+
f"Error: cannot unmask — {len(non_forceable_fails)} non-overridable "
|
|
604
|
+
f"diagnostic check(s) failed:\n",
|
|
605
|
+
file=sys.stderr,
|
|
606
|
+
)
|
|
607
|
+
for d in non_forceable_fails:
|
|
608
|
+
print(f" [{d['check_name']}] {d['message']}", file=sys.stderr)
|
|
609
|
+
print(
|
|
610
|
+
"\nThese checks assess fundamental data integrity for the planned analyses.",
|
|
611
|
+
file=sys.stderr,
|
|
612
|
+
)
|
|
613
|
+
sys.exit(1)
|
|
614
|
+
|
|
615
|
+
# Tier 2: forceable fails → block unless user provides override justification
|
|
616
|
+
force_justification = getattr(args, "force", None)
|
|
617
|
+
if forceable_fails:
|
|
618
|
+
if not force_justification:
|
|
619
|
+
print(
|
|
620
|
+
f"Error: {len(forceable_fails)} diagnostic check(s) require review:\n",
|
|
621
|
+
file=sys.stderr,
|
|
622
|
+
)
|
|
623
|
+
for d in forceable_fails:
|
|
624
|
+
print(f" [{d['check_name']}] {d['message']}", file=sys.stderr)
|
|
625
|
+
print(
|
|
626
|
+
"\nThese do not block unmask unconditionally, but require a documented "
|
|
627
|
+
"justification.\n"
|
|
628
|
+
"Provide one with: research-tool unmask <study_id> --force '<your reason>'\n"
|
|
629
|
+
"The justification is permanently recorded in the study's audit trail.",
|
|
630
|
+
file=sys.stderr,
|
|
631
|
+
)
|
|
632
|
+
sys.exit(1)
|
|
633
|
+
|
|
634
|
+
# Log the override to the DB
|
|
635
|
+
conn = get_connection(args.study_id)
|
|
636
|
+
override_events = []
|
|
637
|
+
for d in forceable_fails:
|
|
638
|
+
override_events.append({
|
|
639
|
+
"check_name": d["check_name"],
|
|
640
|
+
"message": d["message"],
|
|
641
|
+
"justification": force_justification,
|
|
642
|
+
"overridden_at": datetime.now(timezone.utc).isoformat(),
|
|
643
|
+
})
|
|
644
|
+
existing = conn.execute(
|
|
645
|
+
"SELECT unmask_audit_json FROM studies WHERE id=?",
|
|
646
|
+
(args.study_id,),
|
|
647
|
+
).fetchone()
|
|
648
|
+
audit_log = []
|
|
649
|
+
if existing and existing["unmask_audit_json"]:
|
|
650
|
+
audit_log = json.loads(existing["unmask_audit_json"])
|
|
651
|
+
audit_log.extend(override_events)
|
|
652
|
+
conn.execute(
|
|
653
|
+
"UPDATE studies SET unmask_audit_json=? WHERE id=?",
|
|
654
|
+
(json.dumps(audit_log), args.study_id),
|
|
655
|
+
)
|
|
656
|
+
conn.commit()
|
|
657
|
+
conn.close()
|
|
658
|
+
|
|
659
|
+
unmask_study(args.study_id)
|
|
660
|
+
msg = "Study unmasked. Outcome data now visible."
|
|
661
|
+
if forceable_fails:
|
|
662
|
+
msg += (
|
|
663
|
+
f" {len(forceable_fails)} diagnostic override(s) recorded "
|
|
664
|
+
f"in unmask_audit_json."
|
|
665
|
+
)
|
|
666
|
+
print(msg)
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def cmd_table1(args: argparse.Namespace) -> None:
|
|
670
|
+
"""Generate Table 1.
|
|
671
|
+
|
|
672
|
+
When a locked plan exists, defaults to stratified output grouped by the
|
|
673
|
+
plan's comparison variable (treatment_arm). Pass --overall for the
|
|
674
|
+
unstratified single-column view. No baseline p-values (CONSORT).
|
|
675
|
+
"""
|
|
676
|
+
groupby = args.groupby
|
|
677
|
+
if groupby is None and not getattr(args, "overall", False):
|
|
678
|
+
locked_paths = sorted(DATA_ROOT.glob(f"{args.study_id}/study_plan.v*.locked.json"))
|
|
679
|
+
if locked_paths:
|
|
680
|
+
from core.planning.lock import load_plan
|
|
681
|
+
try:
|
|
682
|
+
plan = load_plan(args.study_id)
|
|
683
|
+
groupby = plan.primary_treatment_col or "treatment_arm"
|
|
684
|
+
except (FileNotFoundError, ValueError):
|
|
685
|
+
groupby = "treatment_arm"
|
|
686
|
+
|
|
687
|
+
tbl = generate_table1(args.study_id, groupby=groupby)
|
|
688
|
+
if tbl.empty if hasattr(tbl, 'empty') else len(tbl) == 0:
|
|
689
|
+
print("Table 1 is empty. Classify variables first.")
|
|
690
|
+
return
|
|
691
|
+
print(tbl.to_string())
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def cmd_analyze(args: argparse.Namespace) -> None:
|
|
695
|
+
"""Run pre-registered analyses from locked plan."""
|
|
696
|
+
if is_masked(args.study_id):
|
|
697
|
+
print("Study is still masked. Unmask first with 'research-tool unmask'.",
|
|
698
|
+
file=sys.stderr)
|
|
699
|
+
sys.exit(1)
|
|
700
|
+
try:
|
|
701
|
+
plan = load_plan(args.study_id)
|
|
702
|
+
except FileNotFoundError:
|
|
703
|
+
print("No locked plan found. Lock a plan first.", file=sys.stderr)
|
|
704
|
+
sys.exit(1)
|
|
705
|
+
|
|
706
|
+
conn = get_connection(args.study_id)
|
|
707
|
+
raw_table = f"raw_{args.study_id}"
|
|
708
|
+
import pandas as pd
|
|
709
|
+
df = pd.read_sql_query(f"SELECT * FROM {raw_table}", conn)
|
|
710
|
+
conn.close()
|
|
711
|
+
|
|
712
|
+
from core.stats.multiple_comparisons import correct
|
|
713
|
+
from datetime import datetime, timezone
|
|
714
|
+
import json
|
|
715
|
+
|
|
716
|
+
results = []
|
|
717
|
+
conn = get_connection(args.study_id)
|
|
718
|
+
init_db(conn)
|
|
719
|
+
|
|
720
|
+
# Parse --force, --rerun flags
|
|
721
|
+
force = getattr(args, "force", False)
|
|
722
|
+
rerun = getattr(args, "rerun", False)
|
|
723
|
+
|
|
724
|
+
# Build variable ID → column_name lookup for resolving --test var_id specs
|
|
725
|
+
var_id_to_name = {}
|
|
726
|
+
for row in conn.execute(
|
|
727
|
+
"SELECT id, column_name FROM variables WHERE study_id=?", (args.study_id,)
|
|
728
|
+
).fetchall():
|
|
729
|
+
var_id_to_name[str(row["id"])] = row["column_name"]
|
|
730
|
+
|
|
731
|
+
# Build combined test list: planned tests (pre-registered) + post-hoc tests (exploratory)
|
|
732
|
+
# The --post-hoc flag on analyze is legacy; analyze now runs both lists automatically.
|
|
733
|
+
test_runs: list[tuple[list[dict], int, str]] = [] # (tests, is_pre_registered, label)
|
|
734
|
+
if plan.planned_tests:
|
|
735
|
+
test_runs.append((plan.planned_tests, 1, "pre-registered"))
|
|
736
|
+
if plan.post_hoc_tests:
|
|
737
|
+
test_runs.append((plan.post_hoc_tests, 0, "post-hoc"))
|
|
738
|
+
|
|
739
|
+
# Enforce assumption warnings from plan time
|
|
740
|
+
plan_warnings = getattr(plan, "warnings", {})
|
|
741
|
+
|
|
742
|
+
# Run all test lists
|
|
743
|
+
for test_list, is_pre_registered, _label in test_runs:
|
|
744
|
+
for t in test_list:
|
|
745
|
+
var_name = t.get("variable_name", "")
|
|
746
|
+
test_name = t.get("test_name", "")
|
|
747
|
+
test_rationale = t.get("rationale", "")
|
|
748
|
+
if not test_name or not var_name:
|
|
749
|
+
continue
|
|
750
|
+
|
|
751
|
+
# Resolve variable ID to column name if needed
|
|
752
|
+
# (e.g., --test "8:kaplan_meier_logrank:..." stores "8", not "pfs_days")
|
|
753
|
+
if var_name in var_id_to_name:
|
|
754
|
+
var_name = var_id_to_name[var_name]
|
|
755
|
+
|
|
756
|
+
# Dedup: skip if completed result already exists for this exact test
|
|
757
|
+
if not rerun:
|
|
758
|
+
existing = conn.execute(
|
|
759
|
+
"""SELECT id, computed_at FROM analysis_results
|
|
760
|
+
WHERE study_id=? AND test_name=? AND variable_ids_used=? AND
|
|
761
|
+
study_plan_version=? AND is_pre_registered=?
|
|
762
|
+
AND json_extract(status_json, '$.status') = 'completed'
|
|
763
|
+
AND superseded_previous_result_id IS NULL
|
|
764
|
+
ORDER BY id DESC LIMIT 1""",
|
|
765
|
+
(args.study_id, test_name, json.dumps([var_name]),
|
|
766
|
+
plan.version, is_pre_registered),
|
|
767
|
+
).fetchone()
|
|
768
|
+
if existing:
|
|
769
|
+
print(
|
|
770
|
+
f"Test '{test_name}' on '{var_name}' already completed "
|
|
771
|
+
f"under plan v{plan.version} (result id {existing['id']}, "
|
|
772
|
+
f"computed {existing['computed_at']}). "
|
|
773
|
+
f"Skipping — use --rerun to force recomputation."
|
|
774
|
+
)
|
|
775
|
+
continue
|
|
776
|
+
|
|
777
|
+
# For post-hoc tests, scan forward to find which amendment version
|
|
778
|
+
# first declared this specific test (test_name + rationale match)
|
|
779
|
+
# and record its amendment_reason and declaring version.
|
|
780
|
+
ph_reason = ""
|
|
781
|
+
declaring_version = plan.version
|
|
782
|
+
if is_pre_registered == 0:
|
|
783
|
+
for v in range(1, plan.version + 1):
|
|
784
|
+
try:
|
|
785
|
+
p = load_plan(args.study_id, version=v)
|
|
786
|
+
except Exception:
|
|
787
|
+
continue
|
|
788
|
+
reason = getattr(p, "amendment_reason", "")
|
|
789
|
+
if not reason:
|
|
790
|
+
continue
|
|
791
|
+
for pt in p.post_hoc_tests:
|
|
792
|
+
if pt.get("test_name") == test_name and (
|
|
793
|
+
not test_rationale
|
|
794
|
+
or not pt.get("rationale", "")
|
|
795
|
+
or pt.get("rationale", "") == test_rationale
|
|
796
|
+
):
|
|
797
|
+
ph_reason = reason
|
|
798
|
+
declaring_version = v
|
|
799
|
+
break
|
|
800
|
+
if ph_reason:
|
|
801
|
+
break
|
|
802
|
+
if var_name in plan_warnings and not force:
|
|
803
|
+
# Suggest the appropriate alternative based on test type
|
|
804
|
+
alt_test = {"chi_square": "fishers_exact",
|
|
805
|
+
"t_test": "mann_whitney",
|
|
806
|
+
"paired_t_test": "wilcoxon_signed_rank"}.get(test_name, "")
|
|
807
|
+
alt_hint = ""
|
|
808
|
+
if alt_test:
|
|
809
|
+
alt_hint = (
|
|
810
|
+
f"Redeclare the plan with an alternative test via:\n"
|
|
811
|
+
f" research-tool plan ... --test \"{var_name}:{alt_test}:...\"\n"
|
|
812
|
+
)
|
|
813
|
+
print(
|
|
814
|
+
f"Skipping '{test_name}' on '{var_name}' — "
|
|
815
|
+
f"plan time warning recorded:\n"
|
|
816
|
+
f" {plan_warnings[var_name]}\n"
|
|
817
|
+
f"Use '--force' to run despite warnings, or {alt_hint}",
|
|
818
|
+
file=sys.stderr,
|
|
819
|
+
)
|
|
820
|
+
# Record the skipped test in the DB so the audit trail is complete
|
|
821
|
+
skipped_uro = {
|
|
822
|
+
"test_name": test_name,
|
|
823
|
+
"is_pre_registered": is_pre_registered,
|
|
824
|
+
"statistic": None,
|
|
825
|
+
"p_value": None,
|
|
826
|
+
"ci_lower": None,
|
|
827
|
+
"ci_upper": None,
|
|
828
|
+
"params": {},
|
|
829
|
+
"effect_size": None,
|
|
830
|
+
"sample_counts": {"n_total": len(df), "n_analyzed": 0, "n_excluded": len(df)},
|
|
831
|
+
"status": "skipped_assumption_violation",
|
|
832
|
+
"reason": plan_warnings[var_name],
|
|
833
|
+
"rationale": test_rationale,
|
|
834
|
+
"amendment_reason": ph_reason,
|
|
835
|
+
"declaring_version": declaring_version,
|
|
836
|
+
}
|
|
837
|
+
results.append(skipped_uro)
|
|
838
|
+
continue
|
|
839
|
+
|
|
840
|
+
kwargs = {"outcome_col": var_name, "group_col": "treatment_arm"}
|
|
841
|
+
if test_name in ("kaplan_meier_logrank", "cox_proportional_hazards"):
|
|
842
|
+
# Convention: time_to_event variable named e.g. "pfs_days"
|
|
843
|
+
# has its event indicator in a column with the same prefix + "_event"
|
|
844
|
+
# or the same prefix minus "_days"/"_months" + "_event"
|
|
845
|
+
prefix = var_name.replace("_days", "").replace("_months", "").replace("_time", "")
|
|
846
|
+
kwargs["time_col"] = var_name
|
|
847
|
+
kwargs["event_col"] = f"{prefix}_event"
|
|
848
|
+
try:
|
|
849
|
+
result = run_test(test_name, df, **kwargs)
|
|
850
|
+
except Exception as e:
|
|
851
|
+
# Catch per-test failures, record error, continue batch
|
|
852
|
+
result = {
|
|
853
|
+
"test_name": test_name,
|
|
854
|
+
"statistic": None,
|
|
855
|
+
"p_value": None,
|
|
856
|
+
"ci_lower": None,
|
|
857
|
+
"ci_upper": None,
|
|
858
|
+
"params": {},
|
|
859
|
+
"effect_size": None,
|
|
860
|
+
"sample_counts": {"n_total": len(df), "n_analyzed": 0, "n_excluded": len(df)},
|
|
861
|
+
}
|
|
862
|
+
result["status"] = "error"
|
|
863
|
+
result["reason"] = str(e)
|
|
864
|
+
else:
|
|
865
|
+
result["status"] = "completed"
|
|
866
|
+
result["reason"] = None
|
|
867
|
+
result["rationale"] = t.get("rationale", "")
|
|
868
|
+
result["variable_name"] = var_name
|
|
869
|
+
result["is_pre_registered"] = is_pre_registered
|
|
870
|
+
result["amendment_reason"] = ph_reason
|
|
871
|
+
result["declaring_version"] = declaring_version
|
|
872
|
+
if result.get("params", {}).get("error"):
|
|
873
|
+
result["status"] = "error"
|
|
874
|
+
result["reason"] = result["params"]["error"]
|
|
875
|
+
results.append(result)
|
|
876
|
+
|
|
877
|
+
# Run Cox PH models if declared in the plan
|
|
878
|
+
cox_ph_models = getattr(plan, "cox_ph_models", [])
|
|
879
|
+
if cox_ph_models:
|
|
880
|
+
for model in cox_ph_models:
|
|
881
|
+
model_name = _model_field(model, "model_name")
|
|
882
|
+
survival_time_col = _model_field(model, "survival_time_col")
|
|
883
|
+
event_col = _model_field(model, "event_col")
|
|
884
|
+
primary_treatment_col = _model_field(model, "primary_treatment_col")
|
|
885
|
+
covariate_cols = _model_field(model, "covariate_cols", [])
|
|
886
|
+
model_rationale = _model_field(model, "rationale")
|
|
887
|
+
|
|
888
|
+
if not model_name or not survival_time_col:
|
|
889
|
+
continue
|
|
890
|
+
|
|
891
|
+
# Check for assumption warnings
|
|
892
|
+
warning_key = f"cox_ph_model:{model_name}"
|
|
893
|
+
if warning_key in plan_warnings and not force:
|
|
894
|
+
print(
|
|
895
|
+
f"Skipping Cox PH model '{model_name}' — "
|
|
896
|
+
f"plan time warning recorded:\n"
|
|
897
|
+
f" {plan_warnings[warning_key]}\n"
|
|
898
|
+
f"Use '--force' to run despite warnings.",
|
|
899
|
+
file=sys.stderr,
|
|
900
|
+
)
|
|
901
|
+
skipped_uro = {
|
|
902
|
+
"test_name": "cox_ph_model",
|
|
903
|
+
"statistic": None,
|
|
904
|
+
"p_value": None,
|
|
905
|
+
"ci_lower": None,
|
|
906
|
+
"ci_upper": None,
|
|
907
|
+
"params": {},
|
|
908
|
+
"effect_size": None,
|
|
909
|
+
"sample_counts": {"n_total": len(df), "n_analyzed": 0, "n_excluded": len(df)},
|
|
910
|
+
"status": "skipped_assumption_violation",
|
|
911
|
+
"reason": plan_warnings[warning_key],
|
|
912
|
+
"rationale": model_rationale,
|
|
913
|
+
"amendment_reason": "",
|
|
914
|
+
"declaring_version": plan.version,
|
|
915
|
+
}
|
|
916
|
+
results.append(skipped_uro)
|
|
917
|
+
continue
|
|
918
|
+
|
|
919
|
+
# Dedup check
|
|
920
|
+
if not rerun:
|
|
921
|
+
existing = conn.execute(
|
|
922
|
+
"""SELECT id, computed_at FROM analysis_results
|
|
923
|
+
WHERE study_id=? AND test_name=? AND variable_ids_used=? AND
|
|
924
|
+
study_plan_version=? AND is_pre_registered=?
|
|
925
|
+
AND json_extract(status_json, '$.status') = 'completed'
|
|
926
|
+
AND superseded_previous_result_id IS NULL
|
|
927
|
+
ORDER BY id DESC LIMIT 1""",
|
|
928
|
+
(args.study_id, "cox_ph_model", json.dumps([model_name]),
|
|
929
|
+
plan.version, 1), # Cox PH models are always pre-registered
|
|
930
|
+
).fetchone()
|
|
931
|
+
if existing:
|
|
932
|
+
print(
|
|
933
|
+
f"Cox PH model '{model_name}' already completed "
|
|
934
|
+
f"under plan v{plan.version} (result id {existing['id']}, "
|
|
935
|
+
f"computed {existing['computed_at']}). "
|
|
936
|
+
f"Skipping — use --rerun to force recomputation."
|
|
937
|
+
)
|
|
938
|
+
continue
|
|
939
|
+
|
|
940
|
+
# Look up variable types from classifier
|
|
941
|
+
cur3 = conn.execute(
|
|
942
|
+
"SELECT column_name, data_type FROM variables WHERE study_id=?",
|
|
943
|
+
(args.study_id,),
|
|
944
|
+
)
|
|
945
|
+
var_types = {r["column_name"]: r["data_type"] for r in cur3.fetchall()}
|
|
946
|
+
|
|
947
|
+
interaction_terms = _model_field(model, "interaction_terms", [])
|
|
948
|
+
|
|
949
|
+
# Run the multivariable Cox PH model
|
|
950
|
+
try:
|
|
951
|
+
result = run_test(
|
|
952
|
+
"cox_ph_model",
|
|
953
|
+
df,
|
|
954
|
+
outcome_col=survival_time_col,
|
|
955
|
+
group_col=primary_treatment_col,
|
|
956
|
+
time_col=survival_time_col,
|
|
957
|
+
event_col=event_col,
|
|
958
|
+
covariates=covariate_cols,
|
|
959
|
+
var_types=var_types,
|
|
960
|
+
interaction_terms=interaction_terms,
|
|
961
|
+
)
|
|
962
|
+
except Exception as e:
|
|
963
|
+
result = {
|
|
964
|
+
"test_name": "cox_ph_model",
|
|
965
|
+
"statistic": None,
|
|
966
|
+
"p_value": None,
|
|
967
|
+
"ci_lower": None,
|
|
968
|
+
"ci_upper": None,
|
|
969
|
+
"params": {},
|
|
970
|
+
"effect_size": None,
|
|
971
|
+
"sample_counts": {"n_total": len(df), "n_analyzed": 0, "n_excluded": len(df)},
|
|
972
|
+
}
|
|
973
|
+
result["status"] = "error"
|
|
974
|
+
result["reason"] = str(e)
|
|
975
|
+
else:
|
|
976
|
+
result["status"] = "completed"
|
|
977
|
+
result["reason"] = None
|
|
978
|
+
result["rationale"] = model_rationale
|
|
979
|
+
result["variable_name"] = model_name
|
|
980
|
+
result["test_name"] = "cox_ph_model"
|
|
981
|
+
result["amendment_reason"] = ""
|
|
982
|
+
result["declaring_version"] = plan.version
|
|
983
|
+
if result.get("params", {}).get("error"):
|
|
984
|
+
result["status"] = "error"
|
|
985
|
+
result["reason"] = result["params"]["error"]
|
|
986
|
+
results.append(result)
|
|
987
|
+
|
|
988
|
+
# Promote completed Cox PH results with Schoenfeld violations BEFORE
|
|
989
|
+
# multiple-testing correction (M3 fix: correction must see final statuses)
|
|
990
|
+
for r in results:
|
|
991
|
+
if r.get("status") != "completed":
|
|
992
|
+
continue
|
|
993
|
+
params = r.get("params", {})
|
|
994
|
+
ph_diag = params.get("assumption_diagnostics") if params else None
|
|
995
|
+
if ph_diag:
|
|
996
|
+
covs = ph_diag.get("covariates", [])
|
|
997
|
+
violations = [c for c in covs if c.get("p_value", 1) < 0.05]
|
|
998
|
+
if violations:
|
|
999
|
+
names = "; ".join(f"{v['covariate']} (p={v['p_value']:.4f})" for v in violations)
|
|
1000
|
+
r["status"] = "assumption_violation"
|
|
1001
|
+
r["reason"] = f"Proportional hazards assumption violated: {names}"
|
|
1002
|
+
|
|
1003
|
+
# Apply multiple-testing correction to completed tests only
|
|
1004
|
+
completed = [r for r in results if r["status"] == "completed"]
|
|
1005
|
+
completed_p = [r["p_value"] for r in completed if r["p_value"] is not None]
|
|
1006
|
+
if len(completed_p) > 1:
|
|
1007
|
+
corrected = correct(completed_p)
|
|
1008
|
+
for r, cp in zip(completed, corrected):
|
|
1009
|
+
r["adjusted_p_value"] = cp
|
|
1010
|
+
elif len(completed_p) == 1:
|
|
1011
|
+
completed[0]["adjusted_p_value"] = completed[0]["p_value"]
|
|
1012
|
+
# Skipped/error results keep adjusted_p_value = None (already set by _uro)
|
|
1013
|
+
|
|
1014
|
+
# Track superseded results for --rerun
|
|
1015
|
+
supersede_map: dict[str, int] = {}
|
|
1016
|
+
if rerun:
|
|
1017
|
+
for test_list, is_pre_registered, _label in test_runs:
|
|
1018
|
+
for t in test_list:
|
|
1019
|
+
var_name = t.get("variable_name", "")
|
|
1020
|
+
test_name = t.get("test_name", "")
|
|
1021
|
+
if not test_name or not var_name:
|
|
1022
|
+
continue
|
|
1023
|
+
existing = conn.execute(
|
|
1024
|
+
"""SELECT id FROM analysis_results
|
|
1025
|
+
WHERE study_id=? AND test_name=? AND variable_ids_used=? AND
|
|
1026
|
+
study_plan_version=? AND is_pre_registered=?
|
|
1027
|
+
AND json_extract(status_json, '$.status') = 'completed'
|
|
1028
|
+
AND superseded_previous_result_id IS NULL
|
|
1029
|
+
ORDER BY id DESC LIMIT 1""",
|
|
1030
|
+
(args.study_id, test_name, json.dumps([var_name]),
|
|
1031
|
+
plan.version, is_pre_registered),
|
|
1032
|
+
).fetchone()
|
|
1033
|
+
if existing:
|
|
1034
|
+
supersede_map[test_name] = existing["id"]
|
|
1035
|
+
|
|
1036
|
+
# Also check for Cox PH models
|
|
1037
|
+
if cox_ph_models:
|
|
1038
|
+
for model in cox_ph_models:
|
|
1039
|
+
mn = _model_field(model, "model_name")
|
|
1040
|
+
if not mn:
|
|
1041
|
+
continue
|
|
1042
|
+
existing = conn.execute(
|
|
1043
|
+
"""SELECT id FROM analysis_results
|
|
1044
|
+
WHERE study_id=? AND test_name=? AND variable_ids_used=? AND
|
|
1045
|
+
study_plan_version=? AND is_pre_registered=?
|
|
1046
|
+
AND json_extract(status_json, '$.status') = 'completed'
|
|
1047
|
+
AND superseded_previous_result_id IS NULL
|
|
1048
|
+
ORDER BY id DESC LIMIT 1""",
|
|
1049
|
+
(args.study_id, "cox_ph_model", json.dumps([mn]),
|
|
1050
|
+
plan.version, 1),
|
|
1051
|
+
).fetchone()
|
|
1052
|
+
if existing:
|
|
1053
|
+
supersede_map[f"cox_ph_model:{mn}"] = existing["id"]
|
|
1054
|
+
|
|
1055
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
1056
|
+
for r in results:
|
|
1057
|
+
status_record = {"status": r.get("status", "completed")}
|
|
1058
|
+
if r.get("reason"):
|
|
1059
|
+
status_record["reason"] = r["reason"]
|
|
1060
|
+
|
|
1061
|
+
stored_version = r.get("declaring_version", plan.version)
|
|
1062
|
+
prov = {"plan_version": stored_version}
|
|
1063
|
+
ph_reason = r.get("amendment_reason", "")
|
|
1064
|
+
if ph_reason:
|
|
1065
|
+
prov["amendment_reason"] = ph_reason
|
|
1066
|
+
rationale = r.get("rationale", "")
|
|
1067
|
+
if rationale:
|
|
1068
|
+
prov["rationale"] = rationale
|
|
1069
|
+
superseded_id = supersede_map.get(
|
|
1070
|
+
f"{r.get('test_name', '')}:{r.get('variable_name', '')}"
|
|
1071
|
+
if r.get("test_name") == "cox_ph_model"
|
|
1072
|
+
else r.get("test_name", "")
|
|
1073
|
+
)
|
|
1074
|
+
params = r.get("params", {})
|
|
1075
|
+
lr_test_p = params.get("lr_test_p_value") if params else None
|
|
1076
|
+
concordance = params.get("concordance_index") if params else None
|
|
1077
|
+
ph_diag = params.get("assumption_diagnostics") if params else None
|
|
1078
|
+
cursor = conn.execute(
|
|
1079
|
+
"""INSERT INTO analysis_results
|
|
1080
|
+
(study_id, study_plan_version, variable_ids_used, test_name,
|
|
1081
|
+
statistic, p_value, adjusted_p_value, ci_lower, ci_upper,
|
|
1082
|
+
effect_size_json, sample_counts_json, status_json,
|
|
1083
|
+
is_pre_registered, provenance_json, computed_at,
|
|
1084
|
+
superseded_previous_result_id,
|
|
1085
|
+
lr_test_p, concordance_index, ph_diagnostics_json)
|
|
1086
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
1087
|
+
(args.study_id, stored_version, json.dumps([r.get("variable_name", "")]),
|
|
1088
|
+
r["test_name"], r["statistic"], r["p_value"],
|
|
1089
|
+
r.get("adjusted_p_value"), r.get("ci_lower"), r.get("ci_upper"),
|
|
1090
|
+
json.dumps(r["effect_size"]) if r.get("effect_size") else None,
|
|
1091
|
+
json.dumps(r["sample_counts"]) if r.get("sample_counts") else None,
|
|
1092
|
+
json.dumps(status_record),
|
|
1093
|
+
r.get("is_pre_registered", 1),
|
|
1094
|
+
json.dumps(prov), now,
|
|
1095
|
+
superseded_id,
|
|
1096
|
+
lr_test_p, concordance,
|
|
1097
|
+
json.dumps(ph_diag) if ph_diag else None),
|
|
1098
|
+
)
|
|
1099
|
+
result_id = cursor.lastrowid
|
|
1100
|
+
|
|
1101
|
+
# Insert per-covariate results for Cox PH models
|
|
1102
|
+
if r.get("test_name") == "cox_ph_model" and params:
|
|
1103
|
+
cov_results = params.get("per_covariate_results", [])
|
|
1104
|
+
for cr in cov_results:
|
|
1105
|
+
conn.execute(
|
|
1106
|
+
"""INSERT INTO analysis_covariate_results
|
|
1107
|
+
(result_id, covariate, hr, ci_lower, ci_upper,
|
|
1108
|
+
wald_p, coef, se, z, reference_level, tested_level)
|
|
1109
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
1110
|
+
(result_id,
|
|
1111
|
+
cr.get("covariate"), cr.get("hr"),
|
|
1112
|
+
cr.get("ci_lower"), cr.get("ci_upper"),
|
|
1113
|
+
cr.get("wald_p"), cr.get("coef"),
|
|
1114
|
+
cr.get("se"), cr.get("z"),
|
|
1115
|
+
cr.get("reference_level"),
|
|
1116
|
+
cr.get("tested_level")),
|
|
1117
|
+
)
|
|
1118
|
+
conn.commit()
|
|
1119
|
+
conn.close()
|
|
1120
|
+
|
|
1121
|
+
for r in results:
|
|
1122
|
+
p_str = f"p={r['p_value']:.4f}" if r['p_value'] is not None else "error"
|
|
1123
|
+
print(f" {r['test_name']}: stat={r['statistic']}, {p_str}")
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def cmd_strobe_check(args: argparse.Namespace) -> None:
|
|
1127
|
+
"""Generate STROBE compliance report."""
|
|
1128
|
+
report = generate_report(args.study_id)
|
|
1129
|
+
print(report)
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def cmd_draft(args: argparse.Namespace) -> None:
|
|
1133
|
+
"""Generate manuscript draft."""
|
|
1134
|
+
path = write_draft(args.study_id)
|
|
1135
|
+
print(f"Manuscript draft written to {path}")
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
def cmd_bundle(args: argparse.Namespace) -> None:
|
|
1139
|
+
"""Create a hash-verified portable study archive."""
|
|
1140
|
+
from core.reporting.bundle import create_bundle, format_verification_report
|
|
1141
|
+
try:
|
|
1142
|
+
result = create_bundle(args.study_id)
|
|
1143
|
+
print(f"Bundle created: {result['bundle_path']}")
|
|
1144
|
+
print(f"Composite hash: {result['composite_hash']}")
|
|
1145
|
+
except RuntimeError as e:
|
|
1146
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
1147
|
+
sys.exit(1)
|
|
1148
|
+
|
|
1149
|
+
|
|
1150
|
+
def cmd_verify_bundle(args: argparse.Namespace) -> None:
|
|
1151
|
+
"""Verify a bundle archive's integrity."""
|
|
1152
|
+
from core.reporting.bundle import verify_bundle, format_verification_report
|
|
1153
|
+
try:
|
|
1154
|
+
result = verify_bundle(args.bundle_path)
|
|
1155
|
+
print(format_verification_report(result))
|
|
1156
|
+
if not result["valid"]:
|
|
1157
|
+
sys.exit(1)
|
|
1158
|
+
except (FileNotFoundError, ValueError) as e:
|
|
1159
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
1160
|
+
sys.exit(1)
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
def cmd_plot_km(args: argparse.Namespace) -> None:
|
|
1164
|
+
"""Generate a Kaplan-Meier survival curve plot for a completed KM test."""
|
|
1165
|
+
from core.reporting.plots import generate_km_plot
|
|
1166
|
+
from pathlib import Path
|
|
1167
|
+
|
|
1168
|
+
fmt = getattr(args, "format", "svg")
|
|
1169
|
+
show_risk_table = not getattr(args, "no_risk_table", False)
|
|
1170
|
+
show_medians = False if getattr(args, "no_medians", False) else None
|
|
1171
|
+
base_output = getattr(args, "output", None)
|
|
1172
|
+
time_unit = getattr(args, "time_unit", "months")
|
|
1173
|
+
style = getattr(args, "style", "clean")
|
|
1174
|
+
|
|
1175
|
+
styles_to_generate = ["clean", "scientific", "presentation"] if style == "all" else [style]
|
|
1176
|
+
|
|
1177
|
+
generated: list[Path] = []
|
|
1178
|
+
for s in styles_to_generate:
|
|
1179
|
+
if base_output is not None:
|
|
1180
|
+
base = Path(base_output)
|
|
1181
|
+
if style == "all":
|
|
1182
|
+
out = base.with_name(base.stem + f"_{s}" + base.suffix)
|
|
1183
|
+
else:
|
|
1184
|
+
out = base
|
|
1185
|
+
else:
|
|
1186
|
+
# Always include the style name in the default filename so
|
|
1187
|
+
# sequential invocations (--style clean, --style scientific, ...)
|
|
1188
|
+
# don't silently overwrite each other. This applies even when
|
|
1189
|
+
# the user omits --style entirely (defaults to "clean") — the
|
|
1190
|
+
# resulting km_plot_1_clean.svg is unambiguous and safe.
|
|
1191
|
+
from core.database import DATA_ROOT
|
|
1192
|
+
out = DATA_ROOT / args.study_id / f"km_plot_{args.test_id}_{s}.{fmt}"
|
|
1193
|
+
|
|
1194
|
+
try:
|
|
1195
|
+
path = generate_km_plot(
|
|
1196
|
+
args.study_id, args.test_id,
|
|
1197
|
+
output_path=out,
|
|
1198
|
+
fmt=fmt,
|
|
1199
|
+
show_risk_table=show_risk_table,
|
|
1200
|
+
show_medians=show_medians,
|
|
1201
|
+
time_unit_display=time_unit,
|
|
1202
|
+
style=s,
|
|
1203
|
+
)
|
|
1204
|
+
generated.append(path)
|
|
1205
|
+
except (ValueError, FileNotFoundError) as e:
|
|
1206
|
+
print(f"Error generating {s} style: {e}", file=sys.stderr)
|
|
1207
|
+
if style != "all":
|
|
1208
|
+
sys.exit(1)
|
|
1209
|
+
|
|
1210
|
+
for p in generated:
|
|
1211
|
+
print(f"Kaplan-Meier plot saved to {p}")
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def cmd_export_excel(args: argparse.Namespace) -> None:
|
|
1215
|
+
"""Generate a publication-ready Excel report with KM plot, Table 1, and audit hashes."""
|
|
1216
|
+
from core.reporting.excel_export import generate_excel_report
|
|
1217
|
+
output_path = getattr(args, "output", None)
|
|
1218
|
+
try:
|
|
1219
|
+
path = generate_excel_report(args.study_id, output_path=output_path)
|
|
1220
|
+
print(f"Excel report saved to {path}")
|
|
1221
|
+
except RuntimeError as e:
|
|
1222
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
1223
|
+
sys.exit(1)
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
def cmd_export(args: argparse.Namespace) -> None:
|
|
1227
|
+
"""Export study as study_result.v1.json (portable, reviewer-ready)."""
|
|
1228
|
+
# Check for post-unmask diagnostic violations before export
|
|
1229
|
+
from core.planning.diagnostics import check_violation
|
|
1230
|
+
conn = get_connection(args.study_id)
|
|
1231
|
+
violations: list[str] = []
|
|
1232
|
+
for a in conn.execute(
|
|
1233
|
+
"SELECT status_json, ph_diagnostics_json FROM analysis_results WHERE study_id=?",
|
|
1234
|
+
(args.study_id,),
|
|
1235
|
+
).fetchall():
|
|
1236
|
+
has_v, summary, _ = check_violation(dict(a))
|
|
1237
|
+
if has_v:
|
|
1238
|
+
violations.append(summary)
|
|
1239
|
+
if violations and not getattr(args, "acknowledge_violations", False):
|
|
1240
|
+
print(
|
|
1241
|
+
f"Error: {len(violations)} post-unmask diagnostic violation(s) found:\n",
|
|
1242
|
+
file=sys.stderr,
|
|
1243
|
+
)
|
|
1244
|
+
for v in violations:
|
|
1245
|
+
print(f" • {v}", file=sys.stderr)
|
|
1246
|
+
print(
|
|
1247
|
+
"\nExported results would include these violations — they are disclosed in output\n"
|
|
1248
|
+
"artifacts. Pass --acknowledge-violations to confirm you have reviewed them\n"
|
|
1249
|
+
"and wish to proceed with export.",
|
|
1250
|
+
file=sys.stderr,
|
|
1251
|
+
)
|
|
1252
|
+
conn.close()
|
|
1253
|
+
sys.exit(1)
|
|
1254
|
+
conn.close()
|
|
1255
|
+
|
|
1256
|
+
import json
|
|
1257
|
+
from datetime import datetime, timezone
|
|
1258
|
+
import pandas as pd
|
|
1259
|
+
|
|
1260
|
+
conn = get_connection(args.study_id)
|
|
1261
|
+
raw_table = f"raw_{args.study_id}"
|
|
1262
|
+
|
|
1263
|
+
# ── study metadata ──────────────────────────────────────────────────
|
|
1264
|
+
cur = conn.execute("SELECT * FROM studies WHERE id=?", (args.study_id,))
|
|
1265
|
+
study = cur.fetchone()
|
|
1266
|
+
if not study:
|
|
1267
|
+
print(f"Error: study '{args.study_id}' not found.", file=sys.stderr)
|
|
1268
|
+
sys.exit(1)
|
|
1269
|
+
|
|
1270
|
+
export = {
|
|
1271
|
+
"schema_version": "1.0.0",
|
|
1272
|
+
"export_mode": args.mode,
|
|
1273
|
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
1274
|
+
|
|
1275
|
+
"study_metadata": {
|
|
1276
|
+
"study_id": args.study_id,
|
|
1277
|
+
"name": study["name"],
|
|
1278
|
+
"design_type": study["study_type"] or "cohort",
|
|
1279
|
+
"created_at": study["created_at"],
|
|
1280
|
+
"study_state": {0: "pre_locked", 1: "locked", 2: "unmasked"}.get(study["is_locked"], "unknown"),
|
|
1281
|
+
"unmasked_at": study["unmasked_at"] if study["is_locked"] >= 2 else None,
|
|
1282
|
+
},
|
|
1283
|
+
|
|
1284
|
+
# ── variable catalog ─────────────────────────────────────────────
|
|
1285
|
+
"variable_catalog": [],
|
|
1286
|
+
|
|
1287
|
+
# ── locked plan ──────────────────────────────────────────────────
|
|
1288
|
+
"locked_plan": None,
|
|
1289
|
+
|
|
1290
|
+
# ── URO analysis results ─────────────────────────────────────────
|
|
1291
|
+
"analysis_results": [],
|
|
1292
|
+
|
|
1293
|
+
# ── Table 1 (baseline) ───────────────────────────────────────────
|
|
1294
|
+
"table1": None,
|
|
1295
|
+
|
|
1296
|
+
# ── Analysis summary ─────────────────────────────────────────────
|
|
1297
|
+
"analysis_summary": None,
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
# Variables
|
|
1301
|
+
cur = conn.execute(
|
|
1302
|
+
"SELECT id, column_name, role, data_type FROM variables WHERE study_id=? ORDER BY id",
|
|
1303
|
+
(args.study_id,),
|
|
1304
|
+
)
|
|
1305
|
+
for r in cur.fetchall():
|
|
1306
|
+
export["variable_catalog"].append({
|
|
1307
|
+
"id": r["id"],
|
|
1308
|
+
"column": r["column_name"],
|
|
1309
|
+
"role": r["role"],
|
|
1310
|
+
"data_type": r["data_type"],
|
|
1311
|
+
})
|
|
1312
|
+
|
|
1313
|
+
# Locked plan
|
|
1314
|
+
locked_paths = list(DATA_ROOT.glob(f"{args.study_id}/study_plan.v*.locked.json"))
|
|
1315
|
+
if locked_paths:
|
|
1316
|
+
latest = sorted(locked_paths)[-1]
|
|
1317
|
+
export["locked_plan"] = json.loads(latest.read_text())
|
|
1318
|
+
export["study_metadata"]["plan_version"] = latest.stem.split(".")[1]
|
|
1319
|
+
else:
|
|
1320
|
+
export["study_metadata"]["plan_version"] = None
|
|
1321
|
+
|
|
1322
|
+
# Analysis results (UROs)
|
|
1323
|
+
cur = conn.execute(
|
|
1324
|
+
"""SELECT id, test_name, statistic, p_value, adjusted_p_value,
|
|
1325
|
+
ci_lower, ci_upper, effect_size_json, sample_counts_json,
|
|
1326
|
+
status_json,
|
|
1327
|
+
is_pre_registered, computed_at
|
|
1328
|
+
FROM analysis_results WHERE study_id=? ORDER BY id""",
|
|
1329
|
+
(args.study_id,),
|
|
1330
|
+
)
|
|
1331
|
+
for r in cur.fetchall():
|
|
1332
|
+
status_data = json.loads(r["status_json"]) if r["status_json"] else {"status": "completed"}
|
|
1333
|
+
uro = {
|
|
1334
|
+
"test_id": f"t{r['id']}",
|
|
1335
|
+
"test_name": r["test_name"],
|
|
1336
|
+
"status": status_data.get("status", "completed"),
|
|
1337
|
+
"reason": status_data.get("reason"),
|
|
1338
|
+
"statistic": {"name": r["test_name"], "value": r["statistic"]} if r["statistic"] is not None else None,
|
|
1339
|
+
"p_value": r["p_value"],
|
|
1340
|
+
"adjusted_p_value": r["adjusted_p_value"],
|
|
1341
|
+
"confidence_interval": {
|
|
1342
|
+
"level": 0.95,
|
|
1343
|
+
"low": r["ci_lower"],
|
|
1344
|
+
"high": r["ci_upper"],
|
|
1345
|
+
} if r["ci_lower"] is not None else None,
|
|
1346
|
+
"effect_size": json.loads(r["effect_size_json"]) if r["effect_size_json"] else None,
|
|
1347
|
+
"sample_counts": json.loads(r["sample_counts_json"]) if r["sample_counts_json"] else None,
|
|
1348
|
+
"is_pre_registered": bool(r["is_pre_registered"]),
|
|
1349
|
+
"computed_at": r["computed_at"],
|
|
1350
|
+
}
|
|
1351
|
+
export["analysis_results"].append(uro)
|
|
1352
|
+
|
|
1353
|
+
# Table 1
|
|
1354
|
+
export_groupby = None
|
|
1355
|
+
if locked_paths:
|
|
1356
|
+
from core.planning.lock import load_plan as _load_plan_m9
|
|
1357
|
+
try:
|
|
1358
|
+
_plan = _load_plan_m9(args.study_id)
|
|
1359
|
+
export_groupby = _plan.primary_treatment_col or "treatment_arm"
|
|
1360
|
+
except (FileNotFoundError, ValueError):
|
|
1361
|
+
export_groupby = "treatment_arm"
|
|
1362
|
+
tbl = generate_table1(args.study_id, groupby=export_groupby)
|
|
1363
|
+
if tbl is not None and not tbl.empty:
|
|
1364
|
+
# Flatten MultiIndex columns: ('Grouped by treatment_arm', 'A') → 'A'
|
|
1365
|
+
headers = []
|
|
1366
|
+
for col in tbl.columns:
|
|
1367
|
+
if isinstance(col, tuple):
|
|
1368
|
+
# Take the last non-empty element
|
|
1369
|
+
parts = [str(p).strip() for p in col if str(p).strip()]
|
|
1370
|
+
headers.append(parts[-1] if parts else str(col))
|
|
1371
|
+
else:
|
|
1372
|
+
headers.append(str(col).strip())
|
|
1373
|
+
|
|
1374
|
+
rows = []
|
|
1375
|
+
for idx, row in tbl.iterrows():
|
|
1376
|
+
parts = [str(p).strip() for p in (idx if isinstance(idx, tuple) else [idx])]
|
|
1377
|
+
characteristic = parts[0] if parts else ""
|
|
1378
|
+
|
|
1379
|
+
# Skip the grouping column itself (redundant when groupby is active)
|
|
1380
|
+
if export_groupby and characteristic.lower().startswith(export_groupby.lower()):
|
|
1381
|
+
continue
|
|
1382
|
+
|
|
1383
|
+
category_level = parts[1] if len(parts) > 1 else None
|
|
1384
|
+
vals = {}
|
|
1385
|
+
for h, raw_col in zip(headers, tbl.columns):
|
|
1386
|
+
vals[h] = str(row[raw_col]).strip()
|
|
1387
|
+
entry = {"label": characteristic, **vals}
|
|
1388
|
+
if category_level:
|
|
1389
|
+
entry["category_level"] = category_level
|
|
1390
|
+
rows.append(entry)
|
|
1391
|
+
export["table1"] = {
|
|
1392
|
+
"headers": ["Characteristic"] + headers,
|
|
1393
|
+
"rows": rows,
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
# Analysis summary
|
|
1397
|
+
pre_reg = [r for r in export["analysis_results"] if r.get("is_pre_registered")]
|
|
1398
|
+
post_hoc = [r for r in export["analysis_results"] if not r.get("is_pre_registered")]
|
|
1399
|
+
export["analysis_summary"] = {
|
|
1400
|
+
"n_pre_registered": len(pre_reg),
|
|
1401
|
+
"n_pre_registered_completed": sum(
|
|
1402
|
+
1 for r in pre_reg if r.get("status") == "completed"
|
|
1403
|
+
),
|
|
1404
|
+
"n_post_hoc": len(post_hoc),
|
|
1405
|
+
"n_post_hoc_significant": sum(
|
|
1406
|
+
1 for r in post_hoc
|
|
1407
|
+
if r.get("p_value") is not None and r["p_value"] < 0.05
|
|
1408
|
+
),
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
# Data hash for authenticity (SHA-256 of raw data, canonical JSON)
|
|
1412
|
+
try:
|
|
1413
|
+
from core.provenance.hashing import compute_raw_data_hash
|
|
1414
|
+
export["study_metadata"]["data_hash_sha256"] = compute_raw_data_hash(args.study_id)
|
|
1415
|
+
except Exception:
|
|
1416
|
+
export["study_metadata"]["data_hash_sha256"] = None
|
|
1417
|
+
|
|
1418
|
+
conn.close()
|
|
1419
|
+
|
|
1420
|
+
# Write
|
|
1421
|
+
version_label = args.study_plan_version or "v1"
|
|
1422
|
+
out_path = DATA_ROOT / args.study_id / f"study_result.{version_label}.json"
|
|
1423
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1424
|
+
out_path.write_text(json.dumps(export, indent=2, default=str))
|
|
1425
|
+
print(f"Study result exported to {out_path}")
|
|
1426
|
+
if getattr(args, "format", "json") == "appendix":
|
|
1427
|
+
from core.reporting.appendix import generate_appendix
|
|
1428
|
+
|
|
1429
|
+
appendix_path = out_path.with_name(out_path.stem + ".appendix.md")
|
|
1430
|
+
appendix_path.write_text(generate_appendix(export, args.study_id))
|
|
1431
|
+
print(f"Appendix exported to {appendix_path}")
|
|
1432
|
+
if args.mode == "supplementary":
|
|
1433
|
+
print(" (supplementary mode — no row-level data included)")
|
|
1434
|
+
|
|
1435
|
+
|
|
1436
|
+
def cmd_lineage(args: argparse.Namespace) -> None:
|
|
1437
|
+
"""Render study provenance DAG to stdout (text) or file (SVG)."""
|
|
1438
|
+
from core.reporting.lineage import assemble_events, render_text, render_svg
|
|
1439
|
+
|
|
1440
|
+
events = assemble_events(args.study_id)
|
|
1441
|
+
if args.svg:
|
|
1442
|
+
render_svg(events, args.svg)
|
|
1443
|
+
print(f"Lineage DAG written to {args.svg}")
|
|
1444
|
+
else:
|
|
1445
|
+
print(render_text(events))
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def cmd_forensics(args: argparse.Namespace) -> None:
|
|
1449
|
+
"""Run anomaly-detection checks on study data."""
|
|
1450
|
+
from core.reporting.forensics import run_forensics
|
|
1451
|
+
|
|
1452
|
+
report_path = run_forensics(args.study_id)
|
|
1453
|
+
print(f"Forensics report written to {report_path}")
|
|
1454
|
+
|
|
1455
|
+
|
|
1456
|
+
def cmd_forest_plot(args: argparse.Namespace) -> None:
|
|
1457
|
+
"""Render publication-ready forest plot for Cox PH results."""
|
|
1458
|
+
from core.reporting.forest_plot import render_forest
|
|
1459
|
+
|
|
1460
|
+
try:
|
|
1461
|
+
result = render_forest(args.study_id, args.output, ascii=args.ascii)
|
|
1462
|
+
except ValueError as e:
|
|
1463
|
+
print(f"Error: {e}\nRun 'plan', 'lock', 'unmask', and 'analyze' first.", file=sys.stderr)
|
|
1464
|
+
sys.exit(1)
|
|
1465
|
+
if args.ascii:
|
|
1466
|
+
print(result)
|
|
1467
|
+
else:
|
|
1468
|
+
print(f"Forest plot written to {result}")
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
def cmd_flowchart(args: argparse.Namespace) -> None:
|
|
1472
|
+
"""Render CONSORT/STROBE patient flow diagram."""
|
|
1473
|
+
result = render_flowchart(args.study_id, args.output, ascii=args.ascii,
|
|
1474
|
+
show_title=not args.no_title,
|
|
1475
|
+
show_watermark=args.watermark,
|
|
1476
|
+
verbose=args.verbose,
|
|
1477
|
+
show_study_name=args.show_study_name)
|
|
1478
|
+
if args.ascii:
|
|
1479
|
+
print(result)
|
|
1480
|
+
else:
|
|
1481
|
+
print(f"Flowchart written to {result}")
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
1485
|
+
p = argparse.ArgumentParser(prog="research-tool", description="Retrospective clinical research tool")
|
|
1486
|
+
sub = p.add_subparsers(dest="command")
|
|
1487
|
+
|
|
1488
|
+
# new-study
|
|
1489
|
+
sp = sub.add_parser("new-study", help="Create a new study")
|
|
1490
|
+
sp.add_argument("name")
|
|
1491
|
+
sp.set_defaults(func=cmd_new_study)
|
|
1492
|
+
|
|
1493
|
+
# ingest
|
|
1494
|
+
sp = sub.add_parser("ingest", help="Load CSV/Excel into a study")
|
|
1495
|
+
sp.add_argument("study_id")
|
|
1496
|
+
sp.add_argument("file")
|
|
1497
|
+
sp.add_argument("--na-values",
|
|
1498
|
+
help="Comma-separated strings to treat as missing (e.g. 'unknown,missing,N/A,?'). "
|
|
1499
|
+
"Added on top of pandas' default NA representations.")
|
|
1500
|
+
sp.add_argument("--force-reingest", action="store_true",
|
|
1501
|
+
help="Re-ingest even if study already has data (clears variables, plans, results).")
|
|
1502
|
+
sp.set_defaults(func=cmd_ingest)
|
|
1503
|
+
|
|
1504
|
+
# classify-variables
|
|
1505
|
+
sp = sub.add_parser("classify-variables", help="Classify column roles and data types")
|
|
1506
|
+
sp.add_argument("study_id")
|
|
1507
|
+
sp.set_defaults(func=cmd_classify_variables)
|
|
1508
|
+
|
|
1509
|
+
# explore-baseline
|
|
1510
|
+
sp = sub.add_parser("explore-baseline", help="Explore baseline data (outcomes masked)")
|
|
1511
|
+
sp.add_argument("study_id")
|
|
1512
|
+
sp.add_argument("--head", type=int, default=5)
|
|
1513
|
+
sp.set_defaults(func=cmd_explore_baseline)
|
|
1514
|
+
|
|
1515
|
+
# list-variables
|
|
1516
|
+
sp = sub.add_parser("list-variables", help="List classified variables with their DB IDs")
|
|
1517
|
+
sp.add_argument("study_id")
|
|
1518
|
+
sp.set_defaults(func=cmd_list_variables)
|
|
1519
|
+
|
|
1520
|
+
# plan
|
|
1521
|
+
sp = sub.add_parser("plan", help="Declare the study plan (declare intent before seeing outcomes)")
|
|
1522
|
+
sp.add_argument("study_id")
|
|
1523
|
+
sp.add_argument("--type", dest="study_type", default="cohort",
|
|
1524
|
+
choices=["cohort", "case_control", "cross_sectional"])
|
|
1525
|
+
sp.add_argument("--comparison", required=True, help="Primary comparison or association being tested")
|
|
1526
|
+
sp.add_argument("--outcome-var-ids", required=True, help="Comma-separated variable IDs of primary outcomes")
|
|
1527
|
+
sp.add_argument("--test", action="append", dest="tests", help="Planned test in format 'var_id:test_name:rationale'")
|
|
1528
|
+
sp.add_argument("--covariates", help="Comma-separated variable IDs for covariates")
|
|
1529
|
+
sp.add_argument("--cox-ph-models", action="append", dest="cox_ph_models",
|
|
1530
|
+
help="Multivariable Cox PH model in format 'model_name:survival_time_col:event_col:primary_treatment_col:covariate_col1,covariate_col2,...:rationale'")
|
|
1531
|
+
sp.add_argument("--matching-criteria", help="Comma-separated variable IDs used for matching (case-control studies)")
|
|
1532
|
+
sp.add_argument("--interaction-terms", action="append", dest="interaction_terms",
|
|
1533
|
+
help="Interaction term for a Cox PH model in format 'model_name:var_a:var_b'")
|
|
1534
|
+
sp.add_argument("--override", action="append", dest="overrides", default=[],
|
|
1535
|
+
help="Override a classified role before lock: id=<variable_id>:role=<role>")
|
|
1536
|
+
sp.add_argument("--from-json", type=str, default=None,
|
|
1537
|
+
help="Load plan from a JSON file instead of CLI flags. "
|
|
1538
|
+
"Expected keys: comparison, outcome_var_ids, study_type, tests, "
|
|
1539
|
+
"covariates, cox_ph_models, interaction_terms, matching_criteria, overrides")
|
|
1540
|
+
sp.set_defaults(func=cmd_plan)
|
|
1541
|
+
|
|
1542
|
+
# lock
|
|
1543
|
+
sp = sub.add_parser("lock", help="Lock the study plan (immutable snapshot)")
|
|
1544
|
+
sp.add_argument("study_id")
|
|
1545
|
+
sp.add_argument("--strict-ids", action="store_true",
|
|
1546
|
+
help="Block plan lock if duplicate patient IDs are present in ingested data")
|
|
1547
|
+
sp.add_argument("--allow-duplicate-ids", action="store_true",
|
|
1548
|
+
help="Allow locking even when duplicate patient IDs are present "
|
|
1549
|
+
"(use for longitudinal/repeated-measures designs where "
|
|
1550
|
+
"the same patient legitimately appears in multiple rows)")
|
|
1551
|
+
sp.set_defaults(func=cmd_lock)
|
|
1552
|
+
|
|
1553
|
+
# amend
|
|
1554
|
+
sp = sub.add_parser("amend", help="Amend a locked study plan")
|
|
1555
|
+
sp.add_argument("study_id")
|
|
1556
|
+
sp.add_argument("--post-hoc", action="store_true",
|
|
1557
|
+
help="Post-hoc/exploratory amendment (requires unmasked study)")
|
|
1558
|
+
sp.add_argument("--reason", required=True,
|
|
1559
|
+
help="Required human-readable reason for this amendment")
|
|
1560
|
+
sp.add_argument("--test", action="append", dest="tests",
|
|
1561
|
+
help="Test to add in format 'var_name:test_name:rationale'")
|
|
1562
|
+
sp.set_defaults(func=cmd_amend)
|
|
1563
|
+
|
|
1564
|
+
# unmask
|
|
1565
|
+
sp = sub.add_parser("unmask", help="Unmask outcome data (irreversible)")
|
|
1566
|
+
sp.add_argument("study_id")
|
|
1567
|
+
sp.add_argument("--force", nargs="?", const="", default=None,
|
|
1568
|
+
help="Override forceable diagnostic warnings. Provide a justification string: "
|
|
1569
|
+
"--force 'your reason'")
|
|
1570
|
+
sp.set_defaults(func=cmd_unmask)
|
|
1571
|
+
|
|
1572
|
+
# table1
|
|
1573
|
+
sp = sub.add_parser("table1", help="Generate Table 1 (baseline characteristics)")
|
|
1574
|
+
sp.add_argument("study_id")
|
|
1575
|
+
sp.add_argument("--groupby", help="Column to group by (overrides locked plan default)")
|
|
1576
|
+
sp.add_argument("--overall", action="store_true",
|
|
1577
|
+
help="Force unstratified single-column view (ignores locked plan grouping)")
|
|
1578
|
+
sp.set_defaults(func=cmd_table1)
|
|
1579
|
+
|
|
1580
|
+
# analyze
|
|
1581
|
+
sp = sub.add_parser("analyze", help="Run pre-registered analyses from locked plan")
|
|
1582
|
+
sp.add_argument("study_id")
|
|
1583
|
+
sp.add_argument("--force", action="store_true",
|
|
1584
|
+
help="Run tests even when the plan has recorded assumption warnings")
|
|
1585
|
+
sp.add_argument("--post-hoc", action="store_true",
|
|
1586
|
+
help="Run post-hoc/exploratory tests instead of pre-registered tests")
|
|
1587
|
+
sp.add_argument("--rerun", action="store_true",
|
|
1588
|
+
help="Force recomputation even if a completed result already exists")
|
|
1589
|
+
sp.add_argument("--adjust-p", action="store_true",
|
|
1590
|
+
help="Apply multiple-testing correction across tests (default: False for unadjusted/adjusted sensitivity views of the same endpoint)")
|
|
1591
|
+
sp.set_defaults(func=cmd_analyze)
|
|
1592
|
+
|
|
1593
|
+
# strobe-check
|
|
1594
|
+
sp = sub.add_parser("strobe-check", help="Check STROBE checklist compliance")
|
|
1595
|
+
sp.add_argument("study_id")
|
|
1596
|
+
sp.set_defaults(func=cmd_strobe_check)
|
|
1597
|
+
|
|
1598
|
+
# draft
|
|
1599
|
+
sp = sub.add_parser("draft", help="Generate manuscript draft")
|
|
1600
|
+
sp.add_argument("study_id")
|
|
1601
|
+
sp.set_defaults(func=cmd_draft)
|
|
1602
|
+
|
|
1603
|
+
# bundle
|
|
1604
|
+
sp = sub.add_parser("bundle", help="Create a hash-verified portable study archive")
|
|
1605
|
+
sp.add_argument("study_id")
|
|
1606
|
+
sp.set_defaults(func=cmd_bundle)
|
|
1607
|
+
|
|
1608
|
+
# verify-bundle
|
|
1609
|
+
sp = sub.add_parser("verify-bundle", help="Verify a bundle archive's integrity")
|
|
1610
|
+
sp.add_argument("bundle_path", help="Path to the .tar.gz bundle file")
|
|
1611
|
+
sp.set_defaults(func=cmd_verify_bundle)
|
|
1612
|
+
|
|
1613
|
+
# plot-km
|
|
1614
|
+
sp = sub.add_parser("plot-km", help="Generate a Kaplan-Meier survival curve plot")
|
|
1615
|
+
sp.add_argument("study_id")
|
|
1616
|
+
sp.add_argument("test_id", type=int, help="ID of the completed kaplan_meier_logrank analysis result")
|
|
1617
|
+
sp.add_argument("--format", choices=["svg", "pdf"], default="svg",
|
|
1618
|
+
help="Output format (default: svg)")
|
|
1619
|
+
sp.add_argument("--no-risk-table", action="store_true",
|
|
1620
|
+
help="Hide the at-risk table subplot")
|
|
1621
|
+
sp.add_argument("--no-medians", action="store_true",
|
|
1622
|
+
help="Hide median survival reference lines and callouts")
|
|
1623
|
+
sp.add_argument("--output", type=str,
|
|
1624
|
+
help="Custom output file path (overrides default naming)")
|
|
1625
|
+
sp.add_argument("--time-unit", choices=["days", "months"], default="months",
|
|
1626
|
+
help="Display unit for the x-axis (default: months)")
|
|
1627
|
+
sp.add_argument("--style", choices=["clean", "scientific", "presentation", "all"], default="clean",
|
|
1628
|
+
help="Visual preset for the plot (default: clean; use 'all' to generate all three)")
|
|
1629
|
+
sp.set_defaults(func=cmd_plot_km)
|
|
1630
|
+
|
|
1631
|
+
# export
|
|
1632
|
+
sp = sub.add_parser("export", help="Export study as JSON (portable reviewer format)")
|
|
1633
|
+
sp.add_argument("study_id")
|
|
1634
|
+
sp.add_argument("--mode", default="supplementary", choices=["supplementary", "internal_full"])
|
|
1635
|
+
sp.add_argument("--version", dest="study_plan_version", help="Plan version label (e.g. v1)")
|
|
1636
|
+
sp.add_argument("--format", choices=["json", "appendix"], default="json",
|
|
1637
|
+
help="Export JSON, or JSON plus a manuscript appendix Markdown file")
|
|
1638
|
+
sp.add_argument("--acknowledge-violations", action="store_true",
|
|
1639
|
+
help="Acknowledge post-unmask diagnostic violations and proceed with export")
|
|
1640
|
+
sp.set_defaults(func=cmd_export)
|
|
1641
|
+
|
|
1642
|
+
# export-excel
|
|
1643
|
+
sp = sub.add_parser("export-excel",
|
|
1644
|
+
help="Generate a publication-ready Excel report with KM plot, Table 1, and audit")
|
|
1645
|
+
sp.add_argument("study_id")
|
|
1646
|
+
sp.add_argument("--output", type=str, help="Custom output file path")
|
|
1647
|
+
sp.set_defaults(func=cmd_export_excel)
|
|
1648
|
+
|
|
1649
|
+
# lineage
|
|
1650
|
+
sp = sub.add_parser("lineage", help="Render study provenance DAG")
|
|
1651
|
+
sp.add_argument("study_id")
|
|
1652
|
+
sp.add_argument("--svg", type=str, default=None,
|
|
1653
|
+
help="Output path for SVG file (omit for ASCII terminal output)")
|
|
1654
|
+
sp.set_defaults(func=cmd_lineage)
|
|
1655
|
+
|
|
1656
|
+
# forensics
|
|
1657
|
+
sp = sub.add_parser("forensics",
|
|
1658
|
+
help="Run data anomaly-detection checks")
|
|
1659
|
+
sp.add_argument("study_id")
|
|
1660
|
+
sp.set_defaults(func=cmd_forensics)
|
|
1661
|
+
|
|
1662
|
+
# plot-forest
|
|
1663
|
+
sp = sub.add_parser("plot-forest",
|
|
1664
|
+
help="Render forest plot for Cox PH results")
|
|
1665
|
+
sp.add_argument("study_id")
|
|
1666
|
+
sp.add_argument("extra_args", nargs="*", help=argparse.SUPPRESS)
|
|
1667
|
+
sp.add_argument("--svg", dest="output", type=str, default=None,
|
|
1668
|
+
help="Output SVG path (default: data/studies/<id>/forest_plot.svg)")
|
|
1669
|
+
sp.add_argument("--ascii", action="store_true",
|
|
1670
|
+
help="Render as ASCII text instead of SVG")
|
|
1671
|
+
sp.set_defaults(func=cmd_forest_plot)
|
|
1672
|
+
|
|
1673
|
+
# flowchart
|
|
1674
|
+
sp = sub.add_parser("flowchart",
|
|
1675
|
+
help="Render CONSORT/STROBE patient flow diagram")
|
|
1676
|
+
sp.add_argument("study_id")
|
|
1677
|
+
sp.add_argument("extra_args", nargs="*", help=argparse.SUPPRESS)
|
|
1678
|
+
sp.add_argument("--svg", dest="output", type=str, default=None,
|
|
1679
|
+
help="Output SVG path (default: data/studies/<id>/flowchart.svg)")
|
|
1680
|
+
sp.add_argument("--ascii", action="store_true",
|
|
1681
|
+
help="Render as ASCII text instead of SVG")
|
|
1682
|
+
sp.add_argument("--no-title", action="store_true",
|
|
1683
|
+
help="Omit title from SVG")
|
|
1684
|
+
sp.add_argument("--watermark", action="store_true",
|
|
1685
|
+
help="Show 'Generated by' footer in SVG")
|
|
1686
|
+
sp.add_argument("--verbose", action="store_true",
|
|
1687
|
+
help="Show developer annotations in SVG")
|
|
1688
|
+
sp.add_argument("--show-study-name", action="store_true",
|
|
1689
|
+
help="Show dataset name subtitle below main title")
|
|
1690
|
+
sp.set_defaults(func=cmd_flowchart)
|
|
1691
|
+
|
|
1692
|
+
return p
|
|
1693
|
+
|
|
1694
|
+
|
|
1695
|
+
def main() -> None:
|
|
1696
|
+
parser = build_parser()
|
|
1697
|
+
args = parser.parse_args()
|
|
1698
|
+
if not args.command:
|
|
1699
|
+
parser.print_help()
|
|
1700
|
+
sys.exit(1)
|
|
1701
|
+
args.func(args)
|
|
1702
|
+
|
|
1703
|
+
|
|
1704
|
+
if __name__ == "__main__":
|
|
1705
|
+
main()
|