research-tool-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/backend/_bootstrap.py +15 -0
- app/backend/exports/verification_script.py +19 -0
- app/backend/main.py +59 -0
- app/backend/routers/__init__.py +1 -0
- app/backend/routers/execution.py +478 -0
- app/backend/routers/ingestion.py +233 -0
- app/backend/routers/planning.py +265 -0
- app/backend/routers/reporting.py +531 -0
- app/backend/state.py +44 -0
- core/__init__.py +0 -0
- core/cli/__init__.py +0 -0
- core/cli/main.py +1705 -0
- core/database.py +62 -0
- core/ingestion/__init__.py +0 -0
- core/ingestion/csv_loader.py +191 -0
- core/ingestion/variable_classifier.py +171 -0
- core/masking/__init__.py +0 -0
- core/masking/gate.py +128 -0
- core/models.py +138 -0
- core/planning/__init__.py +0 -0
- core/planning/diagnostics.py +89 -0
- core/planning/lock.py +232 -0
- core/planning/study_plan.py +73 -0
- core/planning/test_selector.py +518 -0
- core/provenance/__init__.py +0 -0
- core/provenance/hashing.py +38 -0
- core/provenance/tracker.py +105 -0
- core/reporting/__init__.py +62 -0
- core/reporting/appendix.py +58 -0
- core/reporting/bundle.py +378 -0
- core/reporting/excel_export.py +683 -0
- core/reporting/flowchart/__init__.py +20 -0
- core/reporting/flowchart/flowchart.py +511 -0
- core/reporting/forensics.py +592 -0
- core/reporting/forest_plot.py +614 -0
- core/reporting/lineage.py +562 -0
- core/reporting/manuscript_draft.py +726 -0
- core/reporting/plots.py +568 -0
- core/reporting/strobe_checklist.py +460 -0
- core/stats/__init__.py +0 -0
- core/stats/descriptive.py +104 -0
- core/stats/inferential.py +540 -0
- core/stats/multiple_comparisons.py +62 -0
- core/stats/post_hoc.py +62 -0
- exports/verification_script.py +19 -0
- research_tool_cli-0.1.0.dist-info/METADATA +16 -0
- research_tool_cli-0.1.0.dist-info/RECORD +93 -0
- research_tool_cli-0.1.0.dist-info/WHEEL +5 -0
- research_tool_cli-0.1.0.dist-info/entry_points.txt +2 -0
- research_tool_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- research_tool_cli-0.1.0.dist-info/top_level.txt +4 -0
- tests/__init__.py +0 -0
- tests/conftest.py +45 -0
- tests/test_amendments.py +296 -0
- tests/test_analyze_batch_robustness.py +162 -0
- tests/test_app_statistical_reporting.py +383 -0
- tests/test_benchmark_21.py +556 -0
- tests/test_bundle.py +277 -0
- tests/test_cox_ph_plan.py +498 -0
- tests/test_csv_loader.py +368 -0
- tests/test_end_to_end.py +302 -0
- tests/test_excel_export.py +164 -0
- tests/test_flowchart.py +244 -0
- tests/test_forensics.py +305 -0
- tests/test_forest_plot.py +374 -0
- tests/test_from_json.py +176 -0
- tests/test_latest_plan_version.py +164 -0
- tests/test_lineage.py +329 -0
- tests/test_lock_immutability.py +133 -0
- tests/test_m1_fk_violation.py +85 -0
- tests/test_m2_data_hash_consistency.py +40 -0
- tests/test_m3_correction_timing.py +59 -0
- tests/test_m4_dedup_field.py +59 -0
- tests/test_m5_excel_hash_scope.py +45 -0
- tests/test_m6_fisher_exact_naming.py +44 -0
- tests/test_m7_duplicate_study_plan.py +55 -0
- tests/test_m8_filter_superseded.py +58 -0
- tests/test_m9_table1_groupby.py +56 -0
- tests/test_manuscript_draft.py +289 -0
- tests/test_masking_gate.py +196 -0
- tests/test_masking_migration.py +111 -0
- tests/test_multiple_comparisons.py +56 -0
- tests/test_plan_validation.py +289 -0
- tests/test_plots.py +394 -0
- tests/test_post_hoc_tagging.py +49 -0
- tests/test_posthoc_analyze.py +203 -0
- tests/test_provenance_tracker.py +110 -0
- tests/test_roadmap_features.py +148 -0
- tests/test_stats_descriptive.py +57 -0
- tests/test_stats_inferential.py +380 -0
- tests/test_strobe_checklist.py +172 -0
- tests/test_test_selector.py +350 -0
- tests/test_variable_classifier.py +124 -0
|
@@ -0,0 +1,683 @@
|
|
|
1
|
+
"""Publication-ready Excel report builder.
|
|
2
|
+
|
|
3
|
+
Consumes only already-computed data (locked plan, analysis results, Table 1
|
|
4
|
+
DataFrame, STROBE report, KM plot). Never computes or alters any statistic.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import tempfile
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
from openpyxl import Workbook
|
|
17
|
+
from openpyxl.drawing.image import Image as XlImage
|
|
18
|
+
from openpyxl.styles import (
|
|
19
|
+
Alignment, Border, Font, PatternFill, Side, numbers,
|
|
20
|
+
)
|
|
21
|
+
from openpyxl.utils import get_column_letter
|
|
22
|
+
|
|
23
|
+
from core.reporting import format_label as _format_label
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
from core.database import get_connection, DATA_ROOT
|
|
27
|
+
|
|
28
|
+
# ── Style constants ────────────────────────────────────────────────────
|
|
29
|
+
|
|
30
|
+
NAVY_FILL = PatternFill(start_color="1F4E78", end_color="1F4E78", fill_type="solid")
|
|
31
|
+
WHITE_FONT = Font(name="Calibri", bold=True, color="FFFFFF", size=11)
|
|
32
|
+
TITLE_FONT = Font(name="Calibri", bold=True, size=16, color="1F4E78")
|
|
33
|
+
HEADER_FONT = Font(name="Calibri", bold=True, size=11, color="1F4E78")
|
|
34
|
+
BODY_FONT = Font(name="Calibri", size=11)
|
|
35
|
+
MONO_FONT = Font(name="Consolas", size=10)
|
|
36
|
+
ZEBRA_FILL = PatternFill(start_color="F2F4F7", end_color="F2F4F7", fill_type="solid")
|
|
37
|
+
THIN_BORDER = Border(
|
|
38
|
+
left=Side(style="thin", color="D0D0D0"),
|
|
39
|
+
right=Side(style="thin", color="D0D0D0"),
|
|
40
|
+
top=Side(style="thin", color="D0D0D0"),
|
|
41
|
+
bottom=Side(style="thin", color="D0D0D0"),
|
|
42
|
+
)
|
|
43
|
+
SUBTLE_HEADER_FILL = PatternFill(start_color="E8EBF0", end_color="E8EBF0", fill_type="solid")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _style_header_row(ws, row: int, max_col: int):
|
|
47
|
+
"""Apply navy header styling to a row."""
|
|
48
|
+
for col in range(1, max_col + 1):
|
|
49
|
+
cell = ws.cell(row=row, column=col)
|
|
50
|
+
cell.font = WHITE_FONT
|
|
51
|
+
cell.fill = NAVY_FILL
|
|
52
|
+
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
|
|
53
|
+
cell.border = THIN_BORDER
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _apply_zebra(ws, start_row: int, end_row: int, max_col: int):
|
|
57
|
+
"""Apply alternating row shading."""
|
|
58
|
+
for r in range(start_row, end_row + 1):
|
|
59
|
+
if (r - start_row) % 2 == 1:
|
|
60
|
+
for c in range(1, max_col + 1):
|
|
61
|
+
ws.cell(row=r, column=c).fill = ZEBRA_FILL
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _autofit_columns(ws, min_width: int = 10, max_width: int = 55):
|
|
65
|
+
"""Auto-fit column widths — only for columns that have data."""
|
|
66
|
+
# Find the max column with actual data
|
|
67
|
+
max_col = 0
|
|
68
|
+
for row in ws.iter_rows():
|
|
69
|
+
for cell in row:
|
|
70
|
+
if cell.value is not None and cell.column > max_col:
|
|
71
|
+
max_col = cell.column
|
|
72
|
+
if max_col == 0:
|
|
73
|
+
return
|
|
74
|
+
for col_idx in range(1, max_col + 1):
|
|
75
|
+
col_letter = get_column_letter(col_idx)
|
|
76
|
+
lengths = []
|
|
77
|
+
for row in ws.iter_rows(min_col=col_idx, max_col=col_idx):
|
|
78
|
+
for cell in row:
|
|
79
|
+
if cell.value is not None:
|
|
80
|
+
lengths.append(len(str(cell.value)))
|
|
81
|
+
if lengths:
|
|
82
|
+
best = min(max(max(lengths) + 2, min_width), max_width)
|
|
83
|
+
ws.column_dimensions[col_letter].width = best
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def generate_excel_report(
|
|
87
|
+
study_id: str,
|
|
88
|
+
output_path: Optional[str | Path] = None,
|
|
89
|
+
) -> Path:
|
|
90
|
+
"""Generate a publication-ready 4-tab Excel workbook for a completed study.
|
|
91
|
+
|
|
92
|
+
Parameters
|
|
93
|
+
----------
|
|
94
|
+
study_id : str
|
|
95
|
+
output_path : str or Path, optional
|
|
96
|
+
Defaults to ``data/studies/{study_id}/study_report.xlsx``.
|
|
97
|
+
|
|
98
|
+
Returns
|
|
99
|
+
-------
|
|
100
|
+
Path to the generated workbook.
|
|
101
|
+
|
|
102
|
+
Raises
|
|
103
|
+
------
|
|
104
|
+
RuntimeError
|
|
105
|
+
If no analysis results exist or no locked plan is found.
|
|
106
|
+
"""
|
|
107
|
+
conn = get_connection(study_id)
|
|
108
|
+
|
|
109
|
+
# ── Study metadata ──────────────────────────────────────────────
|
|
110
|
+
cur = conn.execute("SELECT * FROM studies WHERE id=?", (study_id,))
|
|
111
|
+
study = cur.fetchone()
|
|
112
|
+
if not study:
|
|
113
|
+
conn.close()
|
|
114
|
+
raise RuntimeError(f"Study '{study_id}' not found.")
|
|
115
|
+
|
|
116
|
+
# ── Locked plan ─────────────────────────────────────────────────
|
|
117
|
+
locked_paths = sorted(DATA_ROOT.glob(f"{study_id}/study_plan.v*.locked.json"))
|
|
118
|
+
if not locked_paths:
|
|
119
|
+
conn.close()
|
|
120
|
+
raise RuntimeError(f"No locked plan found for study '{study_id}'.")
|
|
121
|
+
latest_plan = json.loads(locked_paths[-1].read_text())
|
|
122
|
+
|
|
123
|
+
# ── Analysis results ────────────────────────────────────────────
|
|
124
|
+
cur = conn.execute(
|
|
125
|
+
"SELECT * FROM analysis_results WHERE study_id=? ORDER BY id",
|
|
126
|
+
(study_id,),
|
|
127
|
+
)
|
|
128
|
+
analyses = cur.fetchall()
|
|
129
|
+
if not analyses:
|
|
130
|
+
conn.close()
|
|
131
|
+
raise RuntimeError(f"No analysis results found for study '{study_id}'.")
|
|
132
|
+
|
|
133
|
+
conn.close()
|
|
134
|
+
|
|
135
|
+
# ── STROBE report ───────────────────────────────────────────────
|
|
136
|
+
from core.reporting.strobe_checklist import generate_report
|
|
137
|
+
strobe_text = generate_report(study_id)
|
|
138
|
+
strobe_satisfied = f"{strobe_text.count('✓')}/{strobe_text.count('Item')}"
|
|
139
|
+
|
|
140
|
+
# ── Build workbook ──────────────────────────────────────────────
|
|
141
|
+
wb = Workbook()
|
|
142
|
+
ws_exec = wb.active
|
|
143
|
+
temp_files: list[Path] = []
|
|
144
|
+
|
|
145
|
+
_build_tab1_summary(ws_exec, study_id, study, latest_plan, analyses, strobe_satisfied, temp_files)
|
|
146
|
+
_build_tab2_table1(wb, study_id)
|
|
147
|
+
_build_tab3_analyses(wb, study_id, analyses)
|
|
148
|
+
_build_tab4_audit(wb, study_id, latest_plan)
|
|
149
|
+
|
|
150
|
+
# Auto-fit and polish
|
|
151
|
+
for ws in wb.worksheets:
|
|
152
|
+
ws.sheet_properties.showGridLines = False
|
|
153
|
+
_autofit_columns(ws)
|
|
154
|
+
|
|
155
|
+
if output_path is None:
|
|
156
|
+
output_path = DATA_ROOT / study_id / "study_report.xlsx"
|
|
157
|
+
else:
|
|
158
|
+
output_path = Path(output_path)
|
|
159
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
160
|
+
|
|
161
|
+
wb.save(str(output_path))
|
|
162
|
+
|
|
163
|
+
# Clean up temp files
|
|
164
|
+
for tf in temp_files:
|
|
165
|
+
tf.unlink(missing_ok=True)
|
|
166
|
+
|
|
167
|
+
return output_path
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ── Tab 1: Executive Summary ────────────────────────────────────────────
|
|
171
|
+
|
|
172
|
+
def _build_tab1_summary(ws, study_id, study, plan_data, analyses, strobe_satisfied, temp_files: list[Path] = None):
|
|
173
|
+
ws.title = "Executive Summary"
|
|
174
|
+
ws.sheet_properties.tabColor = "1F4E78"
|
|
175
|
+
|
|
176
|
+
# Title
|
|
177
|
+
ws.cell(row=1, column=1, value="STUDY SUMMARY REPORT").font = TITLE_FONT
|
|
178
|
+
ws.merge_cells("A1:D1")
|
|
179
|
+
|
|
180
|
+
# Metadata block
|
|
181
|
+
meta = [
|
|
182
|
+
("Study ID", study_id),
|
|
183
|
+
("Study Title", study["name"] or "N/A"),
|
|
184
|
+
("Design Type", (study["study_type"] or "cohort").replace("_", " ")),
|
|
185
|
+
("Locked Plan", f"v{plan_data.get('version', '?')} (locked: {plan_data.get('locked_at', '?')[:10]})"),
|
|
186
|
+
("STROBE Compliance", f"{strobe_satisfied} items satisfied"),
|
|
187
|
+
("Created", (study["created_at"] or "")[:10]),
|
|
188
|
+
]
|
|
189
|
+
for i, (label, value) in enumerate(meta):
|
|
190
|
+
ws.cell(row=3 + i, column=1, value=label).font = HEADER_FONT
|
|
191
|
+
ws.cell(row=3 + i, column=2, value=value).font = BODY_FONT
|
|
192
|
+
|
|
193
|
+
# ── Primary analysis table ──────────────────────────────────────────
|
|
194
|
+
r = 11
|
|
195
|
+
ws.cell(row=r, column=1, value="PRIMARY OUTCOME").font = HEADER_FONT
|
|
196
|
+
r += 1
|
|
197
|
+
|
|
198
|
+
tbl_headers = ["Test Name", "Statistic", "P-Value", "Pre-Registered", "Result", "Rationale"]
|
|
199
|
+
for ci, h in enumerate(tbl_headers, 1):
|
|
200
|
+
ws.cell(row=r, column=ci, value=h)
|
|
201
|
+
_style_header_row(ws, r, len(tbl_headers))
|
|
202
|
+
r += 1
|
|
203
|
+
|
|
204
|
+
pre_reg = [a for a in analyses if a["is_pre_registered"]]
|
|
205
|
+
post_hoc = [a for a in analyses if not a["is_pre_registered"]]
|
|
206
|
+
|
|
207
|
+
# Pre-registered rows
|
|
208
|
+
pre_start = r
|
|
209
|
+
for a in analyses:
|
|
210
|
+
if not a["is_pre_registered"]:
|
|
211
|
+
continue
|
|
212
|
+
test_name = a["test_name"]
|
|
213
|
+
stat = f"{a['statistic']:.4f}" if a["statistic"] is not None else "N/A"
|
|
214
|
+
pv = f"{a['p_value']:.4f}" if a["p_value"] is not None else "N/A"
|
|
215
|
+
pre = "YES"
|
|
216
|
+
status_data = json.loads(a["status_json"]) if a["status_json"] else {}
|
|
217
|
+
status = status_data.get("status", "completed")
|
|
218
|
+
from core.planning.diagnostics import check_violation
|
|
219
|
+
has_viol, viol_summary, _ = check_violation(dict(a))
|
|
220
|
+
sig = "Significant" if (a["p_value"] is not None and a["p_value"] < 0.05) else "Not Significant"
|
|
221
|
+
if has_viol:
|
|
222
|
+
sig = f"Significant — ⚠ assumption violation"
|
|
223
|
+
vals = [test_name, stat, pv, pre, sig, "Pre-registered primary protocol"]
|
|
224
|
+
for ci, v in enumerate(vals, 1):
|
|
225
|
+
ws.cell(row=r, column=ci, value=v).font = BODY_FONT
|
|
226
|
+
ws.cell(row=r, column=ci).border = THIN_BORDER
|
|
227
|
+
r += 1
|
|
228
|
+
|
|
229
|
+
_apply_zebra(ws, pre_start, r - 1, len(tbl_headers))
|
|
230
|
+
|
|
231
|
+
# Post-hoc section
|
|
232
|
+
if post_hoc:
|
|
233
|
+
r += 1
|
|
234
|
+
ws.cell(row=r, column=1, value="EXPLORATORY / POST-HOC ANALYSES").font = HEADER_FONT
|
|
235
|
+
r += 1
|
|
236
|
+
|
|
237
|
+
ph_headers = ["Test Name", "Statistic", "P-Value", "Result", "Rationale", "Reason"]
|
|
238
|
+
for ci, h in enumerate(ph_headers, 1):
|
|
239
|
+
ws.cell(row=r, column=ci, value=h)
|
|
240
|
+
_style_header_row(ws, r, len(ph_headers))
|
|
241
|
+
r += 1
|
|
242
|
+
|
|
243
|
+
ph_start = r
|
|
244
|
+
for a in post_hoc:
|
|
245
|
+
test_name = a["test_name"]
|
|
246
|
+
stat = f"{a['statistic']:.4f}" if a["statistic"] is not None else "N/A"
|
|
247
|
+
pv = f"{a['p_value']:.4f}" if a["p_value"] is not None else "N/A"
|
|
248
|
+
sig = "Significant" if (a["p_value"] is not None and a["p_value"] < 0.05) else "Not Significant"
|
|
249
|
+
prov = {}
|
|
250
|
+
if a["provenance_json"]:
|
|
251
|
+
try:
|
|
252
|
+
prov = json.loads(a["provenance_json"])
|
|
253
|
+
except (TypeError, json.JSONDecodeError):
|
|
254
|
+
pass
|
|
255
|
+
rationale = prov.get("rationale", "")
|
|
256
|
+
reason = prov.get("amendment_reason", "")
|
|
257
|
+
label = f"{test_name} — {rationale}" if rationale else test_name
|
|
258
|
+
vals = [label, stat, pv, sig, rationale, reason]
|
|
259
|
+
for ci, v in enumerate(vals, 1):
|
|
260
|
+
ws.cell(row=r, column=ci, value=v).font = BODY_FONT
|
|
261
|
+
ws.cell(row=r, column=ci).border = THIN_BORDER
|
|
262
|
+
r += 1
|
|
263
|
+
|
|
264
|
+
_apply_zebra(ws, ph_start, r - 1, len(ph_headers))
|
|
265
|
+
|
|
266
|
+
# Embed KM plot as PNG
|
|
267
|
+
r += 2
|
|
268
|
+
try:
|
|
269
|
+
from core.reporting.plots import generate_km_plot
|
|
270
|
+
km_result = next(
|
|
271
|
+
(a for a in analyses
|
|
272
|
+
if a["test_name"] in ("kaplan_meier_logrank", "kaplan_meier")
|
|
273
|
+
and json.loads(a["status_json"] or "{}").get("status") in ("completed", "assumption_violation")),
|
|
274
|
+
None,
|
|
275
|
+
)
|
|
276
|
+
if km_result:
|
|
277
|
+
test_id = km_result["id"]
|
|
278
|
+
tmp_png = Path(tempfile.mkstemp(suffix=".png")[1])
|
|
279
|
+
generate_km_plot(study_id, test_id=test_id, output_path=tmp_png, fmt="png", style="clean")
|
|
280
|
+
if temp_files is not None:
|
|
281
|
+
temp_files.append(tmp_png)
|
|
282
|
+
img = XlImage(str(tmp_png))
|
|
283
|
+
# Preserve aspect ratio: scale to fit 600px width
|
|
284
|
+
from PIL import Image as PILImage
|
|
285
|
+
with PILImage.open(str(tmp_png)) as pil_img:
|
|
286
|
+
orig_w, orig_h = pil_img.size
|
|
287
|
+
target_w = 600
|
|
288
|
+
img.width = target_w
|
|
289
|
+
img.height = int(orig_h * target_w / orig_w)
|
|
290
|
+
ws.add_image(img, f"A{r}")
|
|
291
|
+
r += 26
|
|
292
|
+
except Exception as e:
|
|
293
|
+
import sys
|
|
294
|
+
print(f"WARNING: KM image embedding failed: {e}", file=sys.stderr)
|
|
295
|
+
|
|
296
|
+
# Embed Forest plot as PNG
|
|
297
|
+
try:
|
|
298
|
+
from core.reporting.forest_plot import generate_forest_plot_png
|
|
299
|
+
cox_result = next(
|
|
300
|
+
(a for a in analyses
|
|
301
|
+
if a["test_name"] in ("cox_ph_model", "cox_proportional_hazards")
|
|
302
|
+
and json.loads(a["status_json"] or "{}").get("status") in ("completed", "assumption_violation")),
|
|
303
|
+
None,
|
|
304
|
+
)
|
|
305
|
+
if cox_result:
|
|
306
|
+
tmp_forest_png = Path(tempfile.mkstemp(suffix=".png")[1])
|
|
307
|
+
generate_forest_plot_png(study_id, output_path=tmp_forest_png)
|
|
308
|
+
if temp_files is not None:
|
|
309
|
+
temp_files.append(tmp_forest_png)
|
|
310
|
+
img_f = XlImage(str(tmp_forest_png))
|
|
311
|
+
with PILImage.open(str(tmp_forest_png)) as pil_img:
|
|
312
|
+
orig_w, orig_h = pil_img.size
|
|
313
|
+
target_w = 600
|
|
314
|
+
img_f.width = target_w
|
|
315
|
+
img_f.height = int(orig_h * target_w / orig_w)
|
|
316
|
+
ws.add_image(img_f, f"A{r}")
|
|
317
|
+
r += 26
|
|
318
|
+
except Exception as e:
|
|
319
|
+
import sys
|
|
320
|
+
print(f"WARNING: Forest plot image embedding failed: {e}", file=sys.stderr)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
# ── Tab 2: Table 1 — Baseline Characteristics ───────────────────────────
|
|
324
|
+
|
|
325
|
+
VALUE_MAPPINGS = {
|
|
326
|
+
"sex": {"f": "Female", "m": "Male"},
|
|
327
|
+
"high_risk_cytogenetics": {"yes": "Yes", "no": "No"},
|
|
328
|
+
"iss_stage": {"I": "Stage I", "II": "Stage II", "III": "Stage III"},
|
|
329
|
+
"treatment_arm": {"A": "Test Arm (A)", "B": "Placebo Arm (B)"},
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _build_tab2_table1(wb, study_id):
|
|
334
|
+
ws = wb.create_sheet(title="Table 1 - Baseline")
|
|
335
|
+
ws.sheet_properties.tabColor = "1F4E78"
|
|
336
|
+
ws.sheet_view.showGridLines = False
|
|
337
|
+
|
|
338
|
+
from core.stats.descriptive import generate_table1
|
|
339
|
+
from core.database import get_connection
|
|
340
|
+
|
|
341
|
+
groupby = "treatment_arm"
|
|
342
|
+
tbl = generate_table1(study_id, groupby=groupby)
|
|
343
|
+
|
|
344
|
+
if tbl.empty:
|
|
345
|
+
ws.cell(row=1, column=1, value="Table 1 not yet computed.").font = BODY_FONT
|
|
346
|
+
return
|
|
347
|
+
|
|
348
|
+
# Flatten columns — drop the "Missing" column
|
|
349
|
+
cols = list(tbl.columns)
|
|
350
|
+
if hasattr(tbl.columns, "levels") or (cols and isinstance(cols[0], tuple)):
|
|
351
|
+
cols = [str(c[-1]).strip() if isinstance(c, tuple) else str(c) for c in cols]
|
|
352
|
+
if cols and cols[0].lower() in ("missing", ""):
|
|
353
|
+
cols = cols[1:]
|
|
354
|
+
tbl = tbl.iloc[:, 1:]
|
|
355
|
+
|
|
356
|
+
# Get sample sizes for header labels
|
|
357
|
+
conn = get_connection(study_id)
|
|
358
|
+
raw = f"raw_{study_id}"
|
|
359
|
+
total_n = conn.execute(f"SELECT COUNT(*) as n FROM {raw}").fetchone()["n"]
|
|
360
|
+
arm_counts = {}
|
|
361
|
+
if groupby:
|
|
362
|
+
col_names = [r["name"] for r in conn.execute(f"PRAGMA table_info({raw})")]
|
|
363
|
+
if groupby in col_names:
|
|
364
|
+
cur = conn.execute(f'SELECT "{groupby}" as arm, COUNT(*) as n FROM {raw} GROUP BY arm')
|
|
365
|
+
arm_counts = {r["arm"]: r["n"] for r in cur.fetchall()}
|
|
366
|
+
conn.close()
|
|
367
|
+
|
|
368
|
+
# Build header row with N counts
|
|
369
|
+
header_parts = ["Characteristic", f"Overall (N={total_n})"]
|
|
370
|
+
for c in cols[1:]:
|
|
371
|
+
n = arm_counts.get(c, "")
|
|
372
|
+
header_parts.append(f"{c} (N={n})" if n else c)
|
|
373
|
+
header_row = header_parts
|
|
374
|
+
ncols = len(header_row)
|
|
375
|
+
|
|
376
|
+
for ci, h in enumerate(header_row, 1):
|
|
377
|
+
ws.cell(row=1, column=ci, value=h)
|
|
378
|
+
_style_header_row(ws, 1, ncols)
|
|
379
|
+
|
|
380
|
+
# Process the original MultiIndex
|
|
381
|
+
orig_idx = list(tbl.index) if hasattr(tbl.index, "levels") else []
|
|
382
|
+
if not orig_idx:
|
|
383
|
+
return
|
|
384
|
+
|
|
385
|
+
data_row = 2
|
|
386
|
+
prev_var_raw = ""
|
|
387
|
+
for ri in range(len(tbl)):
|
|
388
|
+
entry = orig_idx[ri]
|
|
389
|
+
raw_label = str(entry[0]).strip() if isinstance(entry, tuple) else str(entry).strip()
|
|
390
|
+
cat_level = str(entry[1]).strip() if isinstance(entry, tuple) and entry[1] else ""
|
|
391
|
+
|
|
392
|
+
if raw_label.lower() == "n":
|
|
393
|
+
continue
|
|
394
|
+
|
|
395
|
+
is_continuous = not cat_level
|
|
396
|
+
var_base = raw_label.split(",")[0].strip() if "," in raw_label else raw_label
|
|
397
|
+
|
|
398
|
+
if var_base.lower().replace(" ", "_") == "treatment_arm":
|
|
399
|
+
continue
|
|
400
|
+
|
|
401
|
+
if not is_continuous:
|
|
402
|
+
var_key = var_base.lower().replace(" ", "_").replace("-", "_")
|
|
403
|
+
if var_key != prev_var_raw:
|
|
404
|
+
var_parts = raw_label.split(",", 1)
|
|
405
|
+
var_name = _format_label(var_parts[0]).strip()
|
|
406
|
+
suffix = "," + var_parts[1] if len(var_parts) > 1 else ""
|
|
407
|
+
group_name = var_name + suffix
|
|
408
|
+
cell = ws.cell(row=data_row, column=1, value=group_name)
|
|
409
|
+
cell.font = Font(name="Calibri", bold=True, size=11)
|
|
410
|
+
data_row += 1
|
|
411
|
+
prev_var_raw = var_key
|
|
412
|
+
|
|
413
|
+
mapping = VALUE_MAPPINGS.get(var_key, {})
|
|
414
|
+
clean_cat = mapping.get(cat_level, mapping.get(cat_level.lower(), cat_level))
|
|
415
|
+
cell = ws.cell(row=data_row, column=1, value=f" {clean_cat}")
|
|
416
|
+
cell.font = BODY_FONT
|
|
417
|
+
else:
|
|
418
|
+
prev_var_raw = ""
|
|
419
|
+
var_parts = raw_label.split(",", 1)
|
|
420
|
+
var_name = _format_label(var_parts[0]).strip()
|
|
421
|
+
suffix = "," + var_parts[1] if len(var_parts) > 1 else ""
|
|
422
|
+
clean_label = var_name + suffix
|
|
423
|
+
cell = ws.cell(row=data_row, column=1, value=clean_label)
|
|
424
|
+
cell.font = BODY_FONT
|
|
425
|
+
|
|
426
|
+
for ci in range(ncols - 1):
|
|
427
|
+
if ci < len(tbl.columns):
|
|
428
|
+
raw_val = tbl.iloc[ri, ci]
|
|
429
|
+
val_str = str(raw_val) if not pd.isna(raw_val) else ""
|
|
430
|
+
ws.cell(row=data_row, column=ci + 2, value=val_str).font = BODY_FONT
|
|
431
|
+
|
|
432
|
+
for ci in range(1, ncols + 1):
|
|
433
|
+
ws.cell(row=data_row, column=ci).border = THIN_BORDER
|
|
434
|
+
|
|
435
|
+
data_row += 1
|
|
436
|
+
|
|
437
|
+
_apply_zebra(ws, 2, data_row - 1, ncols)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
# ── Tab 3: Statistical Analyses ─────────────────────────────────────────
|
|
441
|
+
|
|
442
|
+
def _build_tab3_analyses(wb, study_id, analyses):
|
|
443
|
+
import json
|
|
444
|
+
ws = wb.create_sheet(title="Statistical Analyses")
|
|
445
|
+
ws.sheet_properties.tabColor = "1F4E78"
|
|
446
|
+
|
|
447
|
+
locked_paths = sorted(DATA_ROOT.glob(f"{study_id}/study_plan.v*.locked.json"))
|
|
448
|
+
plan_data = None
|
|
449
|
+
if locked_paths:
|
|
450
|
+
plan_data = json.loads(locked_paths[-1].read_text())
|
|
451
|
+
|
|
452
|
+
primary_comparison = ""
|
|
453
|
+
var_map = {}
|
|
454
|
+
if plan_data:
|
|
455
|
+
primary_comparison = plan_data.get("primary_comparison", "")
|
|
456
|
+
# Build variable ID → column_name lookup
|
|
457
|
+
conn_tmp = get_connection(study_id)
|
|
458
|
+
var_id_to_name = {}
|
|
459
|
+
for row in conn_tmp.execute(
|
|
460
|
+
"SELECT id, column_name FROM variables WHERE study_id=?", (study_id,)
|
|
461
|
+
).fetchall():
|
|
462
|
+
var_id_to_name[str(row["id"])] = row["column_name"]
|
|
463
|
+
conn_tmp.close()
|
|
464
|
+
|
|
465
|
+
for t in plan_data.get("planned_tests", []):
|
|
466
|
+
vn = t.get("variable_name", "")
|
|
467
|
+
# Resolve variable ID to column name if needed
|
|
468
|
+
if vn in var_id_to_name:
|
|
469
|
+
vn = var_id_to_name[vn]
|
|
470
|
+
var_map[t.get("test_name", "")] = vn
|
|
471
|
+
for t in plan_data.get("post_hoc_tests", []):
|
|
472
|
+
vn = t.get("variable_name", "")
|
|
473
|
+
if vn in var_id_to_name:
|
|
474
|
+
vn = var_id_to_name[vn]
|
|
475
|
+
var_map[t.get("test_name", "")] = vn
|
|
476
|
+
for m in plan_data.get("cox_ph_models", []):
|
|
477
|
+
st = m.get("survival_time_col", "")
|
|
478
|
+
if str(st) in var_id_to_name:
|
|
479
|
+
st = var_id_to_name[str(st)]
|
|
480
|
+
if st:
|
|
481
|
+
var_map["cox_ph_model"] = st
|
|
482
|
+
var_map["cox_proportional_hazards"] = st
|
|
483
|
+
if "cox_ph_model" not in var_map:
|
|
484
|
+
outcomes = plan_data.get("primary_outcome_variable_ids", [])
|
|
485
|
+
if outcomes:
|
|
486
|
+
o0 = outcomes[0]
|
|
487
|
+
if str(o0) in var_id_to_name:
|
|
488
|
+
o0 = var_id_to_name[str(o0)]
|
|
489
|
+
var_map["cox_ph_model"] = o0
|
|
490
|
+
var_map["cox_proportional_hazards"] = o0
|
|
491
|
+
|
|
492
|
+
# Pre-fetch covariate results
|
|
493
|
+
conn = get_connection(study_id)
|
|
494
|
+
covariate_map: dict[int, list[dict]] = {}
|
|
495
|
+
result_ids = [a["id"] for a in analyses]
|
|
496
|
+
if result_ids:
|
|
497
|
+
placeholders = ",".join("?" for _ in result_ids)
|
|
498
|
+
cur = conn.execute(
|
|
499
|
+
f"SELECT * FROM analysis_covariate_results WHERE result_id IN ({placeholders}) ORDER BY id",
|
|
500
|
+
result_ids,
|
|
501
|
+
)
|
|
502
|
+
for row in cur.fetchall():
|
|
503
|
+
covariate_map.setdefault(row["result_id"], []).append({
|
|
504
|
+
"covariate": row["covariate"],
|
|
505
|
+
"hr": row["hr"],
|
|
506
|
+
"ci_lower": row["ci_lower"],
|
|
507
|
+
"ci_upper": row["ci_upper"],
|
|
508
|
+
"wald_p": row["wald_p"],
|
|
509
|
+
})
|
|
510
|
+
conn.close()
|
|
511
|
+
|
|
512
|
+
headers = [
|
|
513
|
+
"Analysis ID", "Test Name", "Comparison", "Variable",
|
|
514
|
+
"N Analyzed", "Statistic", "P-Value", "Adj. P-Value",
|
|
515
|
+
"95% CI (Lower)", "95% CI (Upper)",
|
|
516
|
+
"LR Test P", "Concordance",
|
|
517
|
+
"Status", "Pre-Registered",
|
|
518
|
+
]
|
|
519
|
+
for ci, h in enumerate(headers, 1):
|
|
520
|
+
ws.cell(row=1, column=ci, value=h)
|
|
521
|
+
_style_header_row(ws, 1, len(headers))
|
|
522
|
+
|
|
523
|
+
r = 2
|
|
524
|
+
footnote_idx = 0
|
|
525
|
+
footnotes: list[str] = []
|
|
526
|
+
for a in analyses:
|
|
527
|
+
status_data = json.loads(a["status_json"]) if a["status_json"] else {}
|
|
528
|
+
status = status_data.get("status", "completed")
|
|
529
|
+
from core.planning.diagnostics import check_violation
|
|
530
|
+
has_viol, viol_summary, _ = check_violation(dict(a))
|
|
531
|
+
display_status = status
|
|
532
|
+
if has_viol:
|
|
533
|
+
footnote_idx += 1
|
|
534
|
+
marker = chr(0x2460 + footnote_idx - 1) # circled digits: ① ② ...
|
|
535
|
+
display_status = f"{status} {marker}"
|
|
536
|
+
footnotes.append(f"{marker} {viol_summary}")
|
|
537
|
+
sc = json.loads(a["sample_counts_json"]) if a["sample_counts_json"] else {}
|
|
538
|
+
n = sc.get("n_analyzed", "")
|
|
539
|
+
test_name = a["test_name"]
|
|
540
|
+
comparison = primary_comparison
|
|
541
|
+
variable = var_map.get(test_name, "")
|
|
542
|
+
if variable and variable != "pfs_days" and variable != "os_days":
|
|
543
|
+
v_clean = _format_label(variable)
|
|
544
|
+
comparison = f"{v_clean} by Treatment Arm"
|
|
545
|
+
# For cox_ph_model, CI applies to individual covariates, not the
|
|
546
|
+
# overall omnibus model (which uses LR test). Show "N/A" for main row.
|
|
547
|
+
if test_name == "cox_ph_model":
|
|
548
|
+
ci_lower_str = "N/A"
|
|
549
|
+
ci_upper_str = "N/A"
|
|
550
|
+
else:
|
|
551
|
+
ci_lower_str = f"{a['ci_lower']:.4f}" if a["ci_lower"] is not None else "N/A"
|
|
552
|
+
ci_upper_str = f"{a['ci_upper']:.4f}" if a["ci_upper"] is not None else "N/A"
|
|
553
|
+
lr_p_str = ""
|
|
554
|
+
c_index_str = ""
|
|
555
|
+
try:
|
|
556
|
+
if a["lr_test_p"] is not None:
|
|
557
|
+
lr_p_str = f"{a['lr_test_p']:.4f}"
|
|
558
|
+
except (KeyError, IndexError, TypeError):
|
|
559
|
+
pass
|
|
560
|
+
try:
|
|
561
|
+
if a["concordance_index"] is not None:
|
|
562
|
+
c_index_str = f"{a['concordance_index']:.3f}"
|
|
563
|
+
except (KeyError, IndexError, TypeError):
|
|
564
|
+
pass
|
|
565
|
+
vals = [
|
|
566
|
+
a["id"],
|
|
567
|
+
test_name,
|
|
568
|
+
comparison,
|
|
569
|
+
variable,
|
|
570
|
+
n,
|
|
571
|
+
f"{a['statistic']:.4f}" if a["statistic"] is not None else "",
|
|
572
|
+
f"{a['p_value']:.4f}" if a["p_value"] is not None else "",
|
|
573
|
+
f"{a['adjusted_p_value']:.4f}" if a["adjusted_p_value"] is not None else "",
|
|
574
|
+
ci_lower_str,
|
|
575
|
+
ci_upper_str,
|
|
576
|
+
lr_p_str,
|
|
577
|
+
c_index_str,
|
|
578
|
+
display_status,
|
|
579
|
+
"YES" if a["is_pre_registered"] else "NO [POST-HOC]",
|
|
580
|
+
]
|
|
581
|
+
for ci, v in enumerate(vals, 1):
|
|
582
|
+
ws.cell(row=r, column=ci, value=v).font = BODY_FONT
|
|
583
|
+
ws.cell(row=r, column=ci).border = THIN_BORDER
|
|
584
|
+
r += 1
|
|
585
|
+
|
|
586
|
+
# Per-covariate sub-rows for Cox PH — aligned to parent columns:
|
|
587
|
+
# Col 2: Covariate (label), Col 6: HR (under Statistic),
|
|
588
|
+
# Col 7: P-Value, Col 9: CI Lower, Col 10: CI Upper
|
|
589
|
+
cov_rows = covariate_map.get(a["id"], [])
|
|
590
|
+
if cov_rows:
|
|
591
|
+
cov_start = r
|
|
592
|
+
cov_font = Font(name="Calibri", italic=True, size=10, color="555555")
|
|
593
|
+
ws.cell(row=r, column=4, value="Covariate").font = cov_font
|
|
594
|
+
ws.cell(row=r, column=6, value="HR").font = cov_font
|
|
595
|
+
ws.cell(row=r, column=7, value="P-Value").font = cov_font
|
|
596
|
+
ws.cell(row=r, column=9, value="CI Lower").font = cov_font
|
|
597
|
+
ws.cell(row=r, column=10, value="CI Upper").font = cov_font
|
|
598
|
+
r += 1
|
|
599
|
+
for cr in cov_rows:
|
|
600
|
+
ws.cell(row=r, column=4, value=cr["covariate"]).font = BODY_FONT
|
|
601
|
+
ws.cell(row=r, column=6, value=f"{cr['hr']:.4f}" if cr.get("hr") is not None else "").font = BODY_FONT
|
|
602
|
+
ws.cell(row=r, column=7, value=f"{cr['wald_p']:.4f}" if cr.get("wald_p") is not None else "").font = BODY_FONT
|
|
603
|
+
ws.cell(row=r, column=9, value=f"{cr['ci_lower']:.4f}" if cr.get("ci_lower") is not None else "").font = BODY_FONT
|
|
604
|
+
ws.cell(row=r, column=10, value=f"{cr['ci_upper']:.4f}" if cr.get("ci_upper") is not None else "").font = BODY_FONT
|
|
605
|
+
r += 1
|
|
606
|
+
|
|
607
|
+
_apply_zebra(ws, 2, r - 1, len(headers))
|
|
608
|
+
|
|
609
|
+
# Footnotes for assumption violations
|
|
610
|
+
for note in footnotes:
|
|
611
|
+
ws.cell(row=r, column=1, value=note).font = Font(name="Calibri", italic=True, size=9, color="C0392B")
|
|
612
|
+
r += 1
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
# ── Tab 4: Audit & Hash Manifest ────────────────────────────────────────
|
|
616
|
+
|
|
617
|
+
def _build_tab4_audit(wb, study_id, plan_data):
|
|
618
|
+
ws = wb.create_sheet(title="Audit & Hash Manifest")
|
|
619
|
+
ws.sheet_properties.tabColor = "1F4E78"
|
|
620
|
+
|
|
621
|
+
ws.cell(row=1, column=1, value="FILE").font = WHITE_FONT
|
|
622
|
+
ws.cell(row=1, column=1).fill = NAVY_FILL
|
|
623
|
+
ws.cell(row=1, column=1).border = THIN_BORDER
|
|
624
|
+
ws.cell(row=1, column=2, value="SHA-256 HASH").font = WHITE_FONT
|
|
625
|
+
ws.cell(row=1, column=2).fill = NAVY_FILL
|
|
626
|
+
ws.cell(row=1, column=2).border = THIN_BORDER
|
|
627
|
+
|
|
628
|
+
from core.provenance.hashing import sha256 as _sha256, canonical_json as _canonical_json, compute_raw_data_hash
|
|
629
|
+
|
|
630
|
+
conn = get_connection(study_id)
|
|
631
|
+
raw_data_hash = compute_raw_data_hash(study_id)
|
|
632
|
+
|
|
633
|
+
locked_plan_hash = plan_data.get("content_hash", "N/A")
|
|
634
|
+
|
|
635
|
+
cur = conn.execute(
|
|
636
|
+
"SELECT * FROM analysis_results WHERE study_id=? ORDER BY id", (study_id,)
|
|
637
|
+
)
|
|
638
|
+
results = []
|
|
639
|
+
for r in cur.fetchall():
|
|
640
|
+
row = dict(r)
|
|
641
|
+
for jf in ("variable_ids_used", "effect_size_json",
|
|
642
|
+
"sample_counts_json", "status_json", "provenance_json",
|
|
643
|
+
"ph_diagnostics_json"):
|
|
644
|
+
if row.get(jf):
|
|
645
|
+
try:
|
|
646
|
+
row[jf] = json.loads(row[jf])
|
|
647
|
+
except (TypeError, json.JSONDecodeError):
|
|
648
|
+
pass
|
|
649
|
+
results.append(row)
|
|
650
|
+
conn.close()
|
|
651
|
+
results_json = _canonical_json(results)
|
|
652
|
+
results_hash = _sha256(results_json)
|
|
653
|
+
|
|
654
|
+
# Check if bundle exists and read composite hash from manifest
|
|
655
|
+
composite_hash_str = "Run 'bundle' command to generate"
|
|
656
|
+
bundle_path = DATA_ROOT / study_id / f"{study_id}_bundle.tar.gz"
|
|
657
|
+
if bundle_path.exists():
|
|
658
|
+
try:
|
|
659
|
+
import tarfile
|
|
660
|
+
with tarfile.open(bundle_path, "r:gz") as tf:
|
|
661
|
+
manifest_file = tf.extractfile("manifest.json")
|
|
662
|
+
if manifest_file:
|
|
663
|
+
manifest = json.loads(manifest_file.read().decode("utf-8"))
|
|
664
|
+
composite_hash_str = manifest.get("composite_hash", composite_hash_str)
|
|
665
|
+
except Exception:
|
|
666
|
+
pass
|
|
667
|
+
|
|
668
|
+
rows = [
|
|
669
|
+
("Study Plan (locked)", locked_plan_hash),
|
|
670
|
+
("Raw Data (ingested)", raw_data_hash),
|
|
671
|
+
("Analysis Results", results_hash),
|
|
672
|
+
("Composite Bundle Hash", composite_hash_str),
|
|
673
|
+
]
|
|
674
|
+
|
|
675
|
+
for ri, (label, hash_val) in enumerate(rows):
|
|
676
|
+
r = ri + 2
|
|
677
|
+
ws.cell(row=r, column=1, value=label).font = MONO_FONT
|
|
678
|
+
ws.cell(row=r, column=1).border = THIN_BORDER
|
|
679
|
+
ws.cell(row=r, column=2, value=hash_val).font = MONO_FONT
|
|
680
|
+
ws.cell(row=r, column=2).border = THIN_BORDER
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
import pandas as pd
|