research-tool-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/backend/_bootstrap.py +15 -0
- app/backend/exports/verification_script.py +19 -0
- app/backend/main.py +59 -0
- app/backend/routers/__init__.py +1 -0
- app/backend/routers/execution.py +478 -0
- app/backend/routers/ingestion.py +233 -0
- app/backend/routers/planning.py +265 -0
- app/backend/routers/reporting.py +531 -0
- app/backend/state.py +44 -0
- core/__init__.py +0 -0
- core/cli/__init__.py +0 -0
- core/cli/main.py +1705 -0
- core/database.py +62 -0
- core/ingestion/__init__.py +0 -0
- core/ingestion/csv_loader.py +191 -0
- core/ingestion/variable_classifier.py +171 -0
- core/masking/__init__.py +0 -0
- core/masking/gate.py +128 -0
- core/models.py +138 -0
- core/planning/__init__.py +0 -0
- core/planning/diagnostics.py +89 -0
- core/planning/lock.py +232 -0
- core/planning/study_plan.py +73 -0
- core/planning/test_selector.py +518 -0
- core/provenance/__init__.py +0 -0
- core/provenance/hashing.py +38 -0
- core/provenance/tracker.py +105 -0
- core/reporting/__init__.py +62 -0
- core/reporting/appendix.py +58 -0
- core/reporting/bundle.py +378 -0
- core/reporting/excel_export.py +683 -0
- core/reporting/flowchart/__init__.py +20 -0
- core/reporting/flowchart/flowchart.py +511 -0
- core/reporting/forensics.py +592 -0
- core/reporting/forest_plot.py +614 -0
- core/reporting/lineage.py +562 -0
- core/reporting/manuscript_draft.py +726 -0
- core/reporting/plots.py +568 -0
- core/reporting/strobe_checklist.py +460 -0
- core/stats/__init__.py +0 -0
- core/stats/descriptive.py +104 -0
- core/stats/inferential.py +540 -0
- core/stats/multiple_comparisons.py +62 -0
- core/stats/post_hoc.py +62 -0
- exports/verification_script.py +19 -0
- research_tool_cli-0.1.0.dist-info/METADATA +16 -0
- research_tool_cli-0.1.0.dist-info/RECORD +93 -0
- research_tool_cli-0.1.0.dist-info/WHEEL +5 -0
- research_tool_cli-0.1.0.dist-info/entry_points.txt +2 -0
- research_tool_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- research_tool_cli-0.1.0.dist-info/top_level.txt +4 -0
- tests/__init__.py +0 -0
- tests/conftest.py +45 -0
- tests/test_amendments.py +296 -0
- tests/test_analyze_batch_robustness.py +162 -0
- tests/test_app_statistical_reporting.py +383 -0
- tests/test_benchmark_21.py +556 -0
- tests/test_bundle.py +277 -0
- tests/test_cox_ph_plan.py +498 -0
- tests/test_csv_loader.py +368 -0
- tests/test_end_to_end.py +302 -0
- tests/test_excel_export.py +164 -0
- tests/test_flowchart.py +244 -0
- tests/test_forensics.py +305 -0
- tests/test_forest_plot.py +374 -0
- tests/test_from_json.py +176 -0
- tests/test_latest_plan_version.py +164 -0
- tests/test_lineage.py +329 -0
- tests/test_lock_immutability.py +133 -0
- tests/test_m1_fk_violation.py +85 -0
- tests/test_m2_data_hash_consistency.py +40 -0
- tests/test_m3_correction_timing.py +59 -0
- tests/test_m4_dedup_field.py +59 -0
- tests/test_m5_excel_hash_scope.py +45 -0
- tests/test_m6_fisher_exact_naming.py +44 -0
- tests/test_m7_duplicate_study_plan.py +55 -0
- tests/test_m8_filter_superseded.py +58 -0
- tests/test_m9_table1_groupby.py +56 -0
- tests/test_manuscript_draft.py +289 -0
- tests/test_masking_gate.py +196 -0
- tests/test_masking_migration.py +111 -0
- tests/test_multiple_comparisons.py +56 -0
- tests/test_plan_validation.py +289 -0
- tests/test_plots.py +394 -0
- tests/test_post_hoc_tagging.py +49 -0
- tests/test_posthoc_analyze.py +203 -0
- tests/test_provenance_tracker.py +110 -0
- tests/test_roadmap_features.py +148 -0
- tests/test_stats_descriptive.py +57 -0
- tests/test_stats_inferential.py +380 -0
- tests/test_strobe_checklist.py +172 -0
- tests/test_test_selector.py +350 -0
- tests/test_variable_classifier.py +124 -0
core/database.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""SQLite database connection and initialization.
|
|
2
|
+
|
|
3
|
+
Uses stdlib sqlite3 — no ORM. Every study gets its own .db file under
|
|
4
|
+
data/studies/<study_id>/study.db.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import sqlite3
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import sqlite3
|
|
12
|
+
|
|
13
|
+
from core.models import SCHEMA_SQL, MIGRATIONS_SQL
|
|
14
|
+
|
|
15
|
+
DATA_ROOT = Path("data/studies")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def study_dir(study_id: str) -> Path:
|
|
19
|
+
return DATA_ROOT / study_id
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_connection(study_id: str) -> sqlite3.Connection:
|
|
23
|
+
"""Return a connection to the study's database, creating dirs if needed."""
|
|
24
|
+
d = study_dir(study_id)
|
|
25
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
26
|
+
db_path = d / "study.db"
|
|
27
|
+
conn = sqlite3.connect(str(db_path))
|
|
28
|
+
conn.row_factory = sqlite3.Row
|
|
29
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
30
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
31
|
+
migrate_db(conn)
|
|
32
|
+
return conn
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def init_db(conn: sqlite3.Connection) -> None:
|
|
36
|
+
"""Run CREATE TABLE IF NOT EXISTS for all tables."""
|
|
37
|
+
conn.executescript(SCHEMA_SQL)
|
|
38
|
+
conn.commit()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def migrate_db(conn: sqlite3.Connection) -> None:
|
|
42
|
+
"""Apply migrations for schema additions (new columns, new tables).
|
|
43
|
+
|
|
44
|
+
Safe to call on any version — catches "duplicate column" errors.
|
|
45
|
+
Only runs if the target table already exists (skip on fresh DB before init_db).
|
|
46
|
+
"""
|
|
47
|
+
existing = conn.execute(
|
|
48
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='analysis_results'"
|
|
49
|
+
).fetchone()
|
|
50
|
+
if not existing:
|
|
51
|
+
return
|
|
52
|
+
for stmt in MIGRATIONS_SQL.split(";"):
|
|
53
|
+
stmt = stmt.strip()
|
|
54
|
+
if not stmt:
|
|
55
|
+
continue
|
|
56
|
+
try:
|
|
57
|
+
conn.execute(stmt)
|
|
58
|
+
except sqlite3.OperationalError as e:
|
|
59
|
+
if "duplicate column name" in str(e).lower():
|
|
60
|
+
continue
|
|
61
|
+
raise
|
|
62
|
+
conn.commit()
|
|
File without changes
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Load CSV/Excel into a dynamic per-study raw data table."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from core.database import DATA_ROOT, get_connection, init_db
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _sanitize_col(name: str) -> str:
|
|
13
|
+
"""Make a column name SQL-safe."""
|
|
14
|
+
return "".join(c if c.isalnum() or c == "_" else "_" for c in name).strip("_") or "col"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _check_reingest(study_id: str) -> None:
|
|
18
|
+
"""Check if study already has ingested data. Raises SystemExit if so."""
|
|
19
|
+
conn = get_connection(study_id)
|
|
20
|
+
init_db(conn)
|
|
21
|
+
|
|
22
|
+
# Check if raw_ table exists and has rows
|
|
23
|
+
raw = f"raw_{study_id}"
|
|
24
|
+
cur = conn.execute(
|
|
25
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
|
26
|
+
(raw,),
|
|
27
|
+
)
|
|
28
|
+
if cur.fetchone() is None:
|
|
29
|
+
conn.close()
|
|
30
|
+
return # no raw table → clean
|
|
31
|
+
|
|
32
|
+
cur = conn.execute(f"SELECT COUNT(*) AS cnt FROM {raw}")
|
|
33
|
+
row_count = cur.fetchone()["cnt"]
|
|
34
|
+
|
|
35
|
+
cur = conn.execute(
|
|
36
|
+
"SELECT COUNT(*) AS cnt FROM variables WHERE study_id=?",
|
|
37
|
+
(study_id,),
|
|
38
|
+
)
|
|
39
|
+
var_count = cur.fetchone()["cnt"]
|
|
40
|
+
|
|
41
|
+
# Check for locked plan files on disk
|
|
42
|
+
locked_plans = list(DATA_ROOT.glob(f"{study_id}/study_plan.v*.locked.json"))
|
|
43
|
+
conn.close()
|
|
44
|
+
|
|
45
|
+
if row_count > 0 or var_count > 0 or locked_plans:
|
|
46
|
+
parts = []
|
|
47
|
+
if row_count > 0:
|
|
48
|
+
parts.append(f"{row_count} row(s) in raw table")
|
|
49
|
+
if var_count > 0:
|
|
50
|
+
parts.append(f"{var_count} classified variable(s)")
|
|
51
|
+
if locked_plans:
|
|
52
|
+
parts.append(f"{len(locked_plans)} locked plan(s)")
|
|
53
|
+
msg = (
|
|
54
|
+
f"Study '{study_id}' already has ingested data ({'; '.join(parts)}). "
|
|
55
|
+
f"Re-ingesting would invalidate existing variables/plan/results. "
|
|
56
|
+
f"Use 'new-study' to start fresh, or pass --force-reingest if "
|
|
57
|
+
f"this is intentional (clears all downstream state)."
|
|
58
|
+
)
|
|
59
|
+
print(msg, file=sys.stderr)
|
|
60
|
+
sys.exit(1)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _cascade_clear(study_id: str) -> None:
|
|
64
|
+
"""Drop downstream state so re-ingest can proceed cleanly."""
|
|
65
|
+
conn = get_connection(study_id)
|
|
66
|
+
init_db(conn)
|
|
67
|
+
|
|
68
|
+
masked = f"raw_masked_{study_id}"
|
|
69
|
+
conn.execute(f"DROP TABLE IF EXISTS {masked}")
|
|
70
|
+
# M1 fix: delete child table before parent to avoid FK violation
|
|
71
|
+
conn.execute(
|
|
72
|
+
"DELETE FROM analysis_covariate_results WHERE result_id IN "
|
|
73
|
+
"(SELECT id FROM analysis_results WHERE study_id=?)",
|
|
74
|
+
(study_id,),
|
|
75
|
+
)
|
|
76
|
+
conn.execute("DELETE FROM analysis_results WHERE study_id=?", (study_id,))
|
|
77
|
+
conn.execute("DELETE FROM variables WHERE study_id=?", (study_id,))
|
|
78
|
+
|
|
79
|
+
locked_plans = list(DATA_ROOT.glob(f"{study_id}/study_plan.v*.locked.json"))
|
|
80
|
+
for p in locked_plans:
|
|
81
|
+
p.unlink()
|
|
82
|
+
|
|
83
|
+
conn.commit()
|
|
84
|
+
conn.close()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def load_file(study_id: str, filepath: str, na_values: list[str] | None = None,
|
|
88
|
+
force: bool = False) -> list[str]:
|
|
89
|
+
"""Load a CSV (or Excel) file into the study's database.
|
|
90
|
+
|
|
91
|
+
Creates a dynamic table named `raw_<study_id>` with one column per CSV column
|
|
92
|
+
(all TEXT). Returns the list of column names found.
|
|
93
|
+
|
|
94
|
+
Parameters
|
|
95
|
+
----------
|
|
96
|
+
study_id : str
|
|
97
|
+
filepath : str
|
|
98
|
+
Path to the CSV or Excel file.
|
|
99
|
+
na_values : list[str], optional
|
|
100
|
+
Additional strings to treat as NA/NaN during CSV parsing
|
|
101
|
+
(e.g. ``["unknown", "missing"]``). Added on top of pandas' defaults.
|
|
102
|
+
force : bool
|
|
103
|
+
If True, clear downstream state before re-ingesting.
|
|
104
|
+
"""
|
|
105
|
+
if not force:
|
|
106
|
+
_check_reingest(study_id)
|
|
107
|
+
|
|
108
|
+
path = Path(filepath)
|
|
109
|
+
if path.suffix in (".xls", ".xlsx"):
|
|
110
|
+
df = pd.read_excel(str(path))
|
|
111
|
+
else:
|
|
112
|
+
kwargs = {"keep_default_na": True}
|
|
113
|
+
if na_values:
|
|
114
|
+
kwargs["na_values"] = na_values
|
|
115
|
+
df = pd.read_csv(str(path), **kwargs)
|
|
116
|
+
|
|
117
|
+
conn = get_connection(study_id)
|
|
118
|
+
init_db(conn)
|
|
119
|
+
|
|
120
|
+
if force:
|
|
121
|
+
_cascade_clear(study_id)
|
|
122
|
+
|
|
123
|
+
# Strip leading/trailing whitespace from string columns
|
|
124
|
+
str_cols = df.select_dtypes(include="object").columns
|
|
125
|
+
for c in str_cols:
|
|
126
|
+
df[c] = df[c].apply(lambda v: v.strip() if isinstance(v, str) else v)
|
|
127
|
+
# Also convert whitespace-only strings to None so they register as missing
|
|
128
|
+
df = df.replace(r"^\s*$", pd.NA, regex=True)
|
|
129
|
+
|
|
130
|
+
# ── Duplicate patient_id detection ─────────────────────────────────
|
|
131
|
+
pid_raw = next((c for c in df.columns if c.lower() in ("patient_id", "patientid", "id", "subject_id")), None)
|
|
132
|
+
if pid_raw:
|
|
133
|
+
pid_col = _sanitize_col(pid_raw)
|
|
134
|
+
dupe_counts = df[pid_raw].value_counts()
|
|
135
|
+
dupes = dupe_counts[dupe_counts > 1]
|
|
136
|
+
if not dupes.empty:
|
|
137
|
+
dupe_list = ", ".join(f"'{pid}' ({n}x)" for pid, n in dupes.items())
|
|
138
|
+
print(f"Warning: duplicate patient identifiers found — {dupe_list}. "
|
|
139
|
+
f"These rows will be included in the analysis; "
|
|
140
|
+
f"verify they are genuine repeat records.",
|
|
141
|
+
file=__import__("sys").stderr)
|
|
142
|
+
|
|
143
|
+
col_names = [_sanitize_col(c) for c in df.columns]
|
|
144
|
+
col_defs = ", ".join(f'"{c}" TEXT' for c in col_names)
|
|
145
|
+
table_name = f"raw_{study_id}"
|
|
146
|
+
|
|
147
|
+
conn.execute(f"DROP TABLE IF EXISTS {table_name}")
|
|
148
|
+
conn.execute(f'CREATE TABLE {table_name} ("row_id" INTEGER PRIMARY KEY, {col_defs})')
|
|
149
|
+
|
|
150
|
+
placeholders = ", ".join("?" for _ in col_names)
|
|
151
|
+
col_list = ", ".join(f'"{c}"' for c in col_names)
|
|
152
|
+
|
|
153
|
+
for _, row in df.iterrows():
|
|
154
|
+
vals = tuple(None if pd.isna(v) else str(v) for v in row)
|
|
155
|
+
conn.execute(f"INSERT INTO {table_name} ({col_list}) VALUES ({placeholders})", vals)
|
|
156
|
+
|
|
157
|
+
conn.commit()
|
|
158
|
+
conn.close()
|
|
159
|
+
return col_names
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def find_duplicate_patient_ids(study_id: str) -> list[tuple[str, int]]:
|
|
163
|
+
"""Query the study's raw table for duplicate patient identifiers.
|
|
164
|
+
|
|
165
|
+
Returns a list of ``(patient_id, count)`` tuples for every duplicated
|
|
166
|
+
patient ID, ordered by count descending. Returns an empty list when
|
|
167
|
+
there is no patient_id column or no duplicates exist.
|
|
168
|
+
"""
|
|
169
|
+
from core.database import get_connection
|
|
170
|
+
|
|
171
|
+
conn = get_connection(study_id)
|
|
172
|
+
raw_table = f"raw_{study_id}"
|
|
173
|
+
|
|
174
|
+
# Probe column names for a patient-like column
|
|
175
|
+
cur = conn.execute(f"PRAGMA table_info({raw_table})")
|
|
176
|
+
col_names = [r["name"] for r in cur.fetchall()]
|
|
177
|
+
pid_col = next(
|
|
178
|
+
(c for c in col_names if c.lower() in ("patient_id", "patientid", "id", "subject_id")),
|
|
179
|
+
None,
|
|
180
|
+
)
|
|
181
|
+
if not pid_col:
|
|
182
|
+
conn.close()
|
|
183
|
+
return []
|
|
184
|
+
|
|
185
|
+
cur = conn.execute(
|
|
186
|
+
f'SELECT "{pid_col}" AS pid, COUNT(*) AS cnt FROM {raw_table} '
|
|
187
|
+
f'GROUP BY "{pid_col}" HAVING cnt > 1 ORDER BY cnt DESC'
|
|
188
|
+
)
|
|
189
|
+
dupes = [(r["pid"], r["cnt"]) for r in cur.fetchall()]
|
|
190
|
+
conn.close()
|
|
191
|
+
return dupes
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Help the user classify variables as baseline/outcome and data types."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from core.database import get_connection
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_OUTCOME_KEYWORDS = frozenset({
|
|
9
|
+
"outcome", "response", "survival", "pfs", "os", "status", "event",
|
|
10
|
+
"death", "followup", "lab", "marker", "protein", "score", "level",
|
|
11
|
+
"change", "fold", "day",
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _infer_data_type(study_id: str, column: str) -> str | None:
|
|
16
|
+
"""Infer data type by inspecting actual column values.
|
|
17
|
+
|
|
18
|
+
Returns ``"continuous"``, ``"categorical"``, or ``None`` if the raw table
|
|
19
|
+
cannot be queried (heuristic-only fallback).
|
|
20
|
+
"""
|
|
21
|
+
conn = get_connection(study_id)
|
|
22
|
+
try:
|
|
23
|
+
raw_table = f"raw_{study_id}"
|
|
24
|
+
cur = conn.execute(
|
|
25
|
+
f'SELECT DISTINCT "{column}" FROM {raw_table} WHERE "{column}" IS NOT NULL LIMIT 100'
|
|
26
|
+
)
|
|
27
|
+
values = [row[0] for row in cur.fetchall()]
|
|
28
|
+
if not values:
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
# Attempt numeric parse
|
|
32
|
+
numeric = []
|
|
33
|
+
non_numeric = []
|
|
34
|
+
for v in values:
|
|
35
|
+
try:
|
|
36
|
+
float(v)
|
|
37
|
+
numeric.append(v)
|
|
38
|
+
except (ValueError, TypeError):
|
|
39
|
+
non_numeric.append(v)
|
|
40
|
+
|
|
41
|
+
# If any values are non-numeric strings, it's categorical
|
|
42
|
+
if non_numeric:
|
|
43
|
+
return "categorical"
|
|
44
|
+
|
|
45
|
+
# If all values are numeric, check distinct count
|
|
46
|
+
n_distinct = len(numeric)
|
|
47
|
+
if n_distinct > 5:
|
|
48
|
+
return "continuous"
|
|
49
|
+
return "categorical"
|
|
50
|
+
except Exception:
|
|
51
|
+
return None
|
|
52
|
+
finally:
|
|
53
|
+
conn.close()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _has_low_cardinality_text_values(study_id: str, column: str) -> bool | None:
|
|
57
|
+
"""Check if a column's values are non-numeric text with few distinct values.
|
|
58
|
+
|
|
59
|
+
Returns True for columns like ``high_risk_cytogenetics`` (yes/no),
|
|
60
|
+
False for columns with >5 distinct non-numeric values,
|
|
61
|
+
None when the table can't be queried.
|
|
62
|
+
"""
|
|
63
|
+
conn = get_connection(study_id)
|
|
64
|
+
try:
|
|
65
|
+
raw_table = f"raw_{study_id}"
|
|
66
|
+
cur = conn.execute(
|
|
67
|
+
f'SELECT DISTINCT "{column}" FROM {raw_table} WHERE "{column}" IS NOT NULL LIMIT 100'
|
|
68
|
+
)
|
|
69
|
+
values = [row[0] for row in cur.fetchall()]
|
|
70
|
+
if not values:
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
# Check if any value is non-numeric
|
|
74
|
+
numeric_count = 0
|
|
75
|
+
for v in values:
|
|
76
|
+
try:
|
|
77
|
+
float(v)
|
|
78
|
+
numeric_count += 1
|
|
79
|
+
except (ValueError, TypeError):
|
|
80
|
+
pass
|
|
81
|
+
|
|
82
|
+
# If values are numeric, no opinion
|
|
83
|
+
if numeric_count == len(values):
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
# Text values with ≤5 distinct options → likely baseline/demographic
|
|
87
|
+
return len(values) <= 5
|
|
88
|
+
except Exception:
|
|
89
|
+
return None
|
|
90
|
+
finally:
|
|
91
|
+
conn.close()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def classify_variables_interactive(study_id: str, columns: list[str]) -> list[dict]:
|
|
95
|
+
"""CLI-guided variable classification.
|
|
96
|
+
|
|
97
|
+
Uses heuristic name patterns AND actual data inspection.
|
|
98
|
+
Unrecognized column names get role="unclassified" to force explicit
|
|
99
|
+
resolution before planning.
|
|
100
|
+
"""
|
|
101
|
+
results = []
|
|
102
|
+
for col in columns:
|
|
103
|
+
col_lower = col.lower()
|
|
104
|
+
tokens = set(col_lower.replace("-", "_").split("_"))
|
|
105
|
+
|
|
106
|
+
# ── Role (baseline / outcome / unclassified) ────────────────────
|
|
107
|
+
if tokens & _OUTCOME_KEYWORDS or any(
|
|
108
|
+
kw in col_lower for kw in ("outcome", "response", "survival",
|
|
109
|
+
"pfs", "os", "status", "event", "death",
|
|
110
|
+
"followup", "lab", "protein", "marker",
|
|
111
|
+
"score", "level", "change", "fold")
|
|
112
|
+
):
|
|
113
|
+
default_role = "outcome"
|
|
114
|
+
elif tokens & {"id", "name", "patient", "identifier"}:
|
|
115
|
+
continue # skip identifier columns
|
|
116
|
+
elif tokens & {"age", "bmi", "weight", "height", "creatinine",
|
|
117
|
+
"count", "number", "value", "line", "lines", "prior",
|
|
118
|
+
"sex", "gender", "stage", "site", "location",
|
|
119
|
+
"center", "cohort", "group", "arm", "ecog"}:
|
|
120
|
+
default_role = "baseline"
|
|
121
|
+
else:
|
|
122
|
+
# If the column has low-cardinality text values (yes/no, mild/mod/severe),
|
|
123
|
+
# it's almost certainly baseline even if the name is unfamiliar.
|
|
124
|
+
is_low_card_text = _has_low_cardinality_text_values(study_id, col)
|
|
125
|
+
if is_low_card_text is True:
|
|
126
|
+
default_role = "baseline"
|
|
127
|
+
else:
|
|
128
|
+
# Unknown keyword pattern — flag for explicit user resolution
|
|
129
|
+
default_role = "unclassified"
|
|
130
|
+
|
|
131
|
+
# ── Data type (from actual values if possible) ──────────────────
|
|
132
|
+
# Start with a name-based guess
|
|
133
|
+
if any(kw in col_lower for kw in ("time", "days", "months", "duration", "follow_up", "fu")):
|
|
134
|
+
name_based_dtype = "time_to_event"
|
|
135
|
+
elif tokens & {"age", "bmi", "weight", "height", "value", "count",
|
|
136
|
+
"number", "creatinine", "line", "lines", "prior"}:
|
|
137
|
+
name_based_dtype = "continuous"
|
|
138
|
+
else:
|
|
139
|
+
name_based_dtype = "categorical"
|
|
140
|
+
|
|
141
|
+
# Use data inspection to *refine* — only upgrade categorical→continuous,
|
|
142
|
+
# never override a known name-based type like time_to_event.
|
|
143
|
+
inferred = _infer_data_type(study_id, col)
|
|
144
|
+
if name_based_dtype == "categorical" and inferred == "continuous":
|
|
145
|
+
default_dtype = "continuous"
|
|
146
|
+
elif name_based_dtype == "categorical" and inferred == "categorical":
|
|
147
|
+
default_dtype = "categorical"
|
|
148
|
+
else:
|
|
149
|
+
default_dtype = name_based_dtype
|
|
150
|
+
|
|
151
|
+
results.append({"column": col, "role": default_role, "data_type": default_dtype})
|
|
152
|
+
return results
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _classify_batch(study_id: str, variables: list[dict]) -> None:
|
|
156
|
+
"""Write classified variables to the database.
|
|
157
|
+
|
|
158
|
+
Each dict: {column, role, data_type}
|
|
159
|
+
"""
|
|
160
|
+
conn = get_connection(study_id)
|
|
161
|
+
for v in variables:
|
|
162
|
+
is_outcome = 1 if v["role"] == "outcome" else 0
|
|
163
|
+
conn.execute(
|
|
164
|
+
"""INSERT INTO variables (study_id, column_name, role, data_type, is_masked)
|
|
165
|
+
VALUES (?, ?, ?, ?, ?)
|
|
166
|
+
ON CONFLICT(study_id, column_name) DO UPDATE SET
|
|
167
|
+
role=excluded.role, data_type=excluded.data_type, is_masked=excluded.is_masked""",
|
|
168
|
+
(study_id, v["column"], v["role"], v["data_type"], is_outcome),
|
|
169
|
+
)
|
|
170
|
+
conn.commit()
|
|
171
|
+
conn.close()
|
core/masking/__init__.py
ADDED
|
File without changes
|
core/masking/gate.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""The Outcome-Masking Gate.
|
|
2
|
+
|
|
3
|
+
Enforces outcome masking at the physical storage level.
|
|
4
|
+
|
|
5
|
+
When variables are classified as outcomes, their values are moved from the
|
|
6
|
+
main raw_{study_id} table into a shadow raw_masked_{study_id} table and
|
|
7
|
+
replaced with NULLs. This catches EVERY access path — sqlite3 CLI, raw
|
|
8
|
+
Python sqlite3 module, any process that touches the database file. The only
|
|
9
|
+
way to recover the values is unmask(), which copies them back and transitions
|
|
10
|
+
the study to the unmasked state permanently.
|
|
11
|
+
|
|
12
|
+
State machine:
|
|
13
|
+
0 (pre-lock) → outcome columns are NULL in the main table
|
|
14
|
+
1 (locked) → outcome columns still NULL
|
|
15
|
+
2 (unmasked) → outcome values restored, irreversible
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
import sqlite3
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
from typing import Optional
|
|
22
|
+
from core.database import get_connection, init_db
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class MaskedDataError(Exception):
|
|
26
|
+
"""Raised when outcome data is accessed while the study is still masked."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _get_study_state(conn: sqlite3.Connection, study_id: str) -> int:
|
|
30
|
+
cur = conn.execute("SELECT is_locked FROM studies WHERE id=?", (study_id,))
|
|
31
|
+
row = cur.fetchone()
|
|
32
|
+
return row["is_locked"] if row else 0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def seal_outcomes(study_id: str) -> None:
|
|
36
|
+
"""Move outcome column values into the masked shadow table, NULL them in main.
|
|
37
|
+
|
|
38
|
+
Called AFTER variable classification. This is the physical enforcement:
|
|
39
|
+
outcome values simply do not exist in the main table until unmask.
|
|
40
|
+
"""
|
|
41
|
+
conn = get_connection(study_id)
|
|
42
|
+
init_db(conn)
|
|
43
|
+
|
|
44
|
+
# Get outcome column names
|
|
45
|
+
cur = conn.execute(
|
|
46
|
+
"SELECT column_name FROM variables WHERE study_id=? AND role='outcome'",
|
|
47
|
+
(study_id,),
|
|
48
|
+
)
|
|
49
|
+
outcome_cols = [row["column_name"] for row in cur.fetchall()]
|
|
50
|
+
if not outcome_cols:
|
|
51
|
+
conn.close()
|
|
52
|
+
return
|
|
53
|
+
|
|
54
|
+
raw = f"raw_{study_id}"
|
|
55
|
+
masked = f"raw_masked_{study_id}"
|
|
56
|
+
|
|
57
|
+
# Create the shadow table with same columns + row_id link
|
|
58
|
+
col_defs = ", ".join(f'"{c}" TEXT' for c in outcome_cols)
|
|
59
|
+
conn.execute(f"""
|
|
60
|
+
CREATE TABLE IF NOT EXISTS {masked} (
|
|
61
|
+
"row_id" INTEGER PRIMARY KEY REFERENCES {raw}(row_id),
|
|
62
|
+
{col_defs}
|
|
63
|
+
)
|
|
64
|
+
""")
|
|
65
|
+
|
|
66
|
+
# Copy outcome values from raw to masked
|
|
67
|
+
col_list = ", ".join(f'"{c}"' for c in outcome_cols)
|
|
68
|
+
conn.execute(f"DELETE FROM {masked}")
|
|
69
|
+
conn.execute(f"""
|
|
70
|
+
INSERT INTO {masked} (row_id, {col_list})
|
|
71
|
+
SELECT row_id, {col_list} FROM {raw}
|
|
72
|
+
""")
|
|
73
|
+
conn.commit()
|
|
74
|
+
|
|
75
|
+
# NULL out outcome columns in the main raw table
|
|
76
|
+
for col in outcome_cols:
|
|
77
|
+
conn.execute(f'UPDATE {raw} SET "{col}" = NULL')
|
|
78
|
+
|
|
79
|
+
conn.commit()
|
|
80
|
+
conn.close()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def unmask_study(study_id: str) -> None:
|
|
84
|
+
"""Copy outcome values back from shadow table, transition to unmasked.
|
|
85
|
+
|
|
86
|
+
Irreversible — once called, outcome columns become permanently visible.
|
|
87
|
+
"""
|
|
88
|
+
conn = get_connection(study_id)
|
|
89
|
+
init_db(conn)
|
|
90
|
+
|
|
91
|
+
cur = conn.execute(
|
|
92
|
+
"SELECT column_name FROM variables WHERE study_id=? AND role='outcome'",
|
|
93
|
+
(study_id,),
|
|
94
|
+
)
|
|
95
|
+
outcome_cols = [row["column_name"] for row in cur.fetchall()]
|
|
96
|
+
if not outcome_cols:
|
|
97
|
+
conn.close()
|
|
98
|
+
return
|
|
99
|
+
|
|
100
|
+
raw = f"raw_{study_id}"
|
|
101
|
+
masked = f"raw_masked_{study_id}"
|
|
102
|
+
|
|
103
|
+
for col in outcome_cols:
|
|
104
|
+
conn.execute(f"""
|
|
105
|
+
UPDATE {raw} SET "{col}" = (
|
|
106
|
+
SELECT "{col}" FROM {masked} WHERE {masked}.row_id = {raw}.row_id
|
|
107
|
+
)
|
|
108
|
+
""")
|
|
109
|
+
|
|
110
|
+
conn.execute("UPDATE studies SET is_locked=2, unmasked_at=? WHERE id=?", (datetime.now(timezone.utc).isoformat(), study_id))
|
|
111
|
+
conn.commit()
|
|
112
|
+
conn.close()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def lock_study(study_id: str) -> None:
|
|
116
|
+
"""Transition to locked state."""
|
|
117
|
+
conn = get_connection(study_id)
|
|
118
|
+
init_db(conn)
|
|
119
|
+
conn.execute("UPDATE studies SET is_locked=1 WHERE id=?", (study_id,))
|
|
120
|
+
conn.commit()
|
|
121
|
+
conn.close()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def is_masked(study_id: str) -> bool:
|
|
125
|
+
conn = get_connection(study_id)
|
|
126
|
+
state = _get_study_state(conn, study_id)
|
|
127
|
+
conn.close()
|
|
128
|
+
return state < 2
|
core/models.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Data model dataclasses and SQL schema for the research tool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
import json
|
|
5
|
+
from dataclasses import dataclass, field, asdict
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
# ── SQL schema ──────────────────────────────────────────────────────────────
|
|
10
|
+
|
|
11
|
+
SCHEMA_SQL = """
|
|
12
|
+
CREATE TABLE IF NOT EXISTS studies (
|
|
13
|
+
id TEXT PRIMARY KEY,
|
|
14
|
+
name TEXT NOT NULL,
|
|
15
|
+
created_at TEXT NOT NULL, -- ISO-8601
|
|
16
|
+
study_type TEXT, -- cohort | case_control | cross_sectional
|
|
17
|
+
is_locked INTEGER NOT NULL DEFAULT 0, -- 0 = pre-lock, 1 = locked, 2 = unmasked
|
|
18
|
+
unmasked_at TEXT, -- ISO-8601 timestamp of unmask event
|
|
19
|
+
data_dir TEXT NOT NULL,
|
|
20
|
+
unmask_audit_json TEXT -- JSON array of override events
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
CREATE TABLE IF NOT EXISTS variables (
|
|
24
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
25
|
+
study_id TEXT NOT NULL REFERENCES studies(id),
|
|
26
|
+
column_name TEXT NOT NULL,
|
|
27
|
+
role TEXT NOT NULL, -- baseline | outcome
|
|
28
|
+
data_type TEXT NOT NULL, -- categorical | continuous | time_to_event
|
|
29
|
+
is_masked INTEGER NOT NULL DEFAULT 0,
|
|
30
|
+
UNIQUE(study_id, column_name)
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
CREATE TABLE IF NOT EXISTS raw_data (
|
|
34
|
+
row_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
35
|
+
study_id TEXT NOT NULL REFERENCES studies(id),
|
|
36
|
+
json_row TEXT NOT NULL -- full row as JSON for flexibility
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
CREATE TABLE IF NOT EXISTS analysis_results (
|
|
40
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
41
|
+
study_id TEXT NOT NULL REFERENCES studies(id),
|
|
42
|
+
study_plan_version INTEGER,
|
|
43
|
+
variable_ids_used TEXT, -- JSON list
|
|
44
|
+
test_name TEXT,
|
|
45
|
+
statistic REAL,
|
|
46
|
+
p_value REAL,
|
|
47
|
+
adjusted_p_value REAL,
|
|
48
|
+
ci_lower REAL,
|
|
49
|
+
ci_upper REAL,
|
|
50
|
+
effect_size_json TEXT, -- JSON: {"metric": "Cramér's V", "value": 0.32}
|
|
51
|
+
sample_counts_json TEXT, -- JSON: {"n_total": 21, "n_analyzed": 21, "n_excluded": 0}
|
|
52
|
+
status_json TEXT, -- JSON: {"status": "completed", "reason": "..."}
|
|
53
|
+
is_pre_registered INTEGER NOT NULL DEFAULT 1,
|
|
54
|
+
provenance_json TEXT, -- JSON blob
|
|
55
|
+
computed_at TEXT NOT NULL,
|
|
56
|
+
superseded_previous_result_id INTEGER DEFAULT NULL,
|
|
57
|
+
lr_test_p REAL,
|
|
58
|
+
concordance_index REAL,
|
|
59
|
+
ph_diagnostics_json TEXT
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
CREATE TABLE IF NOT EXISTS analysis_covariate_results (
|
|
63
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
64
|
+
result_id INTEGER NOT NULL REFERENCES analysis_results(id),
|
|
65
|
+
covariate TEXT NOT NULL,
|
|
66
|
+
hr REAL,
|
|
67
|
+
ci_lower REAL,
|
|
68
|
+
ci_upper REAL,
|
|
69
|
+
wald_p REAL,
|
|
70
|
+
coef REAL,
|
|
71
|
+
se REAL,
|
|
72
|
+
z REAL,
|
|
73
|
+
reference_level TEXT,
|
|
74
|
+
tested_level TEXT
|
|
75
|
+
);
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
MIGRATIONS_SQL = """
|
|
79
|
+
ALTER TABLE analysis_results ADD COLUMN lr_test_p REAL;
|
|
80
|
+
ALTER TABLE analysis_results ADD COLUMN concordance_index REAL;
|
|
81
|
+
ALTER TABLE analysis_results ADD COLUMN ph_diagnostics_json TEXT;
|
|
82
|
+
|
|
83
|
+
CREATE TABLE IF NOT EXISTS analysis_covariate_results (
|
|
84
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
85
|
+
result_id INTEGER NOT NULL REFERENCES analysis_results(id),
|
|
86
|
+
covariate TEXT NOT NULL,
|
|
87
|
+
hr REAL,
|
|
88
|
+
ci_lower REAL,
|
|
89
|
+
ci_upper REAL,
|
|
90
|
+
wald_p REAL,
|
|
91
|
+
coef REAL,
|
|
92
|
+
se REAL,
|
|
93
|
+
z REAL,
|
|
94
|
+
reference_level TEXT,
|
|
95
|
+
tested_level TEXT
|
|
96
|
+
);
|
|
97
|
+
ALTER TABLE analysis_covariate_results ADD COLUMN reference_level TEXT;
|
|
98
|
+
ALTER TABLE analysis_covariate_results ADD COLUMN tested_level TEXT;
|
|
99
|
+
ALTER TABLE studies ADD COLUMN unmask_audit_json TEXT;
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
# ── Dataclasses ─────────────────────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass
|
|
106
|
+
class Study:
|
|
107
|
+
id: str
|
|
108
|
+
name: str
|
|
109
|
+
created_at: str # ISO-8601
|
|
110
|
+
study_type: Optional[str] = None
|
|
111
|
+
is_locked: int = 0
|
|
112
|
+
data_dir: str = ""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@dataclass
|
|
116
|
+
class Variable:
|
|
117
|
+
id: int
|
|
118
|
+
study_id: str
|
|
119
|
+
column_name: str
|
|
120
|
+
role: str # "baseline" | "outcome"
|
|
121
|
+
data_type: str # "categorical" | "continuous" | "time_to_event"
|
|
122
|
+
is_masked: bool = True
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@dataclass
|
|
126
|
+
class AnalysisResult:
|
|
127
|
+
id: int
|
|
128
|
+
study_id: str
|
|
129
|
+
study_plan_version: int
|
|
130
|
+
variable_ids_used: list[int]
|
|
131
|
+
test_name: str
|
|
132
|
+
statistic: Optional[float] = None
|
|
133
|
+
p_value: Optional[float] = None
|
|
134
|
+
adjusted_p_value: Optional[float] = None
|
|
135
|
+
confidence_interval: tuple[Optional[float], Optional[float]] = (None, None)
|
|
136
|
+
is_pre_registered: bool = True
|
|
137
|
+
provenance: Optional[dict] = None
|
|
138
|
+
computed_at: Optional[str] = None
|
|
File without changes
|