research-tool-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. app/backend/_bootstrap.py +15 -0
  2. app/backend/exports/verification_script.py +19 -0
  3. app/backend/main.py +59 -0
  4. app/backend/routers/__init__.py +1 -0
  5. app/backend/routers/execution.py +478 -0
  6. app/backend/routers/ingestion.py +233 -0
  7. app/backend/routers/planning.py +265 -0
  8. app/backend/routers/reporting.py +531 -0
  9. app/backend/state.py +44 -0
  10. core/__init__.py +0 -0
  11. core/cli/__init__.py +0 -0
  12. core/cli/main.py +1705 -0
  13. core/database.py +62 -0
  14. core/ingestion/__init__.py +0 -0
  15. core/ingestion/csv_loader.py +191 -0
  16. core/ingestion/variable_classifier.py +171 -0
  17. core/masking/__init__.py +0 -0
  18. core/masking/gate.py +128 -0
  19. core/models.py +138 -0
  20. core/planning/__init__.py +0 -0
  21. core/planning/diagnostics.py +89 -0
  22. core/planning/lock.py +232 -0
  23. core/planning/study_plan.py +73 -0
  24. core/planning/test_selector.py +518 -0
  25. core/provenance/__init__.py +0 -0
  26. core/provenance/hashing.py +38 -0
  27. core/provenance/tracker.py +105 -0
  28. core/reporting/__init__.py +62 -0
  29. core/reporting/appendix.py +58 -0
  30. core/reporting/bundle.py +378 -0
  31. core/reporting/excel_export.py +683 -0
  32. core/reporting/flowchart/__init__.py +20 -0
  33. core/reporting/flowchart/flowchart.py +511 -0
  34. core/reporting/forensics.py +592 -0
  35. core/reporting/forest_plot.py +614 -0
  36. core/reporting/lineage.py +562 -0
  37. core/reporting/manuscript_draft.py +726 -0
  38. core/reporting/plots.py +568 -0
  39. core/reporting/strobe_checklist.py +460 -0
  40. core/stats/__init__.py +0 -0
  41. core/stats/descriptive.py +104 -0
  42. core/stats/inferential.py +540 -0
  43. core/stats/multiple_comparisons.py +62 -0
  44. core/stats/post_hoc.py +62 -0
  45. exports/verification_script.py +19 -0
  46. research_tool_cli-0.1.0.dist-info/METADATA +16 -0
  47. research_tool_cli-0.1.0.dist-info/RECORD +93 -0
  48. research_tool_cli-0.1.0.dist-info/WHEEL +5 -0
  49. research_tool_cli-0.1.0.dist-info/entry_points.txt +2 -0
  50. research_tool_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
  51. research_tool_cli-0.1.0.dist-info/top_level.txt +4 -0
  52. tests/__init__.py +0 -0
  53. tests/conftest.py +45 -0
  54. tests/test_amendments.py +296 -0
  55. tests/test_analyze_batch_robustness.py +162 -0
  56. tests/test_app_statistical_reporting.py +383 -0
  57. tests/test_benchmark_21.py +556 -0
  58. tests/test_bundle.py +277 -0
  59. tests/test_cox_ph_plan.py +498 -0
  60. tests/test_csv_loader.py +368 -0
  61. tests/test_end_to_end.py +302 -0
  62. tests/test_excel_export.py +164 -0
  63. tests/test_flowchart.py +244 -0
  64. tests/test_forensics.py +305 -0
  65. tests/test_forest_plot.py +374 -0
  66. tests/test_from_json.py +176 -0
  67. tests/test_latest_plan_version.py +164 -0
  68. tests/test_lineage.py +329 -0
  69. tests/test_lock_immutability.py +133 -0
  70. tests/test_m1_fk_violation.py +85 -0
  71. tests/test_m2_data_hash_consistency.py +40 -0
  72. tests/test_m3_correction_timing.py +59 -0
  73. tests/test_m4_dedup_field.py +59 -0
  74. tests/test_m5_excel_hash_scope.py +45 -0
  75. tests/test_m6_fisher_exact_naming.py +44 -0
  76. tests/test_m7_duplicate_study_plan.py +55 -0
  77. tests/test_m8_filter_superseded.py +58 -0
  78. tests/test_m9_table1_groupby.py +56 -0
  79. tests/test_manuscript_draft.py +289 -0
  80. tests/test_masking_gate.py +196 -0
  81. tests/test_masking_migration.py +111 -0
  82. tests/test_multiple_comparisons.py +56 -0
  83. tests/test_plan_validation.py +289 -0
  84. tests/test_plots.py +394 -0
  85. tests/test_post_hoc_tagging.py +49 -0
  86. tests/test_posthoc_analyze.py +203 -0
  87. tests/test_provenance_tracker.py +110 -0
  88. tests/test_roadmap_features.py +148 -0
  89. tests/test_stats_descriptive.py +57 -0
  90. tests/test_stats_inferential.py +380 -0
  91. tests/test_strobe_checklist.py +172 -0
  92. tests/test_test_selector.py +350 -0
  93. tests/test_variable_classifier.py +124 -0
core/database.py ADDED
@@ -0,0 +1,62 @@
1
+ """SQLite database connection and initialization.
2
+
3
+ Uses stdlib sqlite3 — no ORM. Every study gets its own .db file under
4
+ data/studies/<study_id>/study.db.
5
+ """
6
+
7
+ import os
8
+ import sqlite3
9
+ from pathlib import Path
10
+
11
+ import sqlite3
12
+
13
+ from core.models import SCHEMA_SQL, MIGRATIONS_SQL
14
+
15
+ DATA_ROOT = Path("data/studies")
16
+
17
+
18
+ def study_dir(study_id: str) -> Path:
19
+ return DATA_ROOT / study_id
20
+
21
+
22
+ def get_connection(study_id: str) -> sqlite3.Connection:
23
+ """Return a connection to the study's database, creating dirs if needed."""
24
+ d = study_dir(study_id)
25
+ d.mkdir(parents=True, exist_ok=True)
26
+ db_path = d / "study.db"
27
+ conn = sqlite3.connect(str(db_path))
28
+ conn.row_factory = sqlite3.Row
29
+ conn.execute("PRAGMA journal_mode=WAL")
30
+ conn.execute("PRAGMA foreign_keys=ON")
31
+ migrate_db(conn)
32
+ return conn
33
+
34
+
35
+ def init_db(conn: sqlite3.Connection) -> None:
36
+ """Run CREATE TABLE IF NOT EXISTS for all tables."""
37
+ conn.executescript(SCHEMA_SQL)
38
+ conn.commit()
39
+
40
+
41
+ def migrate_db(conn: sqlite3.Connection) -> None:
42
+ """Apply migrations for schema additions (new columns, new tables).
43
+
44
+ Safe to call on any version — catches "duplicate column" errors.
45
+ Only runs if the target table already exists (skip on fresh DB before init_db).
46
+ """
47
+ existing = conn.execute(
48
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='analysis_results'"
49
+ ).fetchone()
50
+ if not existing:
51
+ return
52
+ for stmt in MIGRATIONS_SQL.split(";"):
53
+ stmt = stmt.strip()
54
+ if not stmt:
55
+ continue
56
+ try:
57
+ conn.execute(stmt)
58
+ except sqlite3.OperationalError as e:
59
+ if "duplicate column name" in str(e).lower():
60
+ continue
61
+ raise
62
+ conn.commit()
File without changes
@@ -0,0 +1,191 @@
1
+ """Load CSV/Excel into a dynamic per-study raw data table."""
2
+
3
+ import json
4
+ import sys
5
+ from pathlib import Path
6
+
7
+ import pandas as pd
8
+
9
+ from core.database import DATA_ROOT, get_connection, init_db
10
+
11
+
12
+ def _sanitize_col(name: str) -> str:
13
+ """Make a column name SQL-safe."""
14
+ return "".join(c if c.isalnum() or c == "_" else "_" for c in name).strip("_") or "col"
15
+
16
+
17
+ def _check_reingest(study_id: str) -> None:
18
+ """Check if study already has ingested data. Raises SystemExit if so."""
19
+ conn = get_connection(study_id)
20
+ init_db(conn)
21
+
22
+ # Check if raw_ table exists and has rows
23
+ raw = f"raw_{study_id}"
24
+ cur = conn.execute(
25
+ "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
26
+ (raw,),
27
+ )
28
+ if cur.fetchone() is None:
29
+ conn.close()
30
+ return # no raw table → clean
31
+
32
+ cur = conn.execute(f"SELECT COUNT(*) AS cnt FROM {raw}")
33
+ row_count = cur.fetchone()["cnt"]
34
+
35
+ cur = conn.execute(
36
+ "SELECT COUNT(*) AS cnt FROM variables WHERE study_id=?",
37
+ (study_id,),
38
+ )
39
+ var_count = cur.fetchone()["cnt"]
40
+
41
+ # Check for locked plan files on disk
42
+ locked_plans = list(DATA_ROOT.glob(f"{study_id}/study_plan.v*.locked.json"))
43
+ conn.close()
44
+
45
+ if row_count > 0 or var_count > 0 or locked_plans:
46
+ parts = []
47
+ if row_count > 0:
48
+ parts.append(f"{row_count} row(s) in raw table")
49
+ if var_count > 0:
50
+ parts.append(f"{var_count} classified variable(s)")
51
+ if locked_plans:
52
+ parts.append(f"{len(locked_plans)} locked plan(s)")
53
+ msg = (
54
+ f"Study '{study_id}' already has ingested data ({'; '.join(parts)}). "
55
+ f"Re-ingesting would invalidate existing variables/plan/results. "
56
+ f"Use 'new-study' to start fresh, or pass --force-reingest if "
57
+ f"this is intentional (clears all downstream state)."
58
+ )
59
+ print(msg, file=sys.stderr)
60
+ sys.exit(1)
61
+
62
+
63
+ def _cascade_clear(study_id: str) -> None:
64
+ """Drop downstream state so re-ingest can proceed cleanly."""
65
+ conn = get_connection(study_id)
66
+ init_db(conn)
67
+
68
+ masked = f"raw_masked_{study_id}"
69
+ conn.execute(f"DROP TABLE IF EXISTS {masked}")
70
+ # M1 fix: delete child table before parent to avoid FK violation
71
+ conn.execute(
72
+ "DELETE FROM analysis_covariate_results WHERE result_id IN "
73
+ "(SELECT id FROM analysis_results WHERE study_id=?)",
74
+ (study_id,),
75
+ )
76
+ conn.execute("DELETE FROM analysis_results WHERE study_id=?", (study_id,))
77
+ conn.execute("DELETE FROM variables WHERE study_id=?", (study_id,))
78
+
79
+ locked_plans = list(DATA_ROOT.glob(f"{study_id}/study_plan.v*.locked.json"))
80
+ for p in locked_plans:
81
+ p.unlink()
82
+
83
+ conn.commit()
84
+ conn.close()
85
+
86
+
87
+ def load_file(study_id: str, filepath: str, na_values: list[str] | None = None,
88
+ force: bool = False) -> list[str]:
89
+ """Load a CSV (or Excel) file into the study's database.
90
+
91
+ Creates a dynamic table named `raw_<study_id>` with one column per CSV column
92
+ (all TEXT). Returns the list of column names found.
93
+
94
+ Parameters
95
+ ----------
96
+ study_id : str
97
+ filepath : str
98
+ Path to the CSV or Excel file.
99
+ na_values : list[str], optional
100
+ Additional strings to treat as NA/NaN during CSV parsing
101
+ (e.g. ``["unknown", "missing"]``). Added on top of pandas' defaults.
102
+ force : bool
103
+ If True, clear downstream state before re-ingesting.
104
+ """
105
+ if not force:
106
+ _check_reingest(study_id)
107
+
108
+ path = Path(filepath)
109
+ if path.suffix in (".xls", ".xlsx"):
110
+ df = pd.read_excel(str(path))
111
+ else:
112
+ kwargs = {"keep_default_na": True}
113
+ if na_values:
114
+ kwargs["na_values"] = na_values
115
+ df = pd.read_csv(str(path), **kwargs)
116
+
117
+ conn = get_connection(study_id)
118
+ init_db(conn)
119
+
120
+ if force:
121
+ _cascade_clear(study_id)
122
+
123
+ # Strip leading/trailing whitespace from string columns
124
+ str_cols = df.select_dtypes(include="object").columns
125
+ for c in str_cols:
126
+ df[c] = df[c].apply(lambda v: v.strip() if isinstance(v, str) else v)
127
+ # Also convert whitespace-only strings to None so they register as missing
128
+ df = df.replace(r"^\s*$", pd.NA, regex=True)
129
+
130
+ # ── Duplicate patient_id detection ─────────────────────────────────
131
+ pid_raw = next((c for c in df.columns if c.lower() in ("patient_id", "patientid", "id", "subject_id")), None)
132
+ if pid_raw:
133
+ pid_col = _sanitize_col(pid_raw)
134
+ dupe_counts = df[pid_raw].value_counts()
135
+ dupes = dupe_counts[dupe_counts > 1]
136
+ if not dupes.empty:
137
+ dupe_list = ", ".join(f"'{pid}' ({n}x)" for pid, n in dupes.items())
138
+ print(f"Warning: duplicate patient identifiers found — {dupe_list}. "
139
+ f"These rows will be included in the analysis; "
140
+ f"verify they are genuine repeat records.",
141
+ file=__import__("sys").stderr)
142
+
143
+ col_names = [_sanitize_col(c) for c in df.columns]
144
+ col_defs = ", ".join(f'"{c}" TEXT' for c in col_names)
145
+ table_name = f"raw_{study_id}"
146
+
147
+ conn.execute(f"DROP TABLE IF EXISTS {table_name}")
148
+ conn.execute(f'CREATE TABLE {table_name} ("row_id" INTEGER PRIMARY KEY, {col_defs})')
149
+
150
+ placeholders = ", ".join("?" for _ in col_names)
151
+ col_list = ", ".join(f'"{c}"' for c in col_names)
152
+
153
+ for _, row in df.iterrows():
154
+ vals = tuple(None if pd.isna(v) else str(v) for v in row)
155
+ conn.execute(f"INSERT INTO {table_name} ({col_list}) VALUES ({placeholders})", vals)
156
+
157
+ conn.commit()
158
+ conn.close()
159
+ return col_names
160
+
161
+
162
+ def find_duplicate_patient_ids(study_id: str) -> list[tuple[str, int]]:
163
+ """Query the study's raw table for duplicate patient identifiers.
164
+
165
+ Returns a list of ``(patient_id, count)`` tuples for every duplicated
166
+ patient ID, ordered by count descending. Returns an empty list when
167
+ there is no patient_id column or no duplicates exist.
168
+ """
169
+ from core.database import get_connection
170
+
171
+ conn = get_connection(study_id)
172
+ raw_table = f"raw_{study_id}"
173
+
174
+ # Probe column names for a patient-like column
175
+ cur = conn.execute(f"PRAGMA table_info({raw_table})")
176
+ col_names = [r["name"] for r in cur.fetchall()]
177
+ pid_col = next(
178
+ (c for c in col_names if c.lower() in ("patient_id", "patientid", "id", "subject_id")),
179
+ None,
180
+ )
181
+ if not pid_col:
182
+ conn.close()
183
+ return []
184
+
185
+ cur = conn.execute(
186
+ f'SELECT "{pid_col}" AS pid, COUNT(*) AS cnt FROM {raw_table} '
187
+ f'GROUP BY "{pid_col}" HAVING cnt > 1 ORDER BY cnt DESC'
188
+ )
189
+ dupes = [(r["pid"], r["cnt"]) for r in cur.fetchall()]
190
+ conn.close()
191
+ return dupes
@@ -0,0 +1,171 @@
1
+ """Help the user classify variables as baseline/outcome and data types."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from core.database import get_connection
6
+
7
+
8
+ _OUTCOME_KEYWORDS = frozenset({
9
+ "outcome", "response", "survival", "pfs", "os", "status", "event",
10
+ "death", "followup", "lab", "marker", "protein", "score", "level",
11
+ "change", "fold", "day",
12
+ })
13
+
14
+
15
+ def _infer_data_type(study_id: str, column: str) -> str | None:
16
+ """Infer data type by inspecting actual column values.
17
+
18
+ Returns ``"continuous"``, ``"categorical"``, or ``None`` if the raw table
19
+ cannot be queried (heuristic-only fallback).
20
+ """
21
+ conn = get_connection(study_id)
22
+ try:
23
+ raw_table = f"raw_{study_id}"
24
+ cur = conn.execute(
25
+ f'SELECT DISTINCT "{column}" FROM {raw_table} WHERE "{column}" IS NOT NULL LIMIT 100'
26
+ )
27
+ values = [row[0] for row in cur.fetchall()]
28
+ if not values:
29
+ return None
30
+
31
+ # Attempt numeric parse
32
+ numeric = []
33
+ non_numeric = []
34
+ for v in values:
35
+ try:
36
+ float(v)
37
+ numeric.append(v)
38
+ except (ValueError, TypeError):
39
+ non_numeric.append(v)
40
+
41
+ # If any values are non-numeric strings, it's categorical
42
+ if non_numeric:
43
+ return "categorical"
44
+
45
+ # If all values are numeric, check distinct count
46
+ n_distinct = len(numeric)
47
+ if n_distinct > 5:
48
+ return "continuous"
49
+ return "categorical"
50
+ except Exception:
51
+ return None
52
+ finally:
53
+ conn.close()
54
+
55
+
56
+ def _has_low_cardinality_text_values(study_id: str, column: str) -> bool | None:
57
+ """Check if a column's values are non-numeric text with few distinct values.
58
+
59
+ Returns True for columns like ``high_risk_cytogenetics`` (yes/no),
60
+ False for columns with >5 distinct non-numeric values,
61
+ None when the table can't be queried.
62
+ """
63
+ conn = get_connection(study_id)
64
+ try:
65
+ raw_table = f"raw_{study_id}"
66
+ cur = conn.execute(
67
+ f'SELECT DISTINCT "{column}" FROM {raw_table} WHERE "{column}" IS NOT NULL LIMIT 100'
68
+ )
69
+ values = [row[0] for row in cur.fetchall()]
70
+ if not values:
71
+ return None
72
+
73
+ # Check if any value is non-numeric
74
+ numeric_count = 0
75
+ for v in values:
76
+ try:
77
+ float(v)
78
+ numeric_count += 1
79
+ except (ValueError, TypeError):
80
+ pass
81
+
82
+ # If values are numeric, no opinion
83
+ if numeric_count == len(values):
84
+ return None
85
+
86
+ # Text values with ≤5 distinct options → likely baseline/demographic
87
+ return len(values) <= 5
88
+ except Exception:
89
+ return None
90
+ finally:
91
+ conn.close()
92
+
93
+
94
+ def classify_variables_interactive(study_id: str, columns: list[str]) -> list[dict]:
95
+ """CLI-guided variable classification.
96
+
97
+ Uses heuristic name patterns AND actual data inspection.
98
+ Unrecognized column names get role="unclassified" to force explicit
99
+ resolution before planning.
100
+ """
101
+ results = []
102
+ for col in columns:
103
+ col_lower = col.lower()
104
+ tokens = set(col_lower.replace("-", "_").split("_"))
105
+
106
+ # ── Role (baseline / outcome / unclassified) ────────────────────
107
+ if tokens & _OUTCOME_KEYWORDS or any(
108
+ kw in col_lower for kw in ("outcome", "response", "survival",
109
+ "pfs", "os", "status", "event", "death",
110
+ "followup", "lab", "protein", "marker",
111
+ "score", "level", "change", "fold")
112
+ ):
113
+ default_role = "outcome"
114
+ elif tokens & {"id", "name", "patient", "identifier"}:
115
+ continue # skip identifier columns
116
+ elif tokens & {"age", "bmi", "weight", "height", "creatinine",
117
+ "count", "number", "value", "line", "lines", "prior",
118
+ "sex", "gender", "stage", "site", "location",
119
+ "center", "cohort", "group", "arm", "ecog"}:
120
+ default_role = "baseline"
121
+ else:
122
+ # If the column has low-cardinality text values (yes/no, mild/mod/severe),
123
+ # it's almost certainly baseline even if the name is unfamiliar.
124
+ is_low_card_text = _has_low_cardinality_text_values(study_id, col)
125
+ if is_low_card_text is True:
126
+ default_role = "baseline"
127
+ else:
128
+ # Unknown keyword pattern — flag for explicit user resolution
129
+ default_role = "unclassified"
130
+
131
+ # ── Data type (from actual values if possible) ──────────────────
132
+ # Start with a name-based guess
133
+ if any(kw in col_lower for kw in ("time", "days", "months", "duration", "follow_up", "fu")):
134
+ name_based_dtype = "time_to_event"
135
+ elif tokens & {"age", "bmi", "weight", "height", "value", "count",
136
+ "number", "creatinine", "line", "lines", "prior"}:
137
+ name_based_dtype = "continuous"
138
+ else:
139
+ name_based_dtype = "categorical"
140
+
141
+ # Use data inspection to *refine* — only upgrade categorical→continuous,
142
+ # never override a known name-based type like time_to_event.
143
+ inferred = _infer_data_type(study_id, col)
144
+ if name_based_dtype == "categorical" and inferred == "continuous":
145
+ default_dtype = "continuous"
146
+ elif name_based_dtype == "categorical" and inferred == "categorical":
147
+ default_dtype = "categorical"
148
+ else:
149
+ default_dtype = name_based_dtype
150
+
151
+ results.append({"column": col, "role": default_role, "data_type": default_dtype})
152
+ return results
153
+
154
+
155
+ def _classify_batch(study_id: str, variables: list[dict]) -> None:
156
+ """Write classified variables to the database.
157
+
158
+ Each dict: {column, role, data_type}
159
+ """
160
+ conn = get_connection(study_id)
161
+ for v in variables:
162
+ is_outcome = 1 if v["role"] == "outcome" else 0
163
+ conn.execute(
164
+ """INSERT INTO variables (study_id, column_name, role, data_type, is_masked)
165
+ VALUES (?, ?, ?, ?, ?)
166
+ ON CONFLICT(study_id, column_name) DO UPDATE SET
167
+ role=excluded.role, data_type=excluded.data_type, is_masked=excluded.is_masked""",
168
+ (study_id, v["column"], v["role"], v["data_type"], is_outcome),
169
+ )
170
+ conn.commit()
171
+ conn.close()
File without changes
core/masking/gate.py ADDED
@@ -0,0 +1,128 @@
1
+ """The Outcome-Masking Gate.
2
+
3
+ Enforces outcome masking at the physical storage level.
4
+
5
+ When variables are classified as outcomes, their values are moved from the
6
+ main raw_{study_id} table into a shadow raw_masked_{study_id} table and
7
+ replaced with NULLs. This catches EVERY access path — sqlite3 CLI, raw
8
+ Python sqlite3 module, any process that touches the database file. The only
9
+ way to recover the values is unmask(), which copies them back and transitions
10
+ the study to the unmasked state permanently.
11
+
12
+ State machine:
13
+ 0 (pre-lock) → outcome columns are NULL in the main table
14
+ 1 (locked) → outcome columns still NULL
15
+ 2 (unmasked) → outcome values restored, irreversible
16
+ """
17
+
18
+ from __future__ import annotations
19
+ import sqlite3
20
+ from datetime import datetime, timezone
21
+ from typing import Optional
22
+ from core.database import get_connection, init_db
23
+
24
+
25
+ class MaskedDataError(Exception):
26
+ """Raised when outcome data is accessed while the study is still masked."""
27
+
28
+
29
+ def _get_study_state(conn: sqlite3.Connection, study_id: str) -> int:
30
+ cur = conn.execute("SELECT is_locked FROM studies WHERE id=?", (study_id,))
31
+ row = cur.fetchone()
32
+ return row["is_locked"] if row else 0
33
+
34
+
35
+ def seal_outcomes(study_id: str) -> None:
36
+ """Move outcome column values into the masked shadow table, NULL them in main.
37
+
38
+ Called AFTER variable classification. This is the physical enforcement:
39
+ outcome values simply do not exist in the main table until unmask.
40
+ """
41
+ conn = get_connection(study_id)
42
+ init_db(conn)
43
+
44
+ # Get outcome column names
45
+ cur = conn.execute(
46
+ "SELECT column_name FROM variables WHERE study_id=? AND role='outcome'",
47
+ (study_id,),
48
+ )
49
+ outcome_cols = [row["column_name"] for row in cur.fetchall()]
50
+ if not outcome_cols:
51
+ conn.close()
52
+ return
53
+
54
+ raw = f"raw_{study_id}"
55
+ masked = f"raw_masked_{study_id}"
56
+
57
+ # Create the shadow table with same columns + row_id link
58
+ col_defs = ", ".join(f'"{c}" TEXT' for c in outcome_cols)
59
+ conn.execute(f"""
60
+ CREATE TABLE IF NOT EXISTS {masked} (
61
+ "row_id" INTEGER PRIMARY KEY REFERENCES {raw}(row_id),
62
+ {col_defs}
63
+ )
64
+ """)
65
+
66
+ # Copy outcome values from raw to masked
67
+ col_list = ", ".join(f'"{c}"' for c in outcome_cols)
68
+ conn.execute(f"DELETE FROM {masked}")
69
+ conn.execute(f"""
70
+ INSERT INTO {masked} (row_id, {col_list})
71
+ SELECT row_id, {col_list} FROM {raw}
72
+ """)
73
+ conn.commit()
74
+
75
+ # NULL out outcome columns in the main raw table
76
+ for col in outcome_cols:
77
+ conn.execute(f'UPDATE {raw} SET "{col}" = NULL')
78
+
79
+ conn.commit()
80
+ conn.close()
81
+
82
+
83
+ def unmask_study(study_id: str) -> None:
84
+ """Copy outcome values back from shadow table, transition to unmasked.
85
+
86
+ Irreversible — once called, outcome columns become permanently visible.
87
+ """
88
+ conn = get_connection(study_id)
89
+ init_db(conn)
90
+
91
+ cur = conn.execute(
92
+ "SELECT column_name FROM variables WHERE study_id=? AND role='outcome'",
93
+ (study_id,),
94
+ )
95
+ outcome_cols = [row["column_name"] for row in cur.fetchall()]
96
+ if not outcome_cols:
97
+ conn.close()
98
+ return
99
+
100
+ raw = f"raw_{study_id}"
101
+ masked = f"raw_masked_{study_id}"
102
+
103
+ for col in outcome_cols:
104
+ conn.execute(f"""
105
+ UPDATE {raw} SET "{col}" = (
106
+ SELECT "{col}" FROM {masked} WHERE {masked}.row_id = {raw}.row_id
107
+ )
108
+ """)
109
+
110
+ conn.execute("UPDATE studies SET is_locked=2, unmasked_at=? WHERE id=?", (datetime.now(timezone.utc).isoformat(), study_id))
111
+ conn.commit()
112
+ conn.close()
113
+
114
+
115
+ def lock_study(study_id: str) -> None:
116
+ """Transition to locked state."""
117
+ conn = get_connection(study_id)
118
+ init_db(conn)
119
+ conn.execute("UPDATE studies SET is_locked=1 WHERE id=?", (study_id,))
120
+ conn.commit()
121
+ conn.close()
122
+
123
+
124
+ def is_masked(study_id: str) -> bool:
125
+ conn = get_connection(study_id)
126
+ state = _get_study_state(conn, study_id)
127
+ conn.close()
128
+ return state < 2
core/models.py ADDED
@@ -0,0 +1,138 @@
1
+ """Data model dataclasses and SQL schema for the research tool."""
2
+
3
+ from __future__ import annotations
4
+ import json
5
+ from dataclasses import dataclass, field, asdict
6
+ from datetime import datetime, timezone
7
+ from typing import Optional
8
+
9
+ # ── SQL schema ──────────────────────────────────────────────────────────────
10
+
11
+ SCHEMA_SQL = """
12
+ CREATE TABLE IF NOT EXISTS studies (
13
+ id TEXT PRIMARY KEY,
14
+ name TEXT NOT NULL,
15
+ created_at TEXT NOT NULL, -- ISO-8601
16
+ study_type TEXT, -- cohort | case_control | cross_sectional
17
+ is_locked INTEGER NOT NULL DEFAULT 0, -- 0 = pre-lock, 1 = locked, 2 = unmasked
18
+ unmasked_at TEXT, -- ISO-8601 timestamp of unmask event
19
+ data_dir TEXT NOT NULL,
20
+ unmask_audit_json TEXT -- JSON array of override events
21
+ );
22
+
23
+ CREATE TABLE IF NOT EXISTS variables (
24
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
25
+ study_id TEXT NOT NULL REFERENCES studies(id),
26
+ column_name TEXT NOT NULL,
27
+ role TEXT NOT NULL, -- baseline | outcome
28
+ data_type TEXT NOT NULL, -- categorical | continuous | time_to_event
29
+ is_masked INTEGER NOT NULL DEFAULT 0,
30
+ UNIQUE(study_id, column_name)
31
+ );
32
+
33
+ CREATE TABLE IF NOT EXISTS raw_data (
34
+ row_id INTEGER PRIMARY KEY AUTOINCREMENT,
35
+ study_id TEXT NOT NULL REFERENCES studies(id),
36
+ json_row TEXT NOT NULL -- full row as JSON for flexibility
37
+ );
38
+
39
+ CREATE TABLE IF NOT EXISTS analysis_results (
40
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
41
+ study_id TEXT NOT NULL REFERENCES studies(id),
42
+ study_plan_version INTEGER,
43
+ variable_ids_used TEXT, -- JSON list
44
+ test_name TEXT,
45
+ statistic REAL,
46
+ p_value REAL,
47
+ adjusted_p_value REAL,
48
+ ci_lower REAL,
49
+ ci_upper REAL,
50
+ effect_size_json TEXT, -- JSON: {"metric": "Cramér's V", "value": 0.32}
51
+ sample_counts_json TEXT, -- JSON: {"n_total": 21, "n_analyzed": 21, "n_excluded": 0}
52
+ status_json TEXT, -- JSON: {"status": "completed", "reason": "..."}
53
+ is_pre_registered INTEGER NOT NULL DEFAULT 1,
54
+ provenance_json TEXT, -- JSON blob
55
+ computed_at TEXT NOT NULL,
56
+ superseded_previous_result_id INTEGER DEFAULT NULL,
57
+ lr_test_p REAL,
58
+ concordance_index REAL,
59
+ ph_diagnostics_json TEXT
60
+ );
61
+
62
+ CREATE TABLE IF NOT EXISTS analysis_covariate_results (
63
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
64
+ result_id INTEGER NOT NULL REFERENCES analysis_results(id),
65
+ covariate TEXT NOT NULL,
66
+ hr REAL,
67
+ ci_lower REAL,
68
+ ci_upper REAL,
69
+ wald_p REAL,
70
+ coef REAL,
71
+ se REAL,
72
+ z REAL,
73
+ reference_level TEXT,
74
+ tested_level TEXT
75
+ );
76
+ """
77
+
78
+ MIGRATIONS_SQL = """
79
+ ALTER TABLE analysis_results ADD COLUMN lr_test_p REAL;
80
+ ALTER TABLE analysis_results ADD COLUMN concordance_index REAL;
81
+ ALTER TABLE analysis_results ADD COLUMN ph_diagnostics_json TEXT;
82
+
83
+ CREATE TABLE IF NOT EXISTS analysis_covariate_results (
84
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
85
+ result_id INTEGER NOT NULL REFERENCES analysis_results(id),
86
+ covariate TEXT NOT NULL,
87
+ hr REAL,
88
+ ci_lower REAL,
89
+ ci_upper REAL,
90
+ wald_p REAL,
91
+ coef REAL,
92
+ se REAL,
93
+ z REAL,
94
+ reference_level TEXT,
95
+ tested_level TEXT
96
+ );
97
+ ALTER TABLE analysis_covariate_results ADD COLUMN reference_level TEXT;
98
+ ALTER TABLE analysis_covariate_results ADD COLUMN tested_level TEXT;
99
+ ALTER TABLE studies ADD COLUMN unmask_audit_json TEXT;
100
+ """
101
+
102
+ # ── Dataclasses ─────────────────────────────────────────────────────────────
103
+
104
+
105
+ @dataclass
106
+ class Study:
107
+ id: str
108
+ name: str
109
+ created_at: str # ISO-8601
110
+ study_type: Optional[str] = None
111
+ is_locked: int = 0
112
+ data_dir: str = ""
113
+
114
+
115
+ @dataclass
116
+ class Variable:
117
+ id: int
118
+ study_id: str
119
+ column_name: str
120
+ role: str # "baseline" | "outcome"
121
+ data_type: str # "categorical" | "continuous" | "time_to_event"
122
+ is_masked: bool = True
123
+
124
+
125
+ @dataclass
126
+ class AnalysisResult:
127
+ id: int
128
+ study_id: str
129
+ study_plan_version: int
130
+ variable_ids_used: list[int]
131
+ test_name: str
132
+ statistic: Optional[float] = None
133
+ p_value: Optional[float] = None
134
+ adjusted_p_value: Optional[float] = None
135
+ confidence_interval: tuple[Optional[float], Optional[float]] = (None, None)
136
+ is_pre_registered: bool = True
137
+ provenance: Optional[dict] = None
138
+ computed_at: Optional[str] = None
File without changes