research-tool-cli 0.1.0__py3-none-any.whl

app/backend/routers/ingestion.py

@@ -0,0 +1,233 @@
+"""
+Tab 1 — Data & Schema (Blinded Inspection).
+Maps to core.ingestion / core.masking.blind per DECISIONS.md tab-to-CLI mapping.
+
+Read-only inspection only. No drop/impute/interaction here (§3). Outcome column,
+once declared, is vaulted from variable browser / redundancy / cross-tabs for the
+rest of Tab 1 — everything below enforces that by filtering it out of responses,
+except the one fixed cohort banner (explicitly allowed, §3).
+
+NOTE: dtype inference and correlation/cross-tab logic below are simple pandas
+implementations to get Tab 1 running end-to-end. Swap for the real
+core.ingestion.variable_classifier / core stats calls when wiring the actual
+v2.7.0-core engine in — kept as thin as possible so that swap is a drop-in.
+"""
+import hashlib
+import io
+import json
+from itertools import combinations
+
+import pandas as pd
+from fastapi import APIRouter, File, HTTPException, UploadFile
+from pydantic import BaseModel
+
+from state import SESSION
+
+router = APIRouter()
+
+
+# ---------- request schemas ----------
+
+class ColumnMapping(BaseModel):
+    name: str
+    type: str  # "identifier" | "numeric_covariate" | "categorical_covariate" | "primary_outcome" | "time_to_event"
+
+class SchemaIn(BaseModel):
+    column_mappings: list[ColumnMapping]
+
+class OutcomeIn(BaseModel):
+    column_name: str
+    event_value: str
+    censored_value: str
+
+class SentinelsIn(BaseModel):
+    global_na_strings: list[str] = []
+    column_overrides: dict[str, list[str]] = {}
+
+
+# ---------- helpers ----------
+
+def _require_data():
+    if SESSION.raw_df is None:
+        raise HTTPException(400, "No dataset uploaded yet. POST /api/ingest/upload first.")
+
+def _outcome_col():
+    return SESSION.outcome_spec["column_name"] if SESSION.outcome_vaulted() else None
+
+def _apply_sentinels(df: pd.DataFrame) -> pd.DataFrame:
+    out = df.copy()
+    global_na = SESSION.sentinels.get("global_na_strings", [])
+    overrides = SESSION.sentinels.get("column_overrides", {})
+    for col in out.columns:
+        na_list = global_na + overrides.get(col, [])
+        if na_list:
+            out[col] = out[col].replace(na_list, pd.NA)
+    return out
+
+def _visible_columns() -> list[str]:
+    """Columns minus the vaulted outcome identity."""
+    outcome = _outcome_col()
+    return [c for c in SESSION.raw_df.columns if c != outcome]
+
+
+# ---------- endpoints ----------
+
+@router.post("/upload")
+async def upload(file: UploadFile = File(...)):
+    raw_bytes = await file.read()
+    SESSION.dataset_fingerprint = hashlib.sha256(raw_bytes).hexdigest()
+    SESSION.raw_path = file.filename
+
+    if file.filename.lower().endswith((".xlsx", ".xls")):
+        df = pd.read_excel(io.BytesIO(raw_bytes))
+    else:
+        df = pd.read_csv(io.BytesIO(raw_bytes))
+    SESSION.raw_df = df
+
+    # naive dtype guess — placeholder for core.ingestion.variable_classifier
+    guessed = []
+    for col in df.columns:
+        if df[col].nunique() == len(df[col]) and df[col].is_unique:
+            t = "identifier"
+        elif pd.api.types.is_numeric_dtype(df[col]):
+            t = "numeric_covariate"
+        else:
+            t = "categorical_covariate"
+        guessed.append({"name": col, "guessed_type": t})
+
+    return {
+        "dataset_fingerprint": SESSION.dataset_fingerprint,
+        "n_rows": len(df),
+        "columns": guessed,
+    }
+
+
+@router.post("/schema")
+def set_schema(body: SchemaIn):
+    _require_data()
+    mappings = []
+    for m in body.column_mappings:
+        entry = {"name": m.name, "type": m.type}
+        if m.type == "identifier":
+            entry["has_duplicates"] = bool(SESSION.raw_df[m.name].duplicated().any())
+        mappings.append(entry)
+    SESSION.column_mappings = mappings
+    return {"column_mappings": mappings}
+
+
+@router.post("/outcome")
+def declare_outcome(body: OutcomeIn):
+    """Direct declaration only — never auto-detected (§3). Vaults the column immediately."""
+    _require_data()
+    if body.column_name not in SESSION.raw_df.columns:
+        raise HTTPException(400, f"Column '{body.column_name}' not found.")
+    SESSION.outcome_spec = {
+        "column_name": body.column_name,
+        "event_value": body.event_value,
+        "censored_value": body.censored_value,
+        "vaulted": True,
+    }
+    return {"outcome_spec": {"vaulted": True}}  # column identity not echoed back either
+
+
+@router.post("/sentinels")
+def set_sentinels(body: SentinelsIn):
+    _require_data()
+    SESSION.sentinels = {
+        "global_na_strings": body.global_na_strings,
+        "column_overrides": body.column_overrides,
+    }
+    return SESSION.sentinels
+
+
+@router.get("/missingness")
+def missingness():
+    _require_data()
+    df = _apply_sentinels(SESSION.raw_df)
+    cols = _visible_columns()
+    return {c: round(float(df[c].isna().mean()), 4) for c in cols}
+
+
+@router.get("/redundancy")
+def redundancy():
+    """Covariate-covariate correlation only — outcome excluded. |r| > 0.8 flagged (§3)."""
+    _require_data()
+    df = _apply_sentinels(SESSION.raw_df)
+    numeric_cols = [
+        c for c in _visible_columns()
+        if pd.api.types.is_numeric_dtype(df[c])
+    ]
+    pairs = []
+    for a, b in combinations(numeric_cols, 2):
+        r = df[[a, b]].corr().iloc[0, 1]
+        if pd.notna(r) and abs(r) > 0.8:
+            pairs.append({"var1": a, "var2": b, "r": round(float(r), 3)})
+    return {"high_correlation_pairs": pairs}
+
+
+@router.get("/sparse-crosstabs")
+def sparse_crosstabs(min_cell_threshold: int = 5):
+    """Categorical-categorical sparse cell check — outcome excluded."""
+    _require_data()
+    df = _apply_sentinels(SESSION.raw_df)
+    cat_cols = [
+        c for c in _visible_columns()
+        if not pd.api.types.is_numeric_dtype(df[c])
+    ]
+    flagged = []
+    for a, b in combinations(cat_cols, 2):
+        ct = pd.crosstab(df[a], df[b])
+        if ct.size and ct.values.min() < min_cell_threshold:
+            flagged.append({"var1": a, "var2": b, "min_cell_count": int(ct.values.min())})
+    return {"sparse_cross_tabs": flagged}
+
+
+@router.get("/cohort-banner")
+def cohort_banner():
+    """The one allowed exception: touches outcome once, fixed, not recomputed per action (§3)."""
+    _require_data()
+    if not SESSION.outcome_vaulted():
+        raise HTTPException(400, "Declare outcome first — banner needs event count.")
+    df = SESSION.raw_df
+    spec = SESSION.outcome_spec
+    n_total = len(df)
+    e_total = int((df[spec["column_name"]].astype(str) == str(spec["event_value"])).sum())
+    return {
+        "n_total": n_total,
+        "e_total": e_total,
+        "event_rate": round(e_total / n_total, 4) if n_total else 0.0,
+    }
+
+
+@router.post("/lock")
+def lock_stage1():
+    """Emits Stage1Payload (H0) per DECISIONS.md §3. Hash excludes itself, computed then inserted."""
+    _require_data()
+    if not SESSION.outcome_vaulted():
+        raise HTTPException(400, "Cannot lock Stage 1 without a declared outcome.")
+
+    banner = cohort_banner()
+    miss = missingness()
+    redund = redundancy()
+    sparse = sparse_crosstabs()
+
+    payload = {
+        "provenance": {
+            "dataset_fingerprint": SESSION.dataset_fingerprint,
+            "ephemeral_raw_path": SESSION.raw_path,
+        },
+        "cohort_facts": banner,
+        "outcome_spec": SESSION.outcome_spec,
+        "column_mappings": SESSION.column_mappings,
+        "sentinels": SESSION.sentinels,
+        "precomputations_advisory_cache": {
+            "is_advisory": True,
+            "missingness_pct": miss,
+            "high_correlation_pairs": redund["high_correlation_pairs"],
+            "sparse_cross_tabs": sparse["sparse_cross_tabs"],
+        },
+    }
+    canonical = json.dumps(payload, sort_keys=True, default=str).encode()
+    payload["provenance"]["payload_fingerprint_h0"] = hashlib.sha256(canonical).hexdigest()
+    SESSION.h0_payload = payload  # Tab 2 chains plan_fingerprint_h1 off this
+    return payload

app/backend/routers/planning.py

@@ -0,0 +1,265 @@
+"""
+Tab 2 — Socratic Wizard & Lock Gate.
+Maps to core.planning / core.provenance per DECISIONS.md §4-5.
+
+Novice wizard and Expert JSON-import both land here structurally — this file only
+implements the wizard steps (entry_mode="socratic_wizard"). Expert direct-spec
+import is a separate endpoint, not built this session.
+
+Steps A-D per §4:
+  A exposure          -> /exposure
+  B outcome confirm    -> /outcome-confirm (GET, read-only re-display) + /time-column
+  C confounders        -> /confounders, /interactions (Manuscript Mirror at /manuscript-mirror)
+  D rigor / missing    -> /missing-strategy, /epv-live (recomputed live, never cached)
+lock -> /lock, emits Stage2Payload (H1), chained to H0 via payload_fingerprint_h0.
+"""
+import hashlib
+import json
+
+import pandas as pd
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from state import SESSION
+
+router = APIRouter()
+
+
+# ---------- request schemas ----------
+
+class ExposureIn(BaseModel):
+    column_name: str
+    reference_level: str
+
+class TimeColumnIn(BaseModel):
+    time_column: str
+
+class ConfoundersIn(BaseModel):
+    confounders: list[str]
+
+class InteractionIn(BaseModel):
+    term: str
+    rationale: str
+
+class MissingStrategyIn(BaseModel):
+    global_default: str  # "complete_case" | "impute" | "flag_as_missing_category"
+    column_overrides: dict[str, str] = {}
+
+
+# ---------- helpers ----------
+
+def _require_h0():
+    if SESSION.h0_payload is None:
+        raise HTTPException(400, "Lock Stage 1 first — no H0 payload to chain off.")
+
+def _apply_sentinels(df: pd.DataFrame) -> pd.DataFrame:
+    out = df.copy()
+    global_na = SESSION.sentinels.get("global_na_strings", [])
+    overrides = SESSION.sentinels.get("column_overrides", {})
+    for col in out.columns:
+        na_list = global_na + overrides.get(col, [])
+        if na_list:
+            out[col] = out[col].replace(na_list, pd.NA)
+    return out
+
+def _complete_case_columns() -> list[str]:
+    """Columns whose missingness actually costs rows under the current strategy 
+    impute / flag_as_missing_category columns don't drop rows, complete_case ones do."""
+    strat = SESSION.missing_data_strategy
+    cols = []
+    if SESSION.exposure:
+        cols.append(SESSION.exposure["column_name"])
+    cols += SESSION.confounders
+    for c in cols[:]:
+        method = strat["column_overrides"].get(c, strat["global_default"])
+        if method != "complete_case":
+            cols.remove(c)
+    return cols
+
+
+# ---------- endpoints ----------
+
+@router.post("/exposure")
+def set_exposure(body: ExposureIn):
+    _require_h0()
+    vaulted_outcome = SESSION.outcome_spec["column_name"]
+    if body.column_name == vaulted_outcome:
+        raise HTTPException(400, "Exposure can't be the vaulted outcome column.")
+    SESSION.exposure = body.dict()
+    return SESSION.exposure
+
+
+@router.get("/outcome-confirm")
+def outcome_confirm():
+    """Read-only re-display — never a re-selection. Outcome was fixed in Stage 1."""
+    _require_h0()
+    spec = SESSION.outcome_spec
+    return {
+        "column_name": spec["column_name"],
+        "event_value": spec["event_value"],
+        "censored_value": spec["censored_value"],
+        "time_column": SESSION.time_column,
+    }
+
+
+@router.post("/time-column")
+def set_time_column(body: TimeColumnIn):
+    _require_h0()
+    SESSION.time_column = body.time_column
+    return {"time_column": SESSION.time_column}
+
+
+@router.post("/confounders")
+def set_confounders(body: ConfoundersIn):
+    _require_h0()
+    SESSION.confounders = body.confounders
+    return {"confounders": SESSION.confounders}
+
+
+@router.get("/redundancy-warnings")
+def redundancy_warnings():
+    """Surfaces only the pairs relevant to what's actually been selected as confounders."""
+    _require_h0()
+    selected = set(SESSION.confounders)
+    pairs = SESSION.h0_payload["precomputations_advisory_cache"]["high_correlation_pairs"]
+    return {"warnings": [p for p in pairs if p["var1"] in selected and p["var2"] in selected]}
+
+
+@router.post("/interactions")
+def add_interaction(body: InteractionIn):
+    """Interaction terms permitted only here (§4), gated by rationale sanity floor (§5) —
+    ~15 char minimum to block literal 'yes'/'none', not a strict quality gate."""
+    _require_h0()
+    if len(body.rationale.strip()) < 15:
+        raise HTTPException(400, "Rationale too short — needs genuine clinical justification, not a placeholder.")
+    SESSION.interactions.append(body.dict())
+    return {"interactions": SESSION.interactions}
+
+
+@router.get("/manuscript-mirror")
+def manuscript_mirror():
+    """Live preview of how each interaction rationale will read in the exported
+    Methods / STROBE supplement — the real quality incentive, no keyword gating (§5)."""
+    _require_h0()
+    lines = []
+    for i in SESSION.interactions:
+        lines.append(
+            f"An interaction term ({i['term']}) was pre-specified. Rationale: {i['rationale']}"
+        )
+    return {"methods_section_preview": " ".join(lines) or "No interaction terms declared."}
+
+
+@router.post("/missing-strategy")
+def set_missing_strategy(body: MissingStrategyIn):
+    _require_h0()
+    SESSION.missing_data_strategy = body.dict()
+    return SESSION.missing_data_strategy
+
+
+@router.get("/epv-live")
+def epv_live():
+    """Recomputed fresh on every call — never a cached/static number (§4)."""
+    _require_h0()
+    if not SESSION.exposure:
+        raise HTTPException(400, "Set exposure first.")
+
+    df = _apply_sentinels(SESSION.raw_df)
+    cc_cols = _complete_case_columns()
+    outcome_col = SESSION.outcome_spec["column_name"]
+
+    subset_cols = cc_cols + [outcome_col]
+    effective_df = df.dropna(subset=subset_cols) if cc_cols else df
+
+    n_effective = len(effective_df)
+    e_total = (effective_df[outcome_col].astype(str) == str(SESSION.outcome_spec["event_value"])).sum()
+
+    # Calculate degrees of freedom (k) for main effects
+    predictors = [SESSION.exposure["column_name"]] + SESSION.confounders
+    k = 0
+    for col in predictors:
+        if col in effective_df.columns:
+            if pd.api.types.is_numeric_dtype(effective_df[col]):
+                k += 1
+            else:
+                n_uniq = effective_df[col].dropna().nunique()
+                k += max(n_uniq - 1, 1)
+        else:
+            k += 1
+
+    # Calculate degrees of freedom for interaction terms (DoF_A * DoF_B)
+    for inter in SESSION.interactions:
+        term_str = inter.get("term", "")
+        parts = [p.strip() for p in term_str.replace("*", ":").split(":") if p.strip()]
+        if len(parts) >= 2:
+            inter_dof = 1
+            for p in parts:
+                if p in effective_df.columns:
+                    if not pd.api.types.is_numeric_dtype(effective_df[p]):
+                        u = effective_df[p].dropna().nunique()
+                        inter_dof *= max(u - 1, 1)
+            k += inter_dof
+        else:
+            k += 1
+
+    epv = round(e_total / k, 2) if k else None
+
+    return {
+        "n_effective": n_effective,
+        "e_effective": int(e_total),
+        "parameters_k": k,
+        "epv": epv,
+    }
+
+
+@router.get("/amendment-status")
+def amendment_status():
+    return {"pending": SESSION.pending_amendment}
+
+
+@router.post("/lock")
+def lock_stage2():
+    """Emits Stage2Payload (H1), chained off H0's payload_fingerprint_h0."""
+    _require_h0()
+    if not SESSION.exposure:
+        raise HTTPException(400, "Exposure not set — Step A incomplete.")
+
+    feasibility = epv_live()
+    if feasibility["epv"] is not None and feasibility["epv"] < 10:
+        # not a hard block by design choice here — surfaced, not silently allowed either.
+        # (DECISIONS.md doesn't mandate a hard EPV floor at lock; flag only.)
+        pass
+
+    payload = {
+        "provenance": {
+            "payload_fingerprint_h0": SESSION.h0_payload["provenance"]["payload_fingerprint_h0"],
+            "entry_mode": "socratic_wizard",
+        },
+        "protocol": {
+            "study_design": "retrospective_cohort",
+            "exposure": SESSION.exposure,
+            "outcome_confirmation": {
+                "column_name": SESSION.outcome_spec["column_name"],
+                "event_value": SESSION.outcome_spec["event_value"],
+                "time_column": SESSION.time_column,
+            },
+            "confounders": SESSION.confounders,
+            "interactions": SESSION.interactions,
+            "missing_data_strategy": SESSION.missing_data_strategy,
+            "pre_specified_transforms": {},  # NOTE: assumes linear for all continuous covariates
+                                              # until Tab2 UI grows a transform picker (Gate 4 reads this)
+        },
+        "locked_feasibility_metrics": feasibility,
+    }
+    # amendment chaining: if this lock follows a Tab3 FAIL, parent is the failed plan's hash,
+    # not H0 directly — new plan hash still gets computed fresh below.
+    if SESSION.pending_amendment:
+        payload["provenance"]["parent_plan_hash"] = SESSION.pending_amendment["parent_plan_hash"]
+        payload["provenance"]["amendment_rationale"] = SESSION.pending_amendment.get("rationale", "")
+
+    canonical = json.dumps(payload, sort_keys=True, default=str).encode()
+    payload["provenance"]["plan_fingerprint_h1"] = hashlib.sha256(canonical).hexdigest()
+
+    SESSION.h1_payload = payload
+    SESSION.plan_chain.append(payload)
+    SESSION.pending_amendment = None
+    return payload