syntha-ehr 0.5.0__py3-none-any.whl

syntha/__init__.py

@@ -0,0 +1,14 @@
+"""syntha — synthetic patient record generator.
+
+Top-level re-exports for ergonomics. Users can do:
+
+    from syntha import GaussianCopulaGenerator, PipelineConfig, run
+
+instead of having to know the submodule layout.
+"""
+__version__ = "0.5.0"
+
+from .generator.copula import GaussianCopulaGenerator
+from .pipeline import PipelineConfig, run
+
+__all__ = ["GaussianCopulaGenerator", "PipelineConfig", "run", "__version__"]

syntha/cli.py

@@ -0,0 +1,256 @@
+"""Command-line interface for syntha."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import click
+import numpy as np
+import pandas as pd
+
+from .generator.copula import GaussianCopulaGenerator
+from .models.registry import ModelRegistry
+from .pipeline import PipelineConfig, run
+
+
+@click.group()
+@click.version_option()
+def main() -> None:
+    """syntha — synthetic patient record generator."""
+
+
+@main.command()
+@click.option("--input", "input_csv", required=True, type=click.Path(exists=True))
+@click.option("--output", "output_dir", required=True, type=click.Path())
+@click.option("--n", default=1000, show_default=True, help="Number of synthetic episodes (or total encounters in longitudinal mode)")
+@click.option("--cohort", default="strict", show_default=True)
+@click.option("--seed", default=42, show_default=True)
+@click.option("--csv/--no-csv", default=True, show_default=True)
+@click.option("--fhir/--no-fhir", default=True, show_default=True)
+@click.option("--fhir-format", type=click.Choice(["ndjson", "json"]), default="ndjson", show_default=True)
+@click.option("--modules/--no-modules", default=True, show_default=True, help="Run Synthea-style clinical modules during FHIR export")
+@click.option("--longitudinal", is_flag=True, default=False, help="Expand each baseline into multiple encounters over time")
+@click.option("--encounters-per-patient", default=4.0, show_default=True)
+@click.option("--years-of-history", default=3.0, show_default=True)
+@click.option("--registry-dir", default=None, help="Directory for the trained-model registry (default: <output>/models)")
+@click.option("--lab-history/--no-lab-history", default=False, show_default=True,
+              help="Emit 2-4 prior measurements per lab (v0.5.5 longitudinal labs)")
+@click.option("--conditional-missingness/--no-conditional-missingness", default=True,
+              show_default=True, help="Apply comorbidity-conditional missingness (v0.5.2)")
+@click.option("--validation/--no-validation", default=True, show_default=True,
+              help="Compute KS/Wasserstein/correlation report alongside output")
+def generate(input_csv, output_dir, n, cohort, seed, csv, fhir, fhir_format,
+             modules, longitudinal, encounters_per_patient, years_of_history,
+             registry_dir, lab_history, conditional_missingness, validation):
+    """Train copula, sample, run modules, write CSV + FHIR + model card."""
+    cfg = PipelineConfig(
+        n=n, cohort=cohort, random_seed=seed,
+        write_csv=csv, write_fhir=fhir, fhir_format=fhir_format,
+        run_modules=modules, longitudinal=longitudinal,
+        encounters_per_patient_mean=encounters_per_patient,
+        years_of_history=years_of_history, registry_dir=registry_dir,
+        write_validation=validation,
+        apply_conditional_missingness=conditional_missingness,
+        include_lab_history=lab_history,
+    )
+    click.echo(json.dumps(run(input_csv, output_dir, cfg), indent=2))
+
+
+@main.command()
+@click.option("--input", "input_csv", required=True, type=click.Path(exists=True))
+@click.option("--registry", "registry_dir", required=True, type=click.Path())
+@click.option("--name", required=True)
+@click.option("--cohort", default="strict", show_default=True)
+@click.option("--seed", default=42, show_default=True)
+def fit(input_csv, registry_dir, name, cohort, seed):
+    """Fit a copula and store it (with model card) in the registry."""
+    from . import data, preprocess
+    src = data.load_episodes(input_csv)
+    modeled = preprocess.clip_to_physiologic(
+        preprocess.coerce_types(data.filter_to_modeled(src))
+    )
+    feat_df, bcols, ccols = preprocess.split_modeled(modeled)
+    gen = GaussianCopulaGenerator(random_seed=seed).fit(feat_df, bcols, ccols, cohort=cohort)
+    registry = ModelRegistry(registry_dir)
+    card = registry.save(name, gen, input_csv, modeled, bcols, ccols, cohort)
+    click.echo(json.dumps({
+        "registry": str(registry_dir), "name": name,
+        "n_train": card.n_train, "source_sha256": card.source_sha256,
+    }, indent=2))
+
+
+@main.command(name="list-models")
+@click.option("--registry", "registry_dir", required=True, type=click.Path(exists=True))
+def list_models(registry_dir):
+    """List models in a registry."""
+    registry = ModelRegistry(registry_dir)
+    click.echo(json.dumps(registry.list_models(), indent=2))
+
+
+@main.command(name="export-model")
+@click.option("--registry", "registry_dir", required=True, type=click.Path(exists=True))
+@click.option("--name", required=True)
+@click.option("--output", "output_path", required=True, type=click.Path())
+@click.option("--quantiles", default=200, show_default=True, help="Order statistics per continuous marginal")
+def export_model(registry_dir, name, output_path, quantiles):
+    """Export a registered copula to JSON for use by the Tauri desktop app."""
+    from .export_model import export_model_to_json
+    gen, card = ModelRegistry(registry_dir).load(name)
+    path = export_model_to_json(gen, output_path, n_quantiles=quantiles)
+    click.echo(json.dumps({
+        "path": str(path), "cohort": card.cohort,
+        "n_train": card.n_train, "size_kb": path.stat().st_size // 1024,
+    }, indent=2))
+
+
+@main.command(name="show-card")
+@click.option("--registry", "registry_dir", required=True, type=click.Path(exists=True))
+@click.option("--name", required=True)
+def show_card(registry_dir, name):
+    """Show a model card."""
+    registry = ModelRegistry(registry_dir)
+    _, card = registry.load(name)
+    click.echo(card.to_json())
+
+
+@main.command()
+@click.option("--registry", "registry_dir", required=True, type=click.Path(exists=True))
+@click.option("--name", required=True)
+@click.option("--output", "output_csv", required=True, type=click.Path())
+@click.option("--n", default=1000, show_default=True)
+def sample(registry_dir, name, output_csv, n):
+    """Sample n rows from a registered model (raw — no constraints, no FHIR)."""
+    registry = ModelRegistry(registry_dir)
+    gen, _ = registry.load(name)
+    df = gen.sample(n)
+    Path(output_csv).parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(output_csv, index=False)
+    click.echo(json.dumps({"rows": len(df), "csv": output_csv}, indent=2))
+
+
+@main.command(name="sample-conditional")
+@click.option("--registry", "registry_dir", required=True, type=click.Path(exists=True))
+@click.option("--name", required=True)
+@click.option("--output", "output_csv", required=True, type=click.Path())
+@click.option("--n", default=1000, show_default=True, help="Target number of accepted rows")
+@click.option(
+    "--condition",
+    required=True,
+    help=(
+        "pandas-style filter expression in the model's column names. "
+        'Example: --condition "age > 60 & DM_Tum == 1 & bp_systolic >= 140"'
+    ),
+)
+@click.option("--oversample", default=5.0, show_default=True,
+              help="Initial oversample factor for rejection sampling")
+@click.option("--max-rounds", default=10, show_default=True)
+def sample_conditional_cmd(registry_dir, name, output_csv, n, condition,
+                            oversample, max_rounds):
+    """Conditional sampling via rejection — return only rows matching --condition."""
+    from .conditional import sample_conditional
+    registry = ModelRegistry(registry_dir)
+    gen, _ = registry.load(name)
+    result = sample_conditional(
+        gen, n=n, condition=condition,
+        oversample_factor=oversample, max_rounds=max_rounds,
+    )
+    Path(output_csv).parent.mkdir(parents=True, exist_ok=True)
+    result.rows.to_csv(output_csv, index=False)
+    click.echo(json.dumps({
+        "n_requested": result.n_requested,
+        "n_generated": result.n_generated,
+        "rounds": result.rounds,
+        "rejection_rate": round(result.rejection_rate, 3),
+        "csv": output_csv,
+    }, indent=2))
+
+
+@main.command()
+@click.option("--input", "input_csv", required=True, type=click.Path(exists=True))
+@click.option("--output", "output_dir", required=True, type=click.Path())
+@click.option("--format", "fmt", type=click.Choice(["ndjson", "json"]), default="ndjson", show_default=True)
+@click.option("--modules/--no-modules", default=True, show_default=True)
+def fhir(input_csv, output_dir, fmt, modules):
+    """Convert an existing synthetic CSV to FHIR R4 bundles."""
+    from .fhir.export import write_fhir_bundles
+    df = pd.read_csv(input_csv)
+    path = write_fhir_bundles(df, output_dir, fmt=fmt, run_modules=modules)
+    click.echo(json.dumps({"rows": len(df), "fhir": str(path)}, indent=2))
+
+
+@main.command()
+@click.option("--bundles", "bundles_ndjson", required=True, type=click.Path(exists=True), help="Path to bundles.ndjson")
+@click.option("--host", default="127.0.0.1", show_default=True)
+@click.option("--port", default=8080, show_default=True)
+def serve(bundles_ndjson, host, port):
+    """Boot a minimal read-only FHIR R4 server backed by a bundles NDJSON file."""
+    from .server import serve_forever
+    serve_forever(bundles_ndjson, host=host, port=port)
+
+
+@main.command(name="validate")
+@click.option("--source", "source_csv", required=True, type=click.Path(exists=True))
+@click.option("--synthetic", "synthetic_csv", required=True, type=click.Path(exists=True))
+@click.option("--output", "report_path", required=True, type=click.Path())
+def validate_cmd(source_csv, synthetic_csv, report_path):
+    """Compute KS / Wasserstein / correlation-diff between source and synthetic.
+
+    (CLI command name is `validate`; the Python function is `validate_cmd`
+    to avoid shadowing the `validate.validate` module function — per the
+    v0.5 architecture review.)
+    """
+    from . import data, preprocess
+    from .validate import save_report
+    from .validate import validate as _v
+
+    src = preprocess.coerce_types(data.filter_to_modeled(data.load_episodes(source_csv)))
+    syn = pd.read_csv(synthetic_csv)
+    _, bcols, ccols = preprocess.split_modeled(src)
+    report = _v(src, syn, ccols, bcols)
+    save_report(report, report_path)
+    click.echo(json.dumps(report.summary(), indent=2))
+
+
+@main.command()
+@click.option("--source", "source_csv", required=True, type=click.Path(exists=True))
+@click.option("--synthetic", "synthetic_csv", required=True, type=click.Path(exists=True))
+@click.option("--output", "report_path", required=True, type=click.Path())
+@click.option("--split", default=0.8, show_default=True,
+              help="Fraction of source used as train (rest is held out for the attack)")
+def audit(source_csv, synthetic_csv, report_path, split):
+    """Run a privacy audit (membership + attribute inference attacks) against
+    a synthetic CSV and write a JSON report. CI fails on MIA AUC > 0.60."""
+    from . import data, preprocess
+    from .privacy import DEFAULT_MIA_THRESHOLD, run_privacy_audit
+
+    src = preprocess.coerce_types(data.filter_to_modeled(data.load_episodes(source_csv)))
+    syn = pd.read_csv(synthetic_csv)
+    rng = np.random.default_rng(42)
+    idx = rng.permutation(len(src))
+    n_train = int(split * len(src))
+    real_train = src.iloc[idx[:n_train]].copy()
+    real_holdout = src.iloc[idx[n_train:]].copy()
+
+    feature_cols = [c for c in src.columns if c in syn.columns
+                    and c not in ("RF_EPISODE2", "HASTA_ID", "episode_date", "gender")]
+    sensitive_targets = [c for c in ["Hipertansiyon", "DM_Tum", "Hiperlipidemi"]
+                         if c in feature_cols]
+
+    report = run_privacy_audit(
+        real_train, real_holdout, syn,
+        feature_cols=feature_cols,
+        sensitive_targets=sensitive_targets,
+    )
+    summary = report.summary()
+    Path(report_path).parent.mkdir(parents=True, exist_ok=True)
+    Path(report_path).write_text(json.dumps(summary, indent=2))
+    click.echo(json.dumps(summary, indent=2))
+    if summary["membership_inference_auc"] > DEFAULT_MIA_THRESHOLD:
+        raise click.ClickException(
+            f"membership inference AUC {summary['membership_inference_auc']:.3f} > "
+            f"{DEFAULT_MIA_THRESHOLD} — possible memorization"
+        )
+
+
+if __name__ == "__main__":
+    main()

syntha/conditional.py

@@ -0,0 +1,176 @@
+"""Conditional sampling — generate synthetic episodes matching a filter.
+
+Usage from the CLI:
+
+    syntha generate \\
+        --input data/raw/pristine_tolerant_episodes.csv \\
+        --condition 'age > 60 & DM_Tum == 1' \\
+        --n 1000 --output out/diabetic_seniors
+
+Implementation: rejection sampling around the existing copula. Sample
+`oversample_factor × n` rows from the fitted copula, evaluate the
+pandas-style filter expression, keep the matches, repeat up to
+``max_rounds`` until `n` matches are accumulated. Inefficient for very
+rare conditions (e.g. P(condition) ≈ 0.01) but exact — no need to refit.
+
+For rarer conditions, we could implement true conditional Gaussian
+sampling (condition on a subset of variables, sample from the remaining
+marginal of the multivariate normal), but that's a v0.7 task.
+"""
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+
+import pandas as pd
+
+from .generator.constraints import PhysiologicConstraints
+from .generator.copula import GaussianCopulaGenerator
+
+
+@dataclass
+class ConditionalSamplingResult:
+    rows: pd.DataFrame
+    n_requested: int
+    n_generated: int
+    rounds: int
+    rejection_rate: float
+
+
+# Allowed AST node types in a condition expression. The walker rejects
+# anything that could reach out (Attribute, Call, Subscript, Import) or
+# bind new names (Lambda, comprehensions). Everything left over is just
+# comparisons + boolean ops + arithmetic + literals + column references.
+_ALLOWED_AST_NODES = {
+    ast.Expression, ast.Compare, ast.BoolOp, ast.UnaryOp, ast.BinOp,
+    ast.Name, ast.Constant, ast.Load,
+    ast.And, ast.Or, ast.Not, ast.Invert,
+    ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE,
+    ast.In, ast.NotIn, ast.Is, ast.IsNot,
+    ast.Add, ast.Sub, ast.Mult, ast.Div, ast.Mod, ast.FloorDiv, ast.Pow,
+    ast.BitAnd, ast.BitOr, ast.BitXor, ast.LShift, ast.RShift,
+    ast.USub, ast.UAdd,
+    ast.Tuple, ast.List,
+}
+
+
+def _safe_eval_filter(df: pd.DataFrame, expression: str) -> pd.Series:
+    """Apply a pandas-query-style filter on `df`, AST-validated.
+
+    We parse the expression with ``ast.parse(mode="eval")`` and walk the
+    tree rejecting any node not on the allowlist. This catches attribute
+    access (``age.__class__``), function calls (``__import__('os')``),
+    subscripts (``a[0]``), lambdas, comprehensions, named expressions
+    (``:=``), starred expressions, and anything else that could escape
+    the DataFrame namespace. Names are further restricted to actual
+    DataFrame columns.
+    """
+    try:
+        tree = ast.parse(expression, mode="eval")
+    except SyntaxError as e:
+        raise ValueError(f"invalid condition syntax: {e}") from e
+
+    for node in ast.walk(tree):
+        if type(node) not in _ALLOWED_AST_NODES:
+            raise ValueError(
+                f"condition contains disallowed construct "
+                f"{type(node).__name__!r}; only comparisons, boolean ops, "
+                f"arithmetic, literals, and column-name references are allowed. "
+                f"Got: {expression!r}"
+            )
+        if isinstance(node, ast.Name) and node.id not in df.columns:
+            raise ValueError(
+                f"condition references unknown name {node.id!r}; "
+                f"available columns: {list(df.columns)[:8]}…"
+            )
+    mask = df.eval(expression, engine="python")
+    if not isinstance(mask, pd.Series):
+        raise ValueError(
+            f"condition {expression!r} did not evaluate to a Series"
+        )
+    # Accept both numpy bool and pandas nullable boolean. The copula casts
+    # some columns to Int64 (nullable) which propagates to boolean masks.
+    if mask.dtype.name not in {"bool", "boolean"}:
+        raise ValueError(
+            f"condition {expression!r} did not evaluate to a boolean mask "
+            f"(got dtype {mask.dtype.name})"
+        )
+    # Convert nullable boolean to plain numpy bool, treating NA as False so
+    # downstream df[mask] indexing works without choking.
+    return mask.fillna(False).astype(bool)
+
+
+def sample_conditional(
+    generator: GaussianCopulaGenerator,
+    n: int,
+    condition: str,
+    *,
+    oversample_factor: float = 5.0,
+    max_rounds: int = 10,
+    constraints: PhysiologicConstraints | None = None,
+) -> ConditionalSamplingResult:
+    """Rejection-sample n rows that satisfy ``condition``.
+
+    Parameters
+    ----------
+    generator:
+        A fitted GaussianCopulaGenerator.
+    n:
+        Target number of accepted rows.
+    condition:
+        A pandas df.eval-style expression in the model's column names.
+        Example: ``"age > 60 & DM_Tum == 1 & bp_systolic >= 140"``
+    oversample_factor:
+        Initial multiplier — how many candidates to draw per accepted
+        target. 5× is a reasonable default for moderately selective
+        filters (P ≥ 0.05). Raise for rarer conditions.
+    max_rounds:
+        Safety cap. After this many rounds without filling the quota,
+        the function returns what it has and reports the rejection rate.
+    constraints:
+        Optional PhysiologicConstraints to apply before the user filter.
+
+    Returns
+    -------
+    ConditionalSamplingResult with the accepted rows, the round count,
+    and the empirical rejection rate (so users can see how rare their
+    condition is in this copula).
+    """
+    collected: list[pd.DataFrame] = []
+    rounds = 0
+    total_drawn = 0
+    deficit = n
+
+    while sum(len(d) for d in collected) < n and rounds < max_rounds:
+        rounds += 1
+        batch_size = max(1, int(deficit * oversample_factor))
+        drawn = generator.sample(batch_size)
+        total_drawn += len(drawn)
+
+        if constraints is not None:
+            kept, _ = constraints.apply(drawn)
+        else:
+            kept = drawn
+
+        mask = _safe_eval_filter(kept, condition)
+        matches = kept[mask].reset_index(drop=True)
+        collected.append(matches)
+
+        accumulated = sum(len(d) for d in collected)
+        deficit = max(0, n - accumulated)
+        if deficit > 0:
+            # Update oversample_factor: if we got <50% of what we hoped for,
+            # double the next batch to converge faster.
+            empirical_rate = accumulated / total_drawn
+            if empirical_rate > 0:
+                oversample_factor = max(oversample_factor, 1.5 / empirical_rate)
+
+    final = pd.concat(collected, ignore_index=True).head(n).reset_index(drop=True)
+    rejection_rate = 1.0 - (len(final) / total_drawn) if total_drawn else 1.0
+    return ConditionalSamplingResult(
+        rows=final,
+        n_requested=n,
+        n_generated=len(final),
+        rounds=rounds,
+        rejection_rate=rejection_rate,
+    )

syntha/data.py

@@ -0,0 +1,30 @@
+"""CSV loader for pristine-episode source files."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+
+from . import schema
+
+
+def load_episodes(path: str | Path) -> pd.DataFrame:
+    """Read a pristine-episodes CSV. Handles BOM-prefixed first column."""
+    df = pd.read_csv(path, encoding="utf-8-sig", low_memory=False)
+    df.columns = [c.lstrip("").strip() for c in df.columns]
+    return df
+
+
+def filter_to_modeled(df: pd.DataFrame) -> pd.DataFrame:
+    """Keep only the columns we know how to model. Unknown columns are dropped."""
+    keep = [c for c in schema.all_modeled_columns() if c in df.columns]
+    return df[keep].copy()
+
+
+def date_range(df: pd.DataFrame) -> tuple[pd.Timestamp, pd.Timestamp]:
+    if "episode_date" not in df.columns:
+        return pd.Timestamp("2015-01-01"), pd.Timestamp("2024-12-31")
+    s = pd.to_datetime(df["episode_date"], errors="coerce").dropna()
+    if s.empty:
+        return pd.Timestamp("2015-01-01"), pd.Timestamp("2024-12-31")
+    return s.min(), s.max()

syntha/export_model.py

@@ -0,0 +1,50 @@
+"""Export a trained copula model to a compact JSON the Tauri app consumes.
+
+We downsample each continuous marginal to ``n_quantiles`` order statistics to
+keep the JSON small (≈100 KB for the tolerant cohort at 200 quantiles), then
+serialize the correlation matrix and binary marginals verbatim.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+
+from .generator.copula import GaussianCopulaGenerator
+
+
+def _quantile_grid(values: np.ndarray, n: int) -> list[float]:
+    if len(values) == 0:
+        return [0.0]
+    if len(values) <= n:
+        return [float(v) for v in np.sort(values)]
+    qs = np.linspace(0.0, 1.0, n)
+    return [float(v) for v in np.quantile(values, qs)]
+
+
+def export_model_to_json(
+    gen: GaussianCopulaGenerator,
+    path: str | Path,
+    n_quantiles: int = 200,
+) -> Path:
+    if gen.model is None:
+        raise RuntimeError("generator has no fitted model")
+    m = gen.model
+    payload = {
+        "format": "syntha-copula-v1",
+        "cohort": m.cohort,
+        "columns": list(m.columns),
+        "binary_cols": sorted(m.binary_cols),
+        "p_missing": {c: float(v) for c, v in m.p_missing.items()},
+        "binary_p": {c: float(v) for c, v in m.binary_p.items()},
+        "continuous_quantiles": {
+            c: _quantile_grid(q, n_quantiles) for c, q in m.continuous_quantiles.items()
+        },
+        "correlation": m.correlation.tolist(),
+        "n_train": m.n_train,
+    }
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+    return out

syntha/fhir/__init__.py

@@ -0,0 +1,3 @@
+from .export import episode_to_bundle, write_fhir_bundles
+
+__all__ = ["write_fhir_bundles", "episode_to_bundle"]