jpomop 0.0.1__py3-none-any.whl

jpomop/__init__.py

@@ -0,0 +1,3 @@
+"""Convert Japanese DPC inpatient data to the OMOP Common Data Model v5.4."""
+
+__version__ = "0.0.1"

jpomop/cli.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+from collections import Counter
+from pathlib import Path
+
+import click
+import pyarrow as pa
+import pyarrow.csv as pacsv
+
+from jpomop import __version__
+from jpomop.parse import (
+    Violation,
+    check_cross_file,
+    pivot_yoshiki1,
+    read_ef,
+    read_yoshiki1,
+    violations_table,
+)
+
+_existing = click.Path(exists=True, dir_okay=False, path_type=Path)
+
+
+@click.group()
+@click.version_option(__version__)
+def main() -> None:
+    """jpomop: DPC to OMOP CDM v5.4."""
+
+
+@main.command()
+@click.option("--yoshiki1", "y1_paths", type=_existing, multiple=True, help="Form 1 file(s).")
+@click.option("--ef", "ef_paths", type=_existing, multiple=True, help="EF integrated file(s).")
+@click.option(
+    "--out",
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="Write every violation to this CSV file.",
+)
+def validate(y1_paths: tuple[Path, ...], ef_paths: tuple[Path, ...], out: Path | None) -> None:
+    """Check DPC files against the FY2026 specification.
+
+    Exits with status 1 if any violation is found.
+    """
+    if not y1_paths and not ef_paths:
+        raise click.UsageError("give at least one --yoshiki1 or --ef file")
+
+    violations: list[Violation] = []
+    forms = []
+    for p in y1_paths:
+        res = read_yoshiki1(p)
+        tables = pivot_yoshiki1(res.table)
+        violations += res.violations + tables.violations
+        forms.append(tables.forms)
+    efs = []
+    for p in ef_paths:
+        res = read_ef(p)
+        violations += res.violations
+        efs.append(res.table)
+    if forms and efs:
+        violations += check_cross_file(
+            pa.concat_tables(efs),
+            pa.concat_tables(forms),
+            ef_file=ef_paths[0].name,
+            y1_file=y1_paths[0].name,
+        )
+
+    counts = Counter(v.rule for v in violations)
+    if not violations:
+        click.echo("no violations")
+    for rule, n in sorted(counts.items()):
+        click.echo(f"{rule}\t{n}")
+    if out:
+        pacsv.write_csv(violations_table(violations), out)
+        click.echo(f"wrote {len(violations)} rows to {out}")
+    if violations:
+        raise SystemExit(1)
+
+
+@main.command()
+@click.option("--n", "n_patients", default=100, show_default=True, help="Number of patients.")
+@click.option("--seed", default=20261012, show_default=True)
+@click.option("--out", type=click.Path(file_okay=False, path_type=Path), required=True)
+def synth(n_patients: int, seed: int, out: Path) -> None:
+    """Write a synthetic Form 1 and EF file (June 2026). Everything is invented."""
+    from jpomop.synth import generate
+
+    m = generate(out, n_patients, seed=seed)
+    click.echo(f"{m.n_patients} patients, {m.n_stays} stays")
+    click.echo(f"{m.yoshiki1}\n{m.ef}")
+
+
+@main.command()
+@click.option("--yoshiki1", "y1_paths", type=_existing, multiple=True, required=True)
+@click.option("--ef", "ef_paths", type=_existing, multiple=True, required=True)
+@click.option(
+    "--out",
+    type=click.Path(path_type=Path),
+    required=True,
+    help="DuckDB file, or a directory with --format parquet.",
+)
+@click.option(
+    "--format", "fmt", type=click.Choice(["duckdb", "parquet"]), default="duckdb", show_default=True
+)
+@click.option(
+    "--vocab",
+    "vocab_path",
+    type=_existing,
+    help="vocab.duckdb from `jpomop vocab build`. Without it, concept IDs are 0.",
+)
+def convert(
+    y1_paths: tuple[Path, ...],
+    ef_paths: tuple[Path, ...],
+    out: Path,
+    fmt: str,
+    vocab_path: Path | None,
+) -> None:
+    """Convert DPC files to OMOP CDM v5.4 tables."""
+    from jpomop.etl import run_etl
+
+    report = run_etl(
+        list(y1_paths),
+        list(ef_paths),
+        out,
+        fmt,  # type: ignore[arg-type]
+        vocab=vocab_path,
+    )
+    for name, n in report.rows.items():
+        click.echo(f"{name}\t{n}")
+    for kind, n in sorted(report.skipped_forms.items()):
+        click.echo(f"skipped forms {kind}\t{n}")
+    for table, reasons in report.skipped_rows.items():
+        for reason, n in sorted(reasons.items()):
+            click.echo(f"left out of {table}: {reason}\t{n}")
+    if report.n_violations:
+        click.echo(f"input violations\t{report.n_violations} (run jpomop validate for details)")
+    for p in report.invariant_problems:
+        click.echo(f"INVARIANT\t{p}", err=True)
+    if report.invariant_problems:
+        raise SystemExit(2)
+
+
+@main.group()
+def vocab() -> None:
+    """Build the local vocabulary from files you download yourself."""
+
+
+@vocab.command("build")
+@click.option(
+    "--athena",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    required=True,
+    help="Folder with CONCEPT.csv and CONCEPT_RELATIONSHIP.csv.",
+)
+@click.option(
+    "--drug-master",
+    type=_existing,
+    multiple=True,
+    required=True,
+    help="医薬品マスター y_*.csv (repeat for several files).",
+)
+@click.option("--kimura", type=_existing, help="Kimura et al. 2025 mapping (rxnorm_mapping.tsv).")
+@click.option("--out", type=click.Path(dir_okay=False, path_type=Path), required=True)
+def vocab_build(
+    athena: Path, drug_master: tuple[Path, ...], kimura: Path | None, out: Path
+) -> None:
+    from jpomop.vocab import build_vocab
+
+    r = build_vocab(athena, list(drug_master), kimura, out)
+    click.echo(f"ICD-10 codes\t{r.icd10_codes} ({r.icd10_mapped} map to a standard concept)")
+    click.echo(f"drug receipt codes\t{r.receipt_drugs} ({r.receipt_drugs_mapped} mapped)")
+
+
+@main.command()
+@click.option("--cdm", type=_existing, required=True, help="DuckDB file from jpomop convert.")
+@click.option("--top", default=10, show_default=True, help="Unmapped source codes to list.")
+def report(cdm: Path, top: int) -> None:
+    """Show how much of each domain maps to standard concepts."""
+    from jpomop.report import coverage
+
+    for c in coverage(cdm, top):
+        click.echo(
+            f"{c.table}\trows {c.mapped_rows}/{c.rows} ({c.row_rate:.1%})\t"
+            f"codes {c.mapped_codes}/{c.codes} ({c.code_rate:.1%})"
+        )
+        for code, n in c.top_unmapped:
+            click.echo(f"  unmapped {code}\t{n}")

jpomop/etl/__init__.py

@@ -0,0 +1,130 @@
+"""Turn parsed DPC tables into OMOP CDM v5.4 tables."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+import duckdb
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from jpomop.etl.build import (
+    build_death,
+    build_observation_period,
+    build_person,
+    build_visit_detail,
+    build_visit_occurrence,
+    last_service_dates,
+    person_ids,
+    visit_ids,
+    whole_stay_forms,
+)
+from jpomop.etl.checks import check_invariants
+from jpomop.etl.clinical import (
+    build_condition_occurrence,
+    build_drug_exposure,
+    build_procedure_occurrence,
+)
+from jpomop.parse import pivot_yoshiki1, read_ef, read_yoshiki1
+from jpomop.vocab import Vocab
+
+TABLE_ORDER = [
+    "person",
+    "observation_period",
+    "visit_occurrence",
+    "visit_detail",
+    "condition_occurrence",
+    "procedure_occurrence",
+    "drug_exposure",
+    "death",
+]
+
+
+@dataclass
+class EtlReport:
+    rows: dict[str, int]
+    skipped_forms: dict[str, int] = field(default_factory=dict)
+    skipped_rows: dict[str, dict[str, int]] = field(default_factory=dict)
+    n_violations: int = 0
+    invariant_problems: list[str] = field(default_factory=list)
+
+
+def build_cdm(
+    forms: pa.Table, ef: pa.Table, diagnoses: pa.Table | None = None, vocab: Vocab | None = None
+) -> tuple[dict[str, pa.Table], EtlReport]:
+    vocab = vocab or Vocab()
+    whole, skipped = whole_stay_forms(forms)
+    pids, vids = person_ids(whole), visit_ids(whole)
+    visits = build_visit_occurrence(whole, pids, vids, last_service_dates(ef))
+    cdm = {
+        "person": build_person(whole, pids),
+        "observation_period": build_observation_period(visits),
+        "visit_occurrence": visits,
+        "visit_detail": build_visit_detail(ef, visits, vids),
+        "death": build_death(whole, pids),
+    }
+    skipped_rows: dict[str, dict[str, int]] = {}
+    if diagnoses is not None:
+        cdm["condition_occurrence"], skip = build_condition_occurrence(
+            diagnoses, visits, vids, vocab
+        )
+        skipped_rows["condition_occurrence"] = dict(skip)
+    cdm["drug_exposure"], skip = build_drug_exposure(ef, visits, vids, vocab)
+    skipped_rows["drug_exposure"] = dict(skip)
+    cdm["procedure_occurrence"] = build_procedure_occurrence(ef, visits, vids)
+    report = EtlReport(
+        rows={k: t.num_rows for k, t in cdm.items()},
+        skipped_forms=dict(skipped),
+        skipped_rows=skipped_rows,
+        invariant_problems=check_invariants(cdm),
+    )
+    return cdm, report
+
+
+def write_cdm(
+    cdm: dict[str, pa.Table], out: Path, fmt: Literal["duckdb", "parquet"] = "duckdb"
+) -> None:
+    if fmt == "parquet":
+        out.mkdir(parents=True, exist_ok=True)
+        for name in (n for n in TABLE_ORDER if n in cdm):
+            pq.write_table(cdm[name], out / f"{name}.parquet")
+        return
+    out.unlink(missing_ok=True)
+    con = duckdb.connect(str(out))
+    try:
+        for name in (n for n in TABLE_ORDER if n in cdm):
+            tbl = cdm[name]  # noqa: F841  (read by DuckDB through the local variable)
+            con.execute(f"CREATE TABLE {name} AS SELECT * FROM tbl")
+    finally:
+        con.close()
+
+
+def run_etl(
+    yoshiki1: list[Path],
+    ef: list[Path],
+    out: Path,
+    fmt: Literal["duckdb", "parquet"] = "duckdb",
+    vocab: Path | None = None,
+) -> EtlReport:
+    forms, dxs, efs, n_violations = [], [], [], 0
+    for p in yoshiki1:
+        res = read_yoshiki1(p)
+        tables = pivot_yoshiki1(res.table)
+        forms.append(tables.forms)
+        dxs.append(tables.diagnoses)
+        n_violations += len(res.violations) + len(tables.violations)
+    for p in ef:
+        res = read_ef(p)
+        efs.append(res.table)
+        n_violations += len(res.violations)
+    cdm, report = build_cdm(
+        pa.concat_tables(forms),
+        pa.concat_tables(efs),
+        pa.concat_tables(dxs),
+        Vocab.load(vocab) if vocab else None,
+    )
+    report.n_violations = n_violations
+    write_cdm(cdm, out, fmt)
+    return report

jpomop/etl/build.py

@@ -0,0 +1,232 @@
+"""Build CDM tables from parsed DPC tables.
+
+Keys:
+- patient: (facility_code, data_id)
+- stay:    (facility_code, data_id, admission_date)
+
+Only whole-stay forms (統括診療情報番号 0) become visits. Ward-group
+forms, short readmissions (A, B) and the day-90 interim form (Z) are
+counted by `run_etl` but not converted yet; see docs/adr.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections import Counter, defaultdict
+from datetime import date
+from typing import Any
+
+import pyarrow as pa
+
+from jpomop.etl import concepts as C
+from jpomop.etl import schema as S
+from jpomop.spec import load_code_table
+
+Patient = tuple[str, str]
+Stay = tuple[str, str, date]
+
+
+def source_value(facility_code: str, data_id: str) -> str:
+    """Hash of the patient key, so the hospital's data ID never leaves the ETL."""
+    return hashlib.sha256(f"{facility_code}:{data_id}".encode()).hexdigest()[:16]
+
+
+def whole_stay_forms(forms: pa.Table) -> tuple[list[dict[str, Any]], Counter[str]]:
+    """Pick one form per stay and count what was not used.
+
+    - Whole-stay forms (統括診療情報番号 0) are used.
+    - A day-90 interim form (Z) is used only when the stay has no whole-stay
+      form yet, i.e. the patient was still in hospital when the data was cut.
+      The ward-group forms (1, 2, ...) and the aggregated short-readmission
+      forms (A, B) repeat information already in the whole-stay forms and are
+      not used (p42).
+    - Several one-day stays on the same day (回数管理番号 1, 2, ...) cannot be
+      told apart in the EF file, so they become one visit.
+    """
+    counts: Counter[str] = Counter()
+    by_stay: dict[Stay, dict[str, Any]] = {}
+    interim: dict[Stay, dict[str, Any]] = {}
+    for f in forms.to_pylist():
+        stay = (f["facility_code"], f["data_id"], f["admission_date"])
+        if f["summary_no"] == "0":
+            if stay in by_stay:
+                counts["same_day_stays_merged"] += 1
+                continue
+            by_stay[stay] = f
+        elif f["summary_no"] == "Z":
+            interim.setdefault(stay, f)
+        else:
+            counts[f"summary_no={f['summary_no']}"] += 1
+    for stay, f in interim.items():
+        if stay in by_stay:
+            counts["summary_no=Z"] += 1
+        else:
+            by_stay[stay] = f
+            counts["interim_form_used"] += 1
+    return [by_stay[k] for k in sorted(by_stay)], counts
+
+
+def person_ids(forms: list[dict[str, Any]]) -> dict[Patient, int]:
+    keys = sorted({(f["facility_code"], f["data_id"]) for f in forms})
+    return {k: i for i, k in enumerate(keys, start=1)}
+
+
+def visit_ids(forms: list[dict[str, Any]]) -> dict[Stay, int]:
+    keys = sorted({(f["facility_code"], f["data_id"], f["admission_date"]) for f in forms})
+    return {k: i for i, k in enumerate(keys, start=1)}
+
+
+def build_person(forms: list[dict[str, Any]], pids: dict[Patient, int]) -> pa.Table:
+    first: dict[Patient, dict[str, Any]] = {}
+    for f in forms:  # forms are sorted, so the first form per patient is the earliest stay
+        first.setdefault((f["facility_code"], f["data_id"]), f)
+    rows = []
+    for key, pid in sorted(pids.items(), key=lambda kv: kv[1]):
+        f = first[key]
+        b: date | None = f["birth_date"]
+        rows.append(
+            {
+                "person_id": pid,
+                "gender_concept_id": C.GENDER.get(f["sex"] or "", C.UNKNOWN),
+                "year_of_birth": b.year if b else None,
+                "month_of_birth": b.month if b else None,
+                "day_of_birth": b.day if b else None,
+                "race_concept_id": C.UNKNOWN,
+                "ethnicity_concept_id": C.UNKNOWN,
+                "person_source_value": source_value(*key),
+                "gender_source_value": f["sex"],
+            }
+        )
+    return pa.Table.from_pylist(rows, schema=S.PERSON)
+
+
+def last_service_dates(ef: pa.Table) -> dict[Stay, date]:
+    last: dict[Stay, date] = {}
+    for r in ef.select(["facility_code", "data_id", "admission_date", "exec_date"]).to_pylist():
+        if r["exec_date"] is None:
+            continue
+        stay = (r["facility_code"], r["data_id"], r["admission_date"])
+        last[stay] = max(last.get(stay, r["exec_date"]), r["exec_date"])
+    return last
+
+
+def build_visit_occurrence(
+    forms: list[dict[str, Any]],
+    pids: dict[Patient, int],
+    vids: dict[Stay, int],
+    last_service: dict[Stay, date] | None = None,
+) -> pa.Table:
+    """Visit end is the discharge date. A stay with no discharge date yet (an
+    interim form) ends on its last date of service in the EF file, or on the
+    admission date if there is none."""
+    last_service = last_service or {}
+    rows = []
+    for f in forms:
+        stay = (f["facility_code"], f["data_id"], f["admission_date"])
+        end = f["discharge_date"] or last_service.get(stay) or f["admission_date"]
+        rows.append(
+            {
+                "visit_occurrence_id": vids[stay],
+                "person_id": pids[stay[:2]],
+                "visit_concept_id": C.INPATIENT_VISIT,
+                "visit_start_date": f["admission_date"],
+                "visit_end_date": end,
+                "visit_type_concept_id": C.TYPE_CLAIM,
+                "visit_source_value": "dpc_yoshiki1",
+                "visit_source_concept_id": C.UNKNOWN,
+                "admitted_from_concept_id": C.UNKNOWN,
+                "admitted_from_source_value": f["admission_route"],
+                "discharged_to_concept_id": C.UNKNOWN,
+                "discharged_to_source_value": f["discharge_to"],
+            }
+        )
+    return pa.Table.from_pylist(rows, schema=S.VISIT_OCCURRENCE)
+
+
+def build_visit_detail(ef: pa.Table, visits: pa.Table, vids: dict[Stay, int]) -> pa.Table:
+    """One row per run of consecutive service dates on the same ward."""
+    vinfo = {r["visit_occurrence_id"]: r for r in visits.to_pylist()}
+    days: dict[int, dict[date, str]] = defaultdict(dict)
+    for r in ef.select(
+        ["facility_code", "data_id", "admission_date", "exec_date", "ward_code"]
+    ).to_pylist():
+        vid = vids.get((r["facility_code"], r["data_id"], r["admission_date"]))
+        if vid is None or r["exec_date"] is None:
+            continue
+        days[vid].setdefault(r["exec_date"], r["ward_code"])
+    rows = []
+    next_id = 1
+    for vid in sorted(days):
+        v = vinfo[vid]
+        runs: list[list[Any]] = []  # [ward, start, end]
+        for d in sorted(days[vid]):
+            ward = days[vid][d]
+            if runs and runs[-1][0] == ward:
+                runs[-1][2] = d
+            else:
+                runs.append([ward, d, d])
+        prev = None
+        for ward, start, end in runs:
+            start = max(start, v["visit_start_date"])
+            end = min(end, v["visit_end_date"])
+            rows.append(
+                {
+                    "visit_detail_id": next_id,
+                    "person_id": v["person_id"],
+                    "visit_detail_concept_id": C.INPATIENT_VISIT,
+                    "visit_detail_start_date": start,
+                    "visit_detail_end_date": end,
+                    "visit_detail_type_concept_id": C.TYPE_CLAIM,
+                    "visit_detail_source_value": ward,
+                    "visit_detail_source_concept_id": C.UNKNOWN,
+                    "admitted_from_concept_id": C.UNKNOWN,
+                    "discharged_to_concept_id": C.UNKNOWN,
+                    "preceding_visit_detail_id": prev,
+                    "visit_occurrence_id": vid,
+                }
+            )
+            prev = next_id
+            next_id += 1
+    return pa.Table.from_pylist(rows, schema=S.VISIT_DETAIL)
+
+
+def build_observation_period(visits: pa.Table) -> pa.Table:
+    span: dict[int, list[date]] = {}
+    for v in visits.to_pylist():
+        s, e = v["visit_start_date"], v["visit_end_date"]
+        if v["person_id"] in span:
+            span[v["person_id"]][0] = min(span[v["person_id"]][0], s)
+            span[v["person_id"]][1] = max(span[v["person_id"]][1], e)
+        else:
+            span[v["person_id"]] = [s, e]
+    rows = [
+        {
+            "observation_period_id": pid,
+            "person_id": pid,
+            "observation_period_start_date": s,
+            "observation_period_end_date": e,
+            "period_type_concept_id": C.TYPE_CLAIM,
+        }
+        for pid, (s, e) in sorted(span.items())
+    ]
+    return pa.Table.from_pylist(rows, schema=S.OBSERVATION_PERIOD)
+
+
+def build_death(forms: list[dict[str, Any]], pids: dict[Patient, int]) -> pa.Table:
+    """A stay whose discharge outcome (退院時転帰) is death (6 or 7). Latest one per person."""
+    death_codes = set(load_code_table("y1_discharge_outcome")["death"])
+    latest: dict[int, date] = {}
+    for f in forms:
+        if f["discharge_outcome"] in death_codes and f["discharge_date"]:
+            pid = pids[(f["facility_code"], f["data_id"])]
+            latest[pid] = max(latest.get(pid, f["discharge_date"]), f["discharge_date"])
+    rows = [
+        {
+            "person_id": pid,
+            "death_date": d,
+            "death_type_concept_id": C.TYPE_CLAIM,
+            "cause_concept_id": C.UNKNOWN,
+        }
+        for pid, d in sorted(latest.items())
+    ]
+    return pa.Table.from_pylist(rows, schema=S.DEATH)

jpomop/etl/checks.py

@@ -0,0 +1,63 @@
+"""Invariants every CDM output must satisfy. Each returns a list of problems."""
+
+from __future__ import annotations
+
+from collections import Counter
+
+import pyarrow as pa
+
+
+def check_invariants(cdm: dict[str, pa.Table]) -> list[str]:
+    problems: list[str] = []
+    person_ids = cdm["person"].column("person_id").to_pylist()
+    dup = [p for p, n in Counter(person_ids).items() if n > 1]
+    if dup:
+        problems.append(f"person: duplicate person_id {dup[:5]}")
+    known = set(person_ids)
+
+    periods: dict[int, tuple[object, object]] = {}
+    for r in cdm["observation_period"].to_pylist():
+        if r["person_id"] in periods:
+            problems.append(f"observation_period: more than one period for {r['person_id']}")
+        periods[r["person_id"]] = (
+            r["observation_period_start_date"],
+            r["observation_period_end_date"],
+        )
+
+    events = {
+        "visit_occurrence": ("visit_start_date", "visit_end_date"),
+        "visit_detail": ("visit_detail_start_date", "visit_detail_end_date"),
+        "death": ("death_date", "death_date"),
+        "condition_occurrence": ("condition_start_date", "condition_start_date"),
+        "drug_exposure": ("drug_exposure_start_date", "drug_exposure_end_date"),
+        "procedure_occurrence": ("procedure_date", "procedure_date"),
+    }
+    for table, (start_col, end_col) in events.items():
+        if table not in cdm:
+            continue
+        for r in cdm[table].to_pylist():
+            pid, s, e = r["person_id"], r[start_col], r[end_col]
+            if pid not in known:
+                problems.append(f"{table}: person_id {pid} not in person")
+                continue
+            if s > e:
+                problems.append(f"{table}: start after end for person {pid}")
+            ps, pe = periods.get(pid, (None, None))
+            if ps is None or s < ps or e > pe:  # type: ignore[operator]
+                problems.append(f"{table}: outside observation_period for person {pid}")
+
+    visits = {r["visit_occurrence_id"]: r for r in cdm["visit_occurrence"].to_pylist()}
+    died = {r["person_id"]: r["death_date"] for r in cdm["death"].to_pylist()}
+    for v in visits.values():
+        if v["person_id"] in died and v["visit_start_date"] > died[v["person_id"]]:
+            problems.append(f"visit_occurrence: stay starts after death for {v['person_id']}")
+    for r in cdm["visit_detail"].to_pylist():
+        v = visits.get(r["visit_occurrence_id"])
+        if v is None:
+            problems.append(f"visit_detail {r['visit_detail_id']}: unknown visit")
+        elif not (
+            v["visit_start_date"] <= r["visit_detail_start_date"]
+            and r["visit_detail_end_date"] <= v["visit_end_date"]
+        ):
+            problems.append(f"visit_detail {r['visit_detail_id']}: outside its visit")
+    return problems