geniekit 0.0.1__py3-none-any.whl

geniekit/__init__.py

@@ -0,0 +1,30 @@
+"""geniekit: tools for the AACR Project GENIE BPC clinicogenomic cohorts.
+
+Three capabilities, one package:
+
+* ``geniekit.harmonize``: normalise the six public BPC cohorts (NSCLC, CRC,
+  BrCa, PANC, Prostate, BLADDER) to one schema for cross-cohort analysis.
+* ``geniekit.lot``: derive lines of therapy from the regimen data relative to a
+  user-chosen index event.
+* ``geniekit.aggregate``: an aggregate-only analytic layer with disclosure
+  control for users who may not see patient-level records.
+
+Supporting modules: ``geniekit.releases`` (registry of public releases and
+Synapse ids), ``geniekit.dictionary`` (compiled variable synopses),
+``geniekit.io`` (download and load), ``geniekit.synthetic`` (synthetic cohorts
+for tests and demos).
+
+The package ships metadata only. Patient-level GENIE data must be obtained
+from Synapse under the GENIE access requirement.
+"""
+
+from ._version import __version__
+from .releases import COHORTS, get_release, list_releases, normalize_cohort
+
+__all__ = [
+    "COHORTS",
+    "__version__",
+    "get_release",
+    "list_releases",
+    "normalize_cohort",
+]

geniekit/_version.py

@@ -0,0 +1,8 @@
+"""Single source of truth for the package version.
+
+Keep in sync with ``pyproject.toml``. Bump according to semantic versioning:
+MAJOR for breaking API changes, MINOR for backward-compatible features, PATCH
+for backward-compatible fixes.
+"""
+
+__version__ = "0.0.1"

geniekit/cli.py

@@ -0,0 +1,200 @@
+"""Command line interface: ``geniekit <command> [options]``.
+
+Commands: ``releases``, ``dictionary``, ``access``, ``download``, ``inspect``.
+``harmonize``, ``lot`` and ``aggregate`` are added with their modules.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+import pandas as pd
+
+from . import dictionary, io, releases
+from ._version import __version__
+
+
+def _print_frame(frame: pd.DataFrame, max_rows: int | None = None) -> None:
+    with pd.option_context("display.max_rows", max_rows, "display.max_columns", None, "display.width", 200):
+        print(frame.to_string(index=False) if not frame.empty else "(no rows)")
+
+
+def cmd_releases(args: argparse.Namespace) -> int:
+    rels = releases.list_releases(args.cohort)
+    rows = [
+        {
+            "cohort": r.cohort,
+            "version": r.version,
+            "patients": r.n_patients,
+            "institutions": "/".join(r.institutions),
+            "schema_gen": r.schema_generation,
+            "release_folder": r.release_folder,
+            "clinical_files": len(r.clinical),
+            "genomic_files": len(r.genomic),
+        }
+        for r in rels
+    ]
+    _print_frame(pd.DataFrame(rows))
+    if args.files:
+        frame = releases.to_frame()
+        if args.cohort:
+            frame = frame[frame["cohort"] == releases.normalize_cohort(args.cohort)]
+        print()
+        _print_frame(frame[frame["group"].isin(["clinical", "genomic", "documentation"])])
+    print(f"\nregistry generated on {releases.registry_generated_on()}")
+    return 0
+
+
+def cmd_dictionary(args: argparse.Namespace) -> int:
+    dd = dictionary.load()
+    if args.dict_command == "lookup":
+        rows = dd.lookup(args.variable, args.table)
+        if rows.empty:
+            print(f"{args.variable!r} is not declared in any cohort", file=sys.stderr)
+            return 1
+        out = rows[["cohort", "table", "dtype", "n_values", "label"]]
+        _print_frame(out)
+        if args.values:
+            for c, vals in zip(rows["cohort"], rows["values"], strict=True):
+                print(f"\n{c}:")
+                for v in vals:
+                    print(f"  - {v}")
+        return 0
+    if args.dict_command == "diff":
+        kinds = args.kind or None
+        diff = dd.differences(args.table, kinds)
+        _print_frame(diff, max_rows=None)
+        print(f"\n{len(diff)} rows")
+        return 0
+    if args.dict_command == "presence":
+        mat = dd.presence(args.table)
+        if args.only_partial:
+            mat = mat[~mat.all(axis=1)]
+        with pd.option_context("display.max_rows", None, "display.width", 200):
+            print(mat.replace({True: "x", False: "."}).to_string())
+        return 0
+    return 2
+
+
+def cmd_access(args: argparse.Namespace) -> int:
+    try:
+        session = io.SynapseSession(args.token)
+        who = session.whoami()
+        status = session.access_status()
+    except io.SynapseError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    print(f"Synapse user: {who.get('userName')} (owner id {who.get('ownerId')})")
+    print(status.describe())
+    return 0 if status.approved else 2
+
+
+def cmd_download(args: argparse.Namespace) -> int:
+    try:
+        session = io.SynapseSession(args.token)
+        folder = session.download_release(
+            args.cohort,
+            version=args.version,
+            dest=args.dest,
+            groups=args.groups,
+            names=args.tables or None,
+            overwrite=args.overwrite,
+            progress=print,
+        )
+    except (io.SynapseError, KeyError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    print(f"release folder: {folder}")
+    return 0
+
+
+def cmd_inspect(args: argparse.Namespace) -> int:
+    try:
+        data = io.load_release(
+            args.path, cohort=args.cohort, version=args.version, genomic=not args.no_genomic
+        )
+    except (FileNotFoundError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    print(data)
+    _print_frame(data.summary())
+    report = data.validation.get("columns") or {}
+    if report:
+        print("\ncolumn differences versus the packaged dictionary:")
+        for table, r in report.items():
+            print(f"  {table}: missing {len(r['missing'])}, unexpected {len(r['extra'])}")
+            if r["extra"]:
+                print("    unexpected: " + ", ".join(r["extra"][:15]))
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="geniekit", description=__doc__.splitlines()[0])
+    p.add_argument("--version", action="version", version=f"geniekit {__version__}")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    pr = sub.add_parser("releases", help="list the known public BPC releases")
+    pr.add_argument("--cohort", help="restrict to one cohort")
+    pr.add_argument(
+        "--files", action="store_true", help="also list clinical, genomic and documentation files"
+    )
+    pr.set_defaults(func=cmd_releases)
+
+    pdict = sub.add_parser("dictionary", help="query the compiled variable synopses")
+    dsub = pdict.add_subparsers(dest="dict_command", required=True)
+    lk = dsub.add_parser("lookup", help="show how a variable is declared in each cohort")
+    lk.add_argument("variable")
+    lk.add_argument("--table", choices=dictionary.TABLES)
+    lk.add_argument("--values", action="store_true", help="print permitted values per cohort")
+    df = dsub.add_parser("diff", help="list variables not declared identically across cohorts")
+    df.add_argument("--table", choices=dictionary.TABLES)
+    df.add_argument(
+        "--kind",
+        action="append",
+        choices=["missing_in_some", "dtype", "value_set", "value_text", "label"],
+        help="restrict to one or more kinds of difference (repeatable)",
+    )
+    pres = dsub.add_parser("presence", help="presence matrix of variables by cohort")
+    pres.add_argument("--table", choices=dictionary.TABLES)
+    pres.add_argument("--only-partial", action="store_true", help="hide variables present in every cohort")
+    pdict.set_defaults(func=cmd_dictionary)
+
+    pa = sub.add_parser("access", help="check the Synapse token and GENIE access status")
+    pa.add_argument("--token", help="Synapse personal access token (default: SYNAPSE_AUTH_TOKEN)")
+    pa.set_defaults(func=cmd_access)
+
+    pdl = sub.add_parser("download", help="download a public BPC release from Synapse")
+    pdl.add_argument("cohort")
+    pdl.add_argument("--version", help="release version, e.g. v2.0-public (default: latest public)")
+    pdl.add_argument("--dest", help="destination directory (default: $GENIEKIT_DATA_DIR or ./data/genie_bpc)")
+    pdl.add_argument(
+        "--groups",
+        nargs="+",
+        default=["clinical", "genomic"],
+        choices=list(releases.FILE_GROUPS),
+        help="file groups to download",
+    )
+    pdl.add_argument("--tables", nargs="+", help="restrict to these logical table names")
+    pdl.add_argument("--overwrite", action="store_true", help="re-download files that already exist")
+    pdl.add_argument("--token", help="Synapse personal access token (default: SYNAPSE_AUTH_TOKEN)")
+    pdl.set_defaults(func=cmd_download)
+
+    pi = sub.add_parser("inspect", help="load a release folder and summarise its tables")
+    pi.add_argument("path")
+    pi.add_argument("--cohort")
+    pi.add_argument("--version")
+    pi.add_argument("--no-genomic", action="store_true", help="skip genomic files")
+    pi.set_defaults(func=cmd_inspect)
+    return p
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return int(args.func(args))
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

geniekit/dictionary.py

@@ -0,0 +1,317 @@
+"""Machine-readable data dictionary for the six public GENIE BPC cohorts.
+
+AACR publishes a Variable Synopsis workbook with every public BPC release. The
+six workbooks are compiled by ``data-raw/build_dictionary.py`` into one long
+table shipped as ``resources/dictionary.csv.gz``, with one row per
+(cohort, table, variable). This module exposes that table and the
+cross-cohort comparisons that motivate the harmonisation rules:
+
+* which variables exist in which cohorts (``presence``),
+* variables whose declared type differs across cohorts,
+* variables whose permitted value sets differ across cohorts.
+
+The dictionary contains no patient-level data.
+
+Example
+-------
+>>> from geniekit import dictionary
+>>> dd = dictionary.load()
+>>> dd.compare("ca_cigarette", table="cancer").present_in
+('BrCa', 'PANC', 'Prostate', 'BLADDER')
+>>> dd.differences(table="patient").kind.value_counts()  # doctest: +SKIP
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from functools import lru_cache
+from importlib import resources as importlib_resources
+
+import pandas as pd
+
+from .releases import COHORTS, normalize_cohort
+
+__all__ = [
+    "TABLES",
+    "DataDictionary",
+    "VariableComparison",
+    "load",
+    "normalize_level",
+]
+
+#: Logical table names in the order they appear in the synopses.
+TABLES: tuple[str, ...] = (
+    "patient",
+    "cancer",
+    "regimen",
+    "imaging",
+    "pathology",
+    "med_onc",
+    "tumor_marker",
+    "radiation",
+    "cpt",
+)
+
+# hyphen, non-breaking hyphen, figure dash, en dash, em dash, minus sign
+_DASH_CHARS = "".join(chr(c) for c in (0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2212))
+_DASHES = re.compile("[" + re.escape(_DASH_CHARS) + "]")
+_WS = re.compile(r"\s+")
+
+
+def normalize_level(level: str) -> str:
+    """Cosmetic normalisation of a permitted value for set comparison.
+
+    Lower-cases, collapses whitespace, unifies dash characters and strips
+    trailing punctuation so that ``"Mixed Gliomas"`` and ``"Mixed gliomas"``
+    compare equal while genuinely different levels do not.
+    """
+    s = _DASHES.sub("-", str(level))
+    s = _WS.sub(" ", s).strip().lower()
+    s = s.replace(" - ", "-").replace("- ", "-").replace(" -", "-")
+    return s.rstrip(".;, ")
+
+
+@dataclass(frozen=True)
+class VariableComparison:
+    """How one variable is declared across the cohorts."""
+
+    table: str
+    variable: str
+    present_in: tuple[str, ...]
+    missing_in: tuple[str, ...]
+    labels: dict[str, str]
+    dtypes: dict[str, str]
+    values: dict[str, list[str]]
+
+    @property
+    def is_universal(self) -> bool:
+        return not self.missing_in
+
+    @property
+    def dtype_conflict(self) -> bool:
+        return len(set(self.dtypes.values())) > 1
+
+    @property
+    def label_conflict(self) -> bool:
+        return len({normalize_level(v) for v in self.labels.values()}) > 1
+
+    @property
+    def value_set_conflict(self) -> bool:
+        """True when the permitted value sets differ after cosmetic normalisation."""
+        sets = {frozenset(normalize_level(v) for v in vals) for vals in self.values.values() if vals}
+        return len(sets) > 1
+
+    @property
+    def value_text_conflict(self) -> bool:
+        """True when value sets are equal after normalisation but differ as raw text."""
+        if self.value_set_conflict:
+            return False
+        raw = {tuple(sorted(vals)) for vals in self.values.values() if vals}
+        return len(raw) > 1
+
+    @property
+    def union_values(self) -> list[str]:
+        seen: dict[str, str] = {}
+        for vals in self.values.values():
+            for v in vals:
+                seen.setdefault(normalize_level(v), v)
+        return list(seen.values())
+
+    def summary(self) -> str:
+        parts = [f"{self.table}.{self.variable}: present in {', '.join(self.present_in) or 'none'}"]
+        if self.missing_in:
+            parts.append(f"missing in {', '.join(self.missing_in)}")
+        if self.dtype_conflict:
+            parts.append("dtype conflict " + ", ".join(f"{c}={t}" for c, t in self.dtypes.items()))
+        if self.value_set_conflict:
+            parts.append("value sets differ")
+        elif self.value_text_conflict:
+            parts.append("value text differs (cosmetic)")
+        return "; ".join(parts)
+
+
+class DataDictionary:
+    """The compiled dictionary with cross-cohort comparison helpers."""
+
+    def __init__(self, frame: pd.DataFrame) -> None:
+        required = {"cohort", "version", "schema_generation", "table", "variable", "label", "dtype", "values"}
+        missing = required - set(frame.columns)
+        if missing:
+            raise ValueError(f"dictionary frame lacks columns: {sorted(missing)}")
+        frame = frame.copy()
+        if len(frame) and not isinstance(frame["values"].iloc[0], list):
+            frame["values"] = [json.loads(v) if isinstance(v, str) and v else [] for v in frame["values"]]
+        self._frame = frame.reset_index(drop=True)
+        present = [c for c in COHORTS if c in set(self._frame["cohort"])]
+        self.cohorts: tuple[str, ...] = tuple(present)
+
+    # -- basic access --------------------------------------------------------
+    @property
+    def frame(self) -> pd.DataFrame:
+        """The full long table (one row per cohort, table, variable)."""
+        return self._frame
+
+    def tables(self) -> list[str]:
+        return [t for t in TABLES if t in set(self._frame["table"])]
+
+    def schema_generation(self, cohort: str) -> int:
+        c = normalize_cohort(cohort)
+        gens = self._frame.loc[self._frame["cohort"] == c, "schema_generation"].unique()
+        if len(gens) != 1:
+            raise KeyError(f"cohort {c!r} not in dictionary")
+        return int(gens[0])
+
+    def variables(self, cohort: str | None = None, table: str | None = None) -> pd.DataFrame:
+        """Rows filtered by cohort and/or table."""
+        f = self._frame
+        if cohort is not None:
+            f = f[f["cohort"] == normalize_cohort(cohort)]
+        if table is not None:
+            self._check_table(table)
+            f = f[f["table"] == table]
+        return f.reset_index(drop=True)
+
+    def lookup(self, variable: str, table: str | None = None) -> pd.DataFrame:
+        """All declarations of ``variable`` (across cohorts and, if unspecified, tables)."""
+        f = self._frame[self._frame["variable"] == variable]
+        if table is not None:
+            self._check_table(table)
+            f = f[f["table"] == table]
+        return f.reset_index(drop=True)
+
+    def values(self, variable: str, table: str) -> dict[str, list[str]]:
+        """Permitted values per cohort for one variable."""
+        rows = self.lookup(variable, table)
+        return {c: list(v) for c, v in zip(rows["cohort"], rows["values"], strict=True)}
+
+    # -- comparisons ---------------------------------------------------------
+    def presence(self, table: str | None = None, cohorts: Iterable[str] | None = None) -> pd.DataFrame:
+        """Boolean matrix: rows (table, variable), columns cohorts."""
+        f = self.variables(table=table)
+        cols = tuple(normalize_cohort(c) for c in cohorts) if cohorts else self.cohorts
+        f = f[f["cohort"].isin(cols)]
+        mat = f.assign(flag=True).pivot_table(
+            index=["table", "variable"], columns="cohort", values="flag", aggfunc="any", fill_value=False
+        )
+        mat = mat.reindex(columns=list(cols), fill_value=False).astype(bool)
+        mat.columns.name = None
+        # keep synopsis order of first appearance
+        order = f.drop_duplicates(["table", "variable"])[["table", "variable"]]
+        return mat.reindex(pd.MultiIndex.from_frame(order))
+
+    def common_variables(self, table: str, cohorts: Iterable[str] | None = None) -> list[str]:
+        """Variables present in every requested cohort for ``table``."""
+        mat = self.presence(table, cohorts)
+        return [v for (_, v), row in mat.iterrows() if bool(row.all())]
+
+    def compare(self, variable: str, table: str) -> VariableComparison:
+        rows = self.lookup(variable, table)
+        if rows.empty:
+            raise KeyError(f"{table}.{variable} is not declared in any cohort")
+        by = {c: r for c, r in zip(rows["cohort"], rows.to_dict("records"), strict=True)}
+        present = tuple(c for c in self.cohorts if c in by)
+        return VariableComparison(
+            table=table,
+            variable=variable,
+            present_in=present,
+            missing_in=tuple(c for c in self.cohorts if c not in by),
+            labels={c: str(by[c]["label"]) for c in present},
+            dtypes={c: str(by[c]["dtype"]) for c in present},
+            values={c: list(by[c]["values"]) for c in present},
+        )
+
+    def differences(self, table: str | None = None, kinds: Iterable[str] | None = None) -> pd.DataFrame:
+        """Every variable that is not declared identically in all cohorts.
+
+        Returns one row per (table, variable, kind) with ``kind`` in
+        ``{"missing_in_some", "dtype", "value_set", "value_text", "label"}``
+        and a human-readable ``detail`` column.
+        """
+        wanted = set(kinds) if kinds else {"missing_in_some", "dtype", "value_set", "value_text", "label"}
+        f = self.variables(table=table)
+        rows: list[dict] = []
+        for (tbl, var), _ in f.groupby(["table", "variable"], sort=False):
+            cmp = self.compare(var, tbl)
+            if "missing_in_some" in wanted and cmp.missing_in:
+                rows.append(
+                    {
+                        "table": tbl,
+                        "variable": var,
+                        "kind": "missing_in_some",
+                        "present_in": ",".join(cmp.present_in),
+                        "detail": f"missing in {', '.join(cmp.missing_in)}",
+                    }
+                )
+            if "dtype" in wanted and cmp.dtype_conflict:
+                rows.append(
+                    {
+                        "table": tbl,
+                        "variable": var,
+                        "kind": "dtype",
+                        "present_in": ",".join(cmp.present_in),
+                        "detail": ", ".join(f"{c}={t}" for c, t in cmp.dtypes.items()),
+                    }
+                )
+            if "value_set" in wanted and cmp.value_set_conflict:
+                union = {normalize_level(v) for vals in cmp.values.values() for v in vals}
+                lacks = {
+                    c: sorted(union - {normalize_level(v) for v in vals})
+                    for c, vals in cmp.values.items()
+                    if vals
+                }
+                rows.append(
+                    {
+                        "table": tbl,
+                        "variable": var,
+                        "kind": "value_set",
+                        "present_in": ",".join(cmp.present_in),
+                        "detail": "; ".join(
+                            f"{c} lacks {len(missing)}" for c, missing in lacks.items() if missing
+                        ),
+                    }
+                )
+            if "value_text" in wanted and cmp.value_text_conflict:
+                rows.append(
+                    {
+                        "table": tbl,
+                        "variable": var,
+                        "kind": "value_text",
+                        "present_in": ",".join(cmp.present_in),
+                        "detail": "same levels, different spelling/case",
+                    }
+                )
+            if "label" in wanted and cmp.label_conflict:
+                rows.append(
+                    {
+                        "table": tbl,
+                        "variable": var,
+                        "kind": "label",
+                        "present_in": ",".join(cmp.present_in),
+                        "detail": " | ".join(sorted({v for v in cmp.labels.values()}))[:200],
+                    }
+                )
+        return pd.DataFrame(rows, columns=["table", "variable", "kind", "present_in", "detail"])
+
+    # -- helpers -------------------------------------------------------------
+    def _check_table(self, table: str) -> None:
+        if table not in TABLES:
+            raise ValueError(f"Unknown table {table!r}; expected one of {', '.join(TABLES)}")
+
+    def __repr__(self) -> str:
+        n_vars = self._frame.drop_duplicates(["table", "variable"]).shape[0]
+        return f"DataDictionary({len(self._frame)} rows, {n_vars} variables, cohorts={list(self.cohorts)})"
+
+
+@lru_cache(maxsize=1)
+def load() -> DataDictionary:
+    """Load the packaged dictionary (cached)."""
+    path = importlib_resources.files("geniekit.resources").joinpath("dictionary.csv.gz")
+    with path.open("rb") as fh:
+        frame = pd.read_csv(fh, compression="gzip", dtype=str, keep_default_na=False)
+    frame["schema_generation"] = frame["schema_generation"].astype(int)
+    frame["n_values"] = frame["n_values"].astype(int)
+    frame["order"] = frame["order"].astype(int)
+    return DataDictionary(frame)

geniekit/io/__init__.py

@@ -0,0 +1,36 @@
+"""Loading GENIE BPC releases: from Synapse (with a personal access token) or
+from files already on disk. Both paths produce a :class:`CohortData`.
+
+>>> from geniekit import io
+>>> data = io.load_release("data/genie_bpc/CRC_v2.0-public")   # doctest: +SKIP
+>>> data["patient"].shape                                        # doctest: +SKIP
+"""
+
+from .cohort import CLINICAL_TABLES, GENOMIC_TABLES, CohortData
+from .local import FILENAME_TO_TABLE, find_release_files, load_release, read_table
+from .synapse import (
+    GENIE_ACCESS_REQUIREMENT,
+    AccessDeniedError,
+    AccessStatus,
+    SynapseError,
+    SynapseSession,
+    download_release,
+    resolve_token,
+)
+
+__all__ = [
+    "CLINICAL_TABLES",
+    "FILENAME_TO_TABLE",
+    "GENIE_ACCESS_REQUIREMENT",
+    "GENOMIC_TABLES",
+    "AccessDeniedError",
+    "AccessStatus",
+    "CohortData",
+    "SynapseError",
+    "SynapseSession",
+    "download_release",
+    "find_release_files",
+    "load_release",
+    "read_table",
+    "resolve_token",
+]