bpskit 0.2.0__py3-none-any.whl

bpskit/__init__.py

@@ -0,0 +1,14 @@
+"""Shared tools for studies on a BPS (Base Poblacional de Salud, SAS) delivery.
+
+    bpskit.delivery    read the pipe-delimited delivery: sentinels, dates, streaming
+    bpskit.icd         ICD-10 code specs and chapters, ICD-9 -> ICD-10 mapping (CMS GEM)
+    bpskit.persontime  dates as numbers, age bands, Lexis splitting, person streaming
+    bpskit.stats       exact Poisson and binomial tests, BH / Bonferroni
+    bpskit.transfer    the check that no file leaving the data machine names a patient
+    bpskit.synthetic   a synthetic BPS delivery with planted structure, for tests
+
+The command line (`bpskit pullable`, `bpskit check-logs`) runs the transfer
+check on the data machine.
+"""
+
+__version__ = "0.2.0"

bpskit/cli.py

@@ -0,0 +1,102 @@
+"""The transfer checks, run on the data machine before anything is copied off it.
+
+    bpskit pullable [--strict] <path> ...   exit 0 only if no file names a patient
+    bpskit check-logs [workflow_dir]        list the logs that may leave
+    bpskit synthetic <out_dir>              write a synthetic delivery (bpskit.synthetic)
+
+pullable and check-logs run from the workflow directory and read its config
+(default config/config.yaml) for the delivery's id columns and the id files
+under results/work/ids/.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from bpskit import transfer
+from bpskit.delivery import load_config
+
+
+def pullable(args: argparse.Namespace) -> int:
+    root = Path.cwd()
+    cfg = load_config(root / args.config)
+    ids = transfer.ids_for_root(root, cfg)
+    n, problems = transfer.check(args.paths, ids, args.strict, transfer.forbidden_columns(cfg))
+    print(f"[CHECK] {n} file(s), {len(problems)} problem(s)")
+    for p in problems:
+        print(f"[REFUSE] {p}")
+    if problems:
+        print("No transfer: these files stay on this machine.")
+        return 1
+    print("[PASS ] no file names a patient")
+    return 0
+
+
+def check_logs(args: argparse.Namespace) -> int:
+    """Write the clean logs to results/work/logs_pullable.txt (which sync.sh copies).
+
+    A traceback can print a data row, so a log containing an id is WITHHELD,
+    not redacted. The list is emptied first, so any failure leaves nothing listed.
+    """
+    root = Path(args.workflow_dir).resolve()
+    out = root / "results/work/logs_pullable.txt"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text("")
+    ids = transfer.ids_for_root(root, load_config(root / args.config))
+    candidates = sorted(
+        list((root / "results/work/logs").rglob("*.log"))
+        + list((root / ".snakemake/log").glob("*.snakemake.log"))
+        + [p for p in [root / "results_run.txt"] if p.exists()]
+    )
+    clean, withheld = [], []
+    for f in candidates:
+        try:
+            why = transfer.check_file(f, ids, strict=False, forbidden=set())
+        except Exception as exc:  # noqa: BLE001 -- fail closed: any error withholds
+            why = [f"could not be checked: {exc}"]
+        (withheld if why else clean).append((f, why))
+    out.write_text("".join(f"{f.relative_to(root).as_posix()}\n" for f, _ in clean))
+    for f, why in withheld:
+        print(f"WITHHELD ({'; '.join(why)}): {f.relative_to(root)}")
+    print(f"{len(clean)} of {len(candidates)} logs cleared")
+    return 0
+
+
+def synthetic(args: argparse.Namespace) -> int:
+    from bpskit.synthetic import make_delivery
+
+    make_delivery(args.out_dir, args.persons, args.seed)
+    print(f"{args.persons:,} synthetic persons -> {args.out_dir}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="bpskit", description=__doc__.split("\n")[0])
+    parser.add_argument(
+        "--config",
+        default="config/config.yaml",
+        help="workflow config, relative to the workflow directory",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+    p = sub.add_parser("pullable", help="refuse files that name a patient")
+    p.add_argument(
+        "--strict",
+        action="store_true",
+        help="results/out/ rules: csv/txt/png only, no per-person columns",
+    )
+    p.add_argument("paths", nargs="+")
+    p.set_defaults(func=pullable)
+    c = sub.add_parser("check-logs", help="list the logs free of patient ids")
+    c.add_argument("workflow_dir", nargs="?", default=".")
+    c.set_defaults(func=check_logs)
+    s = sub.add_parser("synthetic", help="write a synthetic BPS delivery")
+    s.add_argument("out_dir")
+    s.add_argument("--persons", type=int, default=50_000)
+    s.add_argument("--seed", type=int, default=1)
+    s.set_defaults(func=synthetic)
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bpskit/delivery.py

@@ -0,0 +1,153 @@
+"""Reading a BPS delivery: pipe-delimited CSVs, every column read as text.
+
+Columns stay text and nothing becomes null on its own, because the delivery
+uses sentinel codes ("-1", "-2", "-9", "") for "unknown". Callers decide what
+a sentinel means for each column: a missing date, or an unknown count.
+"""
+
+from collections.abc import Iterator
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+__all__ = [
+    "read_delivery",
+    "read_header",
+    "stream_csv",
+    "parse_date",
+    "normalise_code",
+    "load_config",
+    "delivery_root",
+]
+
+PROFILE = "profiles/default/config.yaml"
+
+
+def read_delivery(path: str | Path, usecols: list[str]) -> pd.DataFrame:
+    """Read a small delivery file whole, quotes and padding stripped.
+
+    For the people file and similar. Diagnosis files can be hundreds of
+    millions of rows: use `stream_csv` for those.
+
+    Raises:
+        KeyError: a needed column is not in the file. The message lists the
+            header that is there, which is usually enough to fix the config.
+    """
+    path = Path(path)
+    for enc in ("utf-8", "latin-1"):
+        try:
+            header = pd.read_csv(path, sep="|", nrows=0, encoding=enc).columns
+            break
+        except UnicodeDecodeError:
+            continue
+    missing = [c for c in usecols if c not in header]
+    if missing:
+        raise KeyError(f"{path.name}: column(s) {missing} not in file; have {list(header)}")
+    df = pd.read_csv(path, sep="|", usecols=usecols, dtype=str, encoding=enc, keep_default_na=False)
+    for c in usecols:
+        df[c] = df[c].str.strip().str.strip('"').str.strip()
+    print(f"[READ ] {path.name}: {len(df):,} rows")
+    return df
+
+
+def parse_date(s: pd.Series, fmt: str, sentinels, mid_month_day: int) -> pd.Series:
+    """Parse delivery date strings, treating sentinel codes as missing.
+
+    Month-precision formats (no `%d`) land on `mid_month_day`, so the average
+    misplacement is zero rather than half a month early. Unparseable values
+    become NaT, but a column in which EVERY non-missing value fails raises:
+    that is a wrong `date_format` in config, not bad data.
+    """
+    s = s.where(~s.isin(sentinels))
+    # An unquoted numeric column can arrive as "20150101.0".
+    s = s.str.replace(r"\.0$", "", regex=True)
+    out = pd.to_datetime(s, format=fmt, errors="coerce")
+    if "%d" not in fmt:
+        # pd.to_timedelta, not pd.Timedelta: NumPy deprecates the latter's
+        # unitless conversion when the column holds NaT.
+        out = out + pd.to_timedelta(mid_month_day - 1, unit="D")
+    if len(out) and out.isna().all() and s.notna().any():
+        raise ValueError(f"every value failed to parse as {fmt!r}, e.g. {s.dropna().iloc[0]!r}")
+    return out
+
+
+def read_header(path: str | Path, encoding: str) -> list[str]:
+    """Column names of a delivery file, without reading any data."""
+    with open(path, encoding=encoding) as fh:
+        return [h.strip().strip('"') for h in fh.readline().rstrip("\r\n").split("|")]
+
+
+def stream_csv(
+    path: str | Path, usecols: list[str], block_mb: float, encoding: str
+) -> Iterator[pd.DataFrame]:
+    """Yield a delivery file as pandas blocks of text columns.
+
+    pyarrow parses `block_mb` of the file at a time, so memory stays flat
+    whatever the file size. Block boundaries carry no meaning: a person's
+    records can be split across blocks, and callers must not assume otherwise.
+
+    Raises:
+        KeyError: a needed column is not in the header.
+    """
+    import pyarrow as pa
+    import pyarrow.csv as pacsv
+
+    path = Path(path)
+    header = read_header(path, encoding)
+    missing = [c for c in usecols if c not in header]
+    if missing:
+        raise KeyError(f"{path.name}: column(s) {missing} not in file; have {header}")
+    reader = pacsv.open_csv(
+        path,
+        read_options=pacsv.ReadOptions(block_size=int(float(block_mb) * 2**20), encoding=encoding),
+        parse_options=pacsv.ParseOptions(delimiter="|"),
+        convert_options=pacsv.ConvertOptions(
+            include_columns=usecols,
+            column_types={c: pa.string() for c in usecols},
+            strings_can_be_null=False,
+        ),
+    )
+    n = 0
+    for batch in reader:
+        df = batch.to_pandas()
+        for c in usecols:
+            df[c] = df[c].str.strip().str.strip('"').str.strip()
+        n += len(df)
+        yield df
+    print(f"[READ ] {path.name}: {n:,} rows streamed")
+
+
+def normalise_code(s: pd.Series) -> pd.Series:
+    """ICD codes as upper case with dots, spaces and quotes removed ('e11.9' -> 'E119')."""
+    return s.astype(str).str.upper().str.replace(r"[.\s\"]", "", regex=True)
+
+
+# --------------------------------------------------------------------------- #
+# Config, for the command-line tools (rules get theirs from Snakemake)
+# --------------------------------------------------------------------------- #
+def load_config(path: str | Path) -> dict:
+    """A workflow's config file (YAML)."""
+    return yaml.safe_load(Path(path).read_text())
+
+
+def delivery_root(root: Path, cfg: dict) -> Path:
+    """Where the delivery is, resolved the way Snakemake resolves it.
+
+    A workflow profile at <root>/profiles/default/config.yaml overrides the
+    config's own `delivery_root`. A `--config delivery_root=...` given to one
+    run is invisible here, so a tool run after such a run must be given the
+    same profile.
+    """
+    root = Path(root)
+    prof = root / "profiles/default/config.yaml"
+    entries = (yaml.safe_load(prof.read_text()) or {}).get("config") if prof.exists() else None
+    entries = entries or []
+    # A profile may write `config:` as a list of "key=value" or as a mapping.
+    if isinstance(entries, dict):
+        entries = [f"{k}={v}" for k, v in entries.items()]
+    for e in entries:
+        k, _, v = str(e).partition("=")
+        if k.strip() == "delivery_root":
+            return (root / v.strip()).resolve()
+    return (root / cfg["delivery_root"]).resolve()

bpskit/icd.py

@@ -0,0 +1,191 @@
+"""ICD-10 code specifications, chapters, and ICD-9 -> ICD-10 mapping.
+
+Codes are handled in normalised form: upper case, no dot ('E119').
+"""
+
+import re
+from collections.abc import Iterable
+
+import numpy as np
+import pandas as pd
+
+from bpskit.delivery import normalise_code
+
+__all__ = [
+    "expand_spec",
+    "prefix_mask",
+    "match_table",
+    "icd10_chapter",
+    "CHAPTER_ROMAN",
+    "Icd9Mapper",
+    "harmonise",
+]
+
+_CAT = re.compile(r"^[A-Z]\d{2}$")
+
+
+def _cat_index(c: str) -> int:
+    """'A00' -> 0, 'B12' -> 112: 3-character categories as integers, for ranges."""
+    if not _CAT.match(c):
+        raise ValueError(f"range end {c!r} is not a 3-character ICD-10 category")
+    return (ord(c[0]) - 65) * 100 + int(c[1:])
+
+
+def _cat_from_index(i: int) -> str:
+    return f"{chr(65 + i // 100)}{i % 100:02d}"
+
+
+def expand_spec(spec: str) -> list[str]:
+    """Expand a code specification into code prefixes.
+
+    'A15-A19, B18, H35.3' -> ['A15', 'A16', 'A17', 'A18', 'A19', 'B18', 'H353'].
+    Range ends must be 3-character categories; a single code may be longer and
+    matches as a prefix. Raises ValueError on a malformed or empty range.
+    """
+    out: list[str] = []
+    for tok in str(spec).split(","):
+        tok = tok.strip().upper().replace(".", "")
+        if not tok:
+            continue
+        if "-" in tok:
+            a, b = (t.strip() for t in tok.split("-"))
+            ia, ib = _cat_index(a), _cat_index(b)
+            if ib < ia:
+                raise ValueError(f"empty range {tok!r}")
+            out += [_cat_from_index(i) for i in range(ia, ib + 1)]
+        else:
+            out.append(tok)
+    return out
+
+
+def prefix_mask(codes: pd.Series, prefixes: list[str]) -> np.ndarray:
+    """True where a code starts with any of `prefixes`."""
+    mask = np.zeros(len(codes), dtype=bool)
+    # One isin per prefix length is far faster than one startswith per prefix.
+    for length in sorted({len(p) for p in prefixes}):
+        want = {p for p in prefixes if len(p) == length}
+        mask |= codes.str[:length].isin(want).to_numpy()
+    return mask
+
+
+def match_table(unique_codes: pd.Series, conditions: pd.DataFrame) -> pd.DataFrame:
+    """(code, condition_id) for every code each condition's `codes` spec matches.
+
+    A condition row may carry an `exclude` spec, removed from what `codes`
+    matched. A code can belong to several conditions (a chapter row and a
+    condition inside it), so this is a table, not a mapping.
+    """
+    codes = pd.Series(unique_codes, dtype=str).reset_index(drop=True)
+    rows = []
+    for _, c in conditions.iterrows():
+        m = prefix_mask(codes, expand_spec(c["codes"]))
+        if c.get("exclude"):
+            m &= ~prefix_mask(codes, expand_spec(c["exclude"]))
+        rows.append(pd.DataFrame({"code": codes[m].to_numpy(), "condition_id": c["condition_id"]}))
+    return pd.concat(rows, ignore_index=True)
+
+
+# First 3-character category of each WHO ICD-10 chapter, in code order. U
+# (chapter XXII, special purposes) sorts between T and V.
+_CHAPTER_STARTS = [
+    ("A00", 1),
+    ("C00", 2),
+    ("D50", 3),
+    ("E00", 4),
+    ("F00", 5),
+    ("G00", 6),
+    ("H00", 7),
+    ("H60", 8),
+    ("I00", 9),
+    ("J00", 10),
+    ("K00", 11),
+    ("L00", 12),
+    ("M00", 13),
+    ("N00", 14),
+    ("O00", 15),
+    ("P00", 16),
+    ("Q00", 17),
+    ("R00", 18),
+    ("S00", 19),
+    ("U00", 22),
+    ("V00", 20),
+    ("Z00", 21),
+]
+CHAPTER_ROMAN = {
+    i: r
+    for i, r in enumerate(
+        "- I II III IV V VI VII VIII IX X XI XII XIII XIV XV XVI XVII XVIII XIX XX XXI XXII".split()
+    )
+}
+
+
+def icd10_chapter(codes: Iterable[str]) -> np.ndarray:
+    """Chapter number (1-22) of 3-character ICD-10 categories; 0 for anything else."""
+    codes = pd.Series(codes, dtype=str)
+    ok = codes.str.fullmatch(r"[A-Z]\d{2}").to_numpy()
+    idx = np.full(len(codes), -1)
+    idx[ok] = [(ord(c[0]) - 65) * 100 + int(c[1:]) for c in codes[ok]]
+    starts = np.array([_cat_index(s) for s, _ in _CHAPTER_STARTS])
+    nums = np.array([n for _, n in _CHAPTER_STARTS])
+    pos = np.searchsorted(starts, idx, side="right") - 1
+    return np.where(ok & (pos >= 0), nums[np.clip(pos, 0, None)], 0)
+
+
+# --------------------------------------------------------------------------- #
+# ICD-9 -> ICD-10
+# --------------------------------------------------------------------------- #
+class Icd9Mapper:
+    """Map ICD-9-CM codes to ICD-10-CM with the CMS GEM bundled in icd-mappings.
+
+    Results are cached per code, because a delivery repeats a few thousand
+    codes hundreds of millions of times. With `enabled=False`, or when
+    icd-mappings is not installed, every code maps to None and is dropped as
+    unmapped rather than passed through as if it were ICD-10.
+    """
+
+    def __init__(self, enabled: bool = True):
+        self._mapper = None
+        if enabled:
+            try:
+                from icdmappings import Mapper
+
+                self._mapper = Mapper()
+            except ImportError:
+                print("[WARN ] icd-mappings not installed: ICD-9 records are dropped, not mapped")
+        self._cache: dict[str, str | None] = {}
+
+    def map(self, codes: pd.Series) -> pd.Series:
+        new = sorted(set(codes.unique()) - self._cache.keys())
+        if new:
+            got = (
+                self._mapper.map(new, source="icd9", target="icd10")
+                if self._mapper
+                else [None] * len(new)
+            )
+            self._cache.update(zip(new, got))
+        return codes.map(self._cache)
+
+
+def harmonise(df: pd.DataFrame, changeover_day: float | None, mapper: Icd9Mapper) -> pd.Series:
+    """Map ICD-9 codes in `df.code` to ICD-10 in place; return each row's system.
+
+    A code is ICD-9 when it is all digits. Before `changeover_day` (None: never)
+    the letter-led supplementary (V01-V91) and external-cause (E800-E999) codes
+    count as ICD-9 too. Those shapes are also valid ICD-10 (E11 is diabetes),
+    so only a date can place them, and only where the changeover was measured
+    on that file.
+
+    Returns:
+        "icd10", "icd9_mapped" or "icd9_unmapped" per row. Unmapped rows keep
+        their ICD-9 code; the caller drops them.
+    """
+    is9 = df.code.str.fullmatch(r"\d+")
+    if changeover_day is not None:
+        is9 |= (df.day < changeover_day) & df.code.str.match(r"^(V\d{2}|E[89]\d{2})")
+    system = pd.Series(np.where(is9, "icd9", "icd10"), index=df.index)
+    if is9.any():
+        m = mapper.map(df.code[is9])
+        hit = m.notna()
+        system[is9] = np.where(hit, "icd9_mapped", "icd9_unmapped")
+        df.loc[hit[hit].index, "code"] = normalise_code(m[hit])
+    return system

bpskit/persontime.py

@@ -0,0 +1,132 @@
+"""Person-time: dates as numbers, age bands, Lexis splitting, streaming persons.
+
+Every per-person array holds dates as float days since 1970-01-01, so
+person-time is a subtraction and NaN means "no date" (alive, never recorded).
+Ages are in years of DAYS_PER_YEAR days.
+"""
+
+from collections.abc import Iterator
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+__all__ = [
+    "DAYS_PER_YEAR",
+    "to_days",
+    "day_of",
+    "follow_up",
+    "age_bands",
+    "lexis",
+    "band_index",
+    "accumulate",
+    "person_chunks",
+    "read_universe",
+]
+
+DAYS_PER_YEAR = 365.25
+EPOCH = np.datetime64("1970-01-01", "D")
+
+
+def to_days(s: pd.Series) -> np.ndarray:
+    """Datetimes to float days since 1970-01-01; NaT becomes NaN."""
+    v = s.to_numpy("datetime64[D]")
+    out = ((v - EPOCH) / np.timedelta64(1, "D")).astype(float)
+    out[np.isnat(v)] = np.nan
+    return out
+
+
+def day_of(date: str | pd.Timestamp) -> float:
+    """One date (string or Timestamp) as days since 1970-01-01."""
+    return to_days(pd.Series([pd.Timestamp(date)]))[0]
+
+
+def follow_up(
+    birth_d: np.ndarray, death_d: np.ndarray, start_d: float, end_d: float
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """Entry day, exit day and whether any follow-up falls inside [start, end].
+
+    A person enters at the later of the period start and birth, and leaves at
+    the earlier of death and the period end; NaN death means alive.
+    """
+    entry = np.maximum(start_d, birth_d)
+    exit_ = np.minimum(end_d, np.where(np.isnan(death_d), np.inf, death_d))
+    return entry, exit_, exit_ > entry
+
+
+def age_bands(width: int, open_from: int) -> pd.DataFrame:
+    """Age bands of `width` years from 0, with a last open band from `open_from`.
+
+    Columns: age_band (label), age_lo, age_hi (200 for the open band), age_mid.
+    """
+    lo = np.arange(0, open_from + 1, width)
+    hi = np.append(lo[1:], 200.0).astype(float)
+    label = [f"{a}-{a + width - 1}" for a in lo[:-1]] + [f"{lo[-1]}+"]
+    mid = np.append(lo[:-1] + width / 2, lo[-1] + width / 2)
+    return pd.DataFrame({"age_band": label, "age_lo": lo, "age_hi": hi, "age_mid": mid})
+
+
+def lexis(age_start: np.ndarray, age_end: np.ndarray, bands: pd.DataFrame) -> np.ndarray:
+    """Person-years each interval [age_start, age_end) spends in each band (N x B).
+
+    An empty or reversed interval contributes zero everywhere.
+    """
+    lo = bands["age_lo"].to_numpy(float)[None, :]
+    hi = bands["age_hi"].to_numpy(float)[None, :]
+    a = np.asarray(age_start, float)[:, None]
+    b = np.asarray(age_end, float)[:, None]
+    return np.clip(np.minimum(b, hi) - np.maximum(a, lo), 0.0, None)
+
+
+def band_index(age: np.ndarray, bands: pd.DataFrame) -> np.ndarray:
+    """Band of each age; ages outside the bands are clipped to the first or last."""
+    lo = bands["age_lo"].to_numpy(float)
+    return np.clip(np.searchsorted(lo, np.asarray(age, float), side="right") - 1, 0, len(lo) - 1)
+
+
+def accumulate(acc: np.ndarray, key: np.ndarray, values: np.ndarray) -> None:
+    """acc[key[i], b] += values[i, b] for every row i and column b, in place."""
+    for b in range(values.shape[1]):
+        acc[:, b] += np.bincount(key, weights=values[:, b], minlength=acc.shape[0])
+
+
+def person_chunks(
+    path: str | Path, batch_rows: int, columns: list[str] | None = None
+) -> Iterator[pd.DataFrame]:
+    """Stream a parquet sorted by `idx` as DataFrames that never split a person.
+
+    Each yielded frame holds whole persons; the last person of a batch is held
+    back and joined to the next one. The file MUST be sorted by idx, or a
+    person can come out in two chunks.
+    """
+    import pyarrow.parquet as pq
+
+    left = None
+    for batch in pq.ParquetFile(path).iter_batches(batch_size=int(batch_rows), columns=columns):
+        df = batch.to_pandas()
+        if left is not None:
+            df = pd.concat([left, df], ignore_index=True)
+        cut = int(np.searchsorted(df.idx.to_numpy(), df.idx.iat[-1], side="left"))
+        if cut:
+            yield df.iloc[:cut].reset_index(drop=True)
+        left = df.iloc[cut:]
+    if left is not None and len(left):
+        yield left.reset_index(drop=True)
+
+
+def read_universe(path: str | Path, columns: list[str]) -> pd.DataFrame:
+    """Read universe columns, refusing a file written by an older workflow version.
+
+    Snakemake does not rebuild an intermediate whose rule did not change, so a
+    universe.parquet from an older layout can outlive the code that wrote it.
+    """
+    import pyarrow.parquet as pq
+
+    have = set(pq.read_schema(path).names)
+    missing = [c for c in columns if c not in have]
+    if missing:
+        raise RuntimeError(
+            f"{path} has no column(s) {missing}: it was written by an older version of "
+            "this workflow. Rebuild everything downstream of it: snakemake -c 20 --forceall"
+        )
+    return pd.read_parquet(path, columns=columns)

bpskit/stats.py

@@ -0,0 +1,76 @@
+"""Exact tests and multiplicity adjustment, vectorised over many tests at once.
+
+Missing inputs give NaN p-values, and NaN p-values stay out of the adjustment
+family rather than counting as tests.
+"""
+
+import numpy as np
+import pandas as pd
+from numpy.typing import ArrayLike
+from scipy import stats
+
+__all__ = ["bh", "adjust", "poisson_rr", "binom_half_p", "binom_p"]
+
+
+def bh(p: pd.Series) -> pd.Series:
+    """Benjamini-Hochberg q-values over the non-missing p-values."""
+    p = pd.Series(p, dtype=float)
+    ok = p.notna()
+    q = pd.Series(np.nan, index=p.index)
+    if ok.sum():
+        v = p[ok].to_numpy()
+        order = np.argsort(v)
+        m = len(v)
+        ranked = v[order] * m / np.arange(1, m + 1)
+        ranked = np.minimum.accumulate(ranked[::-1])[::-1]
+        out = np.empty(m)
+        out[order] = np.minimum(ranked, 1.0)
+        q[ok] = out
+    return q
+
+
+def adjust(p: pd.Series, method: str) -> pd.Series:
+    """BH q-values, or Bonferroni-adjusted p-values, over the non-missing entries."""
+    if method == "bh":
+        return bh(p)
+    if method == "bonferroni":
+        p = pd.Series(p, dtype=float)
+        return (p * p.notna().sum()).clip(upper=1.0)
+    raise ValueError(f"multiplicity must be 'bh' or 'bonferroni', not {method!r}")
+
+
+def poisson_rr(
+    obs: ArrayLike, exp: ArrayLike, alpha: float
+) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """Observed/expected ratio, exact (Garwood) interval and two-sided exact p.
+
+    `exp` is treated as known. exp == 0 gives an infinite or NaN ratio.
+    """
+    obs, exp = np.asarray(obs, float), np.asarray(exp, float)
+    with np.errstate(invalid="ignore", divide="ignore"):
+        rr = obs / exp
+        lo = np.where(obs > 0, stats.chi2.ppf(alpha / 2, 2 * obs) / 2, 0.0) / exp
+        hi = stats.chi2.ppf(1 - alpha / 2, 2 * obs + 2) / 2 / exp
+        p = np.minimum(
+            1.0, 2 * np.minimum(stats.poisson.sf(obs - 1, exp), stats.poisson.cdf(obs, exp))
+        )
+    return rr, lo, hi, p
+
+
+def binom_half_p(k: ArrayLike, n: ArrayLike) -> np.ndarray:
+    """Two-sided exact binomial test of k successes in n against p = 0.5; NaN when n = 0."""
+    k, n = np.asarray(k, float), np.asarray(n, float)
+    with np.errstate(invalid="ignore"):
+        p = np.minimum(1.0, 2 * stats.binom.cdf(np.minimum(k, n - k), n, 0.5))
+    return np.where(n > 0, p, np.nan)
+
+
+def binom_p(k: ArrayLike, n: ArrayLike, p0: ArrayLike) -> np.ndarray:
+    """Two-sided exact binomial test of k in n against p0 (doubled smaller tail).
+
+    NaN when n = 0 or p0 is 0 or 1, where the test is undefined.
+    """
+    k, n, p0 = np.asarray(k, float), np.asarray(n, float), np.asarray(p0, float)
+    with np.errstate(invalid="ignore"):
+        p = np.minimum(1.0, 2 * np.minimum(stats.binom.cdf(k, n, p0), stats.binom.sf(k - 1, n, p0)))
+    return np.where((n > 0) & (p0 > 0) & (p0 < 1), p, np.nan)

bpskit/synthetic.py

@@ -0,0 +1,179 @@
+"""A synthetic BPS delivery with known structure, for tests and examples.
+
+The files have the BPS column names and formats (pipe-delimited, `AAAAMM` or
+`AAAAMMDD` dates, ICD-10 codes with and without dots), so a workflow written
+for the real delivery runs on it unchanged. What is planted, so a correct
+analysis can be checked against it:
+
+    E11 -> N18 -> I50 -> death   directional: hazard x4 after the predecessor
+                                 (x5 for death after I50)
+    J44 ~ M81                    associated through a shared frailty, no direction
+    150 codes (A/B/C/G/K/L00-24) independent of everything, with rates scaled by a
+                                 person-level healthcare-use intensity (lognormal)
+    R53                          routine visits, Poisson in that intensity
+    U07                          from 2020 only, independent of everything
+
+Directional tests need power: at 50,000 persons the planted pairs are found;
+at a few thousand the files are only good for exercising code.
+
+    from bpskit.synthetic import make_delivery
+    cfg = make_delivery("data/synthetic", n_persons=2_000)   # writes the files
+    cfg["universe"], cfg["sources"]                           # how to read them
+"""
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+__all__ = ["FILES", "config", "make_delivery"]
+
+# One people file and three diagnosis sources, named by what they hold.
+FILES = {
+    "universe": "sociodemograficos.csv",
+    "ap": "atencion_primaria.csv",
+    "cmbd": "hospitalizaciones.csv",
+    "bps": "patologias_bps.csv",
+}
+
+
+def config(delivery_root: str | Path = ".") -> dict:
+    """The `delivery_root`, `universe` and `sources` config of a synthetic delivery.
+
+    The same shape a BPS workflow config uses, so it can be merged into one:
+    `universe` names the people file's columns and sex labels, and each source
+    its id, date and ICD-code columns.
+    """
+    return {
+        "delivery_root": str(delivery_root),
+        "universe": {
+            "file": FILES["universe"],
+            "id": "COD_ID",
+            "sex": "DESC_SEXO",
+            "sex_values": {"Mujer": "F", "Hombre": "M"},
+            "birth": "ANYO_NAC",
+            "death": "ANYO_FAC",
+            "date_format": "%Y%m",
+        },
+        "sources": {
+            "ap": {
+                "file": FILES["ap"],
+                "id": "COD_ID",
+                "date": "FECHA_INICIO_DIAGNOSTICO",
+                "date_format": "%Y%m%d",
+                "code": "COD_CIE",
+            },
+            "cmbd": {
+                "file": FILES["cmbd"],
+                "id": "COD_ID",
+                "date": "COD_FEC_INGRESO",
+                "date_format": "%Y%m%d",
+                "code": "COD_CIE",
+                "position": "COD_POSICION",
+            },
+            "bps": {
+                "file": FILES["bps"],
+                "id": "COD_ID",
+                "date": "FEC_INI_PATOLOGIA",
+                "date_format": "%Y%m",
+                "code": "COD_CIE",
+            },
+        },
+    }
+
+
+def make_delivery(out_dir: str | Path, n_persons: int = 50_000, seed: int = 1) -> dict:
+    """Write a synthetic delivery of `n_persons` into `out_dir`; return its config.
+
+    The same `n_persons` and `seed` always write the same bytes. Persons are
+    followed 2015-2025; identifiers look like `P00000042`.
+    """
+    n = n_persons
+    N = int(n)
+    rng = np.random.default_rng(int(seed))
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    T0, T1 = 2015.0, 2026.0  # decimal years
+
+    birth = rng.uniform(1930, 1975, N)
+    sex = rng.choice(["Mujer", "Hombre"], N)
+    frail = rng.random(N) < 0.1
+    use = rng.lognormal(0, 0.8, N)
+
+    def first_time(rate):
+        t = T0 + rng.exponential(1 / rate)
+        return np.where(t < T1, t, np.nan)
+
+    def after(base, mult, trigger):
+        """Hazard `base`, multiplied by `mult` from the trigger's time on."""
+        t0 = first_time(np.full(N, base))
+        t1 = trigger + rng.exponential(1 / (base * mult), N)
+        t = np.where(np.isnan(trigger) | (t0 < trigger), t0, t1)
+        return np.where(t < T1, t, np.nan)
+
+    codes = {"E11": first_time(0.01 * np.exp((2020 - birth - 65) / 15))}
+    codes["N18"] = after(0.004, 4, codes["E11"])
+    codes["I50"] = after(0.004, 4, codes["N18"])
+    codes["J44"] = first_time(np.where(frail, 0.03, 0.004))
+    codes["M81"] = first_time(np.where(frail, 0.03, 0.004))
+    for c in [f"{letter}{n:02d}" for letter in "ABCGKL" for n in range(25)]:
+        codes[c] = first_time(rng.uniform(0.002, 0.02) * use)
+    death = after(0.01, 5, codes["I50"])
+
+    rows = []
+    for c, t in codes.items():
+        ids = np.flatnonzero(~np.isnan(t))
+        for rep in range(2):  # first record + one later repeat
+            tt = t[ids] + rep * rng.uniform(0, 2, len(ids))
+            ok = (tt < T1) & (np.isnan(death[ids]) | (tt <= death[ids]))
+            rows.append(pd.DataFrame({"i": ids[ok], "t": tt[ok], "code": c}))
+    k = rng.poisson(3 * use * 11)
+    vi = np.repeat(np.arange(N), k)
+    vt = rng.uniform(T0, T1, len(vi))
+    ok = np.isnan(death[vi]) | (vt <= death[vi])
+    rows.append(pd.DataFrame({"i": vi[ok], "t": vt[ok], "code": "R53"}))
+    m = rng.random(N) < 0.15
+    rows.append(
+        pd.DataFrame({"i": np.flatnonzero(m), "t": rng.uniform(2020, T1, m.sum()), "code": "U07"})
+    )
+    ev = pd.concat(rows, ignore_index=True)
+
+    def fmt(t, f):
+        d = pd.to_datetime("2015-01-01") + pd.to_timedelta((pd.Series(t) - T0) * 365.25, unit="D")
+        return d.dt.strftime(f).to_numpy()
+
+    pid = np.array([f"P{i:08d}" for i in range(N)])
+    pd.DataFrame(
+        {
+            "COD_ID": pid,
+            "DESC_SEXO": sex,
+            "ANYO_NAC": fmt(birth, "%Y%m"),
+            "ANYO_FAC": np.where(np.isnan(death), "", fmt(np.nan_to_num(death, nan=T0), "%Y%m")),
+        }
+    ).to_csv(out / FILES["universe"], sep="|", index=False)
+    # routine visits and U07 are primary care; the rest is split across the sources
+    src = np.where(
+        ev.code.isin(["R53", "U07"]),
+        "ap",
+        rng.choice(["ap", "cmbd", "bps"], len(ev), p=[0.6, 0.25, 0.15]),
+    )
+    a, c, b = (ev[src == s] for s in ("ap", "cmbd", "bps"))
+    pd.DataFrame(
+        {
+            "COD_ID": pid[a.i],
+            "FECHA_INICIO_DIAGNOSTICO": fmt(a.t, "%Y%m%d"),
+            "COD_CIE": a.code + ".9",
+        }
+    ).to_csv(out / FILES["ap"], sep="|", index=False)
+    pd.DataFrame(
+        {
+            "COD_ID": pid[c.i],
+            "COD_FEC_INGRESO": fmt(c.t, "%Y%m%d"),
+            "COD_CIE": c.code,
+            "COD_POSICION": rng.integers(1, 4, len(c)),
+        }
+    ).to_csv(out / FILES["cmbd"], sep="|", index=False)
+    pd.DataFrame(
+        {"COD_ID": pid[b.i], "FEC_INI_PATOLOGIA": fmt(b.t, "%Y%m"), "COD_CIE": b.code}
+    ).to_csv(out / FILES["bps"], sep="|", index=False)
+    return config(out)

bpskit/transfer.py

@@ -0,0 +1,211 @@
+"""Does a file name a patient? The check behind every transfer off the data machine.
+
+The rule: no row carrying a patient identifier leaves the machine that holds
+the delivery. It is tested on the bytes, against the identifiers the workflow
+actually READ, not against a guessed id format:
+
+    results/work/ids/universe.parquet   every id in the people file, written
+                                        before the universe drops anyone
+    results/work/ids/<source>.parquet   ids in a diagnosis file that are not in
+                                        the universe (ingest drops their rows)
+
+An id the workflow never read cannot be in anything it wrote, so this set is
+complete. The check fails closed: an unreadable file, an unknown suffix or an
+empty id set refuses, because a guard that waves through what it could not
+parse is trusted and wrong.
+
+A small aggregate cell is not a row and names nobody, so cell counts are not
+checked here; a publication floor is a separate, later decision.
+"""
+
+import re
+from collections.abc import Iterable
+from pathlib import Path
+
+import pandas as pd
+
+from bpskit.delivery import delivery_root, stream_csv
+
+__all__ = [
+    "IDS_DIR",
+    "MIN_ID_LEN",
+    "write_ids",
+    "load_ids",
+    "ids_for_root",
+    "forbidden_columns",
+    "check_file",
+    "check",
+]
+
+IDS_DIR = "results/work/ids"
+# Shorter ids would match ordinary counts in every table. How many are skipped
+# is printed, so a delivery with short ids cannot weaken the check unnoticed.
+MIN_ID_LEN = 4
+# results/out/ is copied wholesale, so only formats this check can read (and
+# figures, which are drawn from tables that must pass on their own).
+OUT_SUFFIXES = {".csv", ".txt", ".png"}
+TEXT_SUFFIXES = {".csv", ".tsv", ".txt", ".log", ".md", ".json"}
+OPAQUE_SUFFIXES = {".png"}
+# A CSV with one of these columns is a per-person table whatever else it holds.
+INTERNAL_KEYS = {"pid", "idx", "bucket", "birth", "death", "birth_d", "death_d", "day"}
+ID_LIKE = re.compile(r"(^|_)(pid|patient|cod_id|id_paciente|nuhsa)($|_)", re.I)
+# Deliberately wide: an id inside a delimited row, a quoted cell, `key=value`,
+# a traceback repr or a sentence must come out as a token of its own.
+TOKENISERS = (
+    re.compile(r"[^\s,;|\"'\[\](){}:=<>/\\]+"),
+    re.compile(r"[A-Za-z0-9_-]+"),
+    re.compile(r"[A-Za-z0-9]+"),
+    re.compile(r"\d+"),
+)
+
+
+# --------------------------------------------------------------------------- #
+# Identifiers
+# --------------------------------------------------------------------------- #
+def _clean(values) -> set[str]:
+    s = pd.Series(list(values), dtype="string").dropna().str.strip().str.strip('"').str.strip()
+    return set(s[s != ""])
+
+
+def _finish(ids: set[str], where: str) -> set[str]:
+    short = sum(len(i) < MIN_ID_LEN for i in ids)
+    ids = {i for i in ids if len(i) >= MIN_ID_LEN}
+    if not ids:
+        raise SystemExit(f"[REFUSE] no patient identifiers loaded from {where}; nothing is cleared")
+    warn = f"  [WARN ] {short:,} shorter than {MIN_ID_LEN} chars are NOT checked" if short else ""
+    print(f"[IDS  ] {len(ids):,} identifiers from {where}{warn}")
+    return ids
+
+
+def write_ids(ids: Iterable[str], path: str | Path) -> None:
+    """Write an id list for the transfer check (one `pid` column)."""
+    pd.DataFrame({"pid": pd.Series(list(ids), dtype="string")}).to_parquet(path, index=False)
+
+
+def load_ids(paths: Iterable[str | Path]) -> set[str]:
+    """The identifier set from id files, as declared inputs of a rule."""
+    paths = list(paths)
+    ids: set[str] = set()
+    for p in paths:
+        ids |= _clean(pd.read_parquet(p, columns=["pid"]).pid)
+    return _finish(ids, f"{len(paths)} id file(s)")
+
+
+def ids_for_root(root: Path, cfg: dict) -> set[str]:
+    """Every identifier the workflow at `root` has read, for the hand-run checks.
+
+    Uses the id files where they exist. Where one is missing (a run that died
+    before `universe` or inside an `ingest_source`), it reads that file's id
+    column from the delivery itself, because the job may have printed a row
+    before dying.
+
+    Raises:
+        SystemExit: an id file is missing and the delivery file cannot be read.
+    """
+    root = Path(root)
+    ids_dir = root / IDS_DIR
+    wanted = {"universe": cfg["universe"]} | dict(cfg["sources"])
+    ids: set[str] = set()
+    fallback = []
+    for name, spec in wanted.items():
+        f = ids_dir / f"{name}.parquet"
+        if f.exists():
+            ids |= _clean(pd.read_parquet(f, columns=["pid"]).pid)
+            continue
+        path = delivery_root(root, cfg) / spec["file"]
+        sc = cfg["scale"]
+        try:
+            for block in stream_csv(path, [spec["id"]], sc["csv_block_mb"], sc["encoding"]):
+                ids |= _clean(block[spec["id"]].unique())
+        except Exception as exc:  # noqa: BLE001 -- fail closed: no ids, nothing cleared
+            raise SystemExit(
+                f"[REFUSE] no id file for {name!r} and its delivery file could not be read "
+                f"({exc}); nothing is cleared"
+            ) from exc
+        fallback.append(name)
+    where = IDS_DIR + (
+        f" + delivery files for {', '.join(fallback)} (no id file yet)" if fallback else ""
+    )
+    return _finish(ids, where)
+
+
+def forbidden_columns(cfg: dict) -> set[str]:
+    """Lower-cased column names that make a CSV per-person.
+
+    The internal keys plus the delivery's own id and date columns, taken from
+    config/config.yaml so a new source is covered without editing this.
+    """
+    cols = set(INTERNAL_KEYS)
+    u = cfg["universe"]
+    cols |= {u["id"], u["birth"], u["death"]}
+    for s in cfg["sources"].values():
+        cols |= {s["id"], s["date"]}
+    return {c.lower() for c in cols}
+
+
+# --------------------------------------------------------------------------- #
+# Files
+# --------------------------------------------------------------------------- #
+def _tokens(text: str) -> set[str]:
+    out: set[str] = set()
+    for t in TOKENISERS:
+        out.update(t.findall(text))
+    return out
+
+
+def check_file(path: Path, ids: set[str], strict: bool, forbidden: set[str]) -> list[str]:
+    """Reasons this file may not leave; empty if it may.
+
+    `strict` adds the rules for results/out/: an allowed suffix, and no CSV
+    header column in `forbidden` or shaped like an identifier. Without it
+    (logs), only the identifier scan applies.
+    """
+    path = Path(path)
+    suffix = path.suffix.lower()
+    if strict and suffix not in OUT_SUFFIXES:
+        return [f"suffix {suffix or '(none)'} is not one of {sorted(OUT_SUFFIXES)}"]
+    if suffix in OPAQUE_SUFFIXES:
+        return []
+    if suffix not in TEXT_SUFFIXES:
+        return [f"suffix {suffix or '(none)'} cannot be checked"]
+    bad = []
+    text = path.read_text(encoding="utf-8", errors="replace")
+    if hits := ids & _tokens(text):
+        bad.append(f"contains {len(hits)} patient identifier(s)")
+    if strict and suffix == ".csv" and text:
+        header = [c.strip().strip('"').strip() for c in text.split("\n", 1)[0].split(",")]
+        if cols := [c for c in header if c.lower() in forbidden or ID_LIKE.search(c)]:
+            bad.append(f"per-person or identifier column(s) {cols}")
+    return bad
+
+
+def check(
+    paths, ids: set[str], strict: bool, forbidden: set[str], skip=()
+) -> tuple[int, list[str]]:
+    """Check files and directories (recursively).
+
+    Returns:
+        (number of files checked, one "path: reason" line per problem). A
+        listed path that does not exist is skipped: not built yet is not a leak.
+    """
+    skip = {Path(p).resolve() for p in skip}
+    files = []
+    for p in map(Path, paths):
+        if p.is_dir():
+            files += sorted(f for f in p.rglob("*") if f.is_file())
+        elif p.exists():
+            files.append(p)
+    problems, n = [], 0
+    for f in files:
+        if f.resolve() in skip:
+            continue
+        # Snakemake's empty marker inside a directory output; sync.sh never copies it.
+        if f.name == ".snakemake_timestamp" and f.stat().st_size == 0:
+            continue
+        n += 1
+        try:
+            why = check_file(f, ids, strict, forbidden)
+        except Exception as exc:  # noqa: BLE001 -- fail closed: ANY error refuses
+            why = [f"could not be checked: {exc}"]
+        problems += [f"{f}: {w}" for w in why]
+    return n, problems

bpskit-0.2.0.dist-info/METADATA

@@ -0,0 +1,181 @@
+Metadata-Version: 2.5
+Name: bpskit
+Version: 0.2.0
+Summary: Tools for studies on a BPS (Base Poblacional de Salud) health-records delivery, including an identifier check for outputs leaving a secure data environment
+Project-URL: Homepage, https://github.com/loucerac/bpskit
+Project-URL: Changelog, https://github.com/loucerac/bpskit/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/loucerac/bpskit/issues
+Author-email: Carlos Loucera <clou@us.es>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ICD-10,ICD-9,disclosure control,electronic health records,epidemiology,person-time,snakemake,trusted research environment
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: icd-mappings<0.7,>=0.6.2
+Requires-Dist: numpy>=2
+Requires-Dist: pandas>=2.2
+Requires-Dist: pyarrow>=15
+Requires-Dist: pyyaml>=6
+Requires-Dist: scipy>=1.11
+Description-Content-Type: text/markdown
+
+# bpskit
+
+Tools for studies on a BPS (Base Poblacional de Salud, Servicio Andaluz de
+Salud) health-records delivery, and for working with pseudonymised
+health-record extracts in a secure data environment generally.
+
+- **Read the delivery.** Pipe-delimited files read as text, sentinel codes for
+  "unknown", month- and day-precision dates, streaming in fixed-size blocks so a
+  400-million-row file never sits in memory.
+- **ICD codes.** Code specifications like `A15-A19, H35.3`, ICD-10 chapters,
+  and ICD-9 → ICD-10 mapping with the CMS GEM. That includes the V/E
+  supplementary codes, which are also valid ICD-10 and can only be placed by
+  the record's date relative to a changeover.
+- **Person-time.** Dates as days since 1970, age bands, Lexis splitting,
+  person-sorted streaming.
+- **Exact tests.** Poisson rate ratios (Garwood intervals), binomial tests,
+  BH and Bonferroni.
+- **An output check for leaving the secure environment.** `bpskit.transfer`
+  refuses any file that contains an identifier the workflow read, or a
+  per-person column, and fails closed when it cannot read something.
+- **A synthetic delivery** with BPS column names and planted structure, for
+  tests and examples.
+
+## Installing
+
+```bash
+pip install "bpskit @ git+https://github.com/loucerac/bpskit.git@v0.2.0"
+```
+
+In a pixi project (`pixi.toml`):
+
+```toml
+[pypi-dependencies]
+bpskit = { git = "https://github.com/loucerac/bpskit.git", tag = "v0.2.0" }
+```
+
+In a Snakemake rule environment (`workflow/envs/*.yaml`):
+
+```yaml
+dependencies:
+  - pip
+  - pip:
+      - git+https://github.com/loucerac/bpskit.git@v0.2.0
+```
+
+Pin a release. Bumping the tag changes the environment file, and Snakemake's
+`software-env` rerun trigger reruns the rules that use it. A module imported
+from a sibling folder gets no such tracking.
+
+## A worked example
+
+[`examples/quickstart.py`](examples/quickstart.py) runs in a few seconds with no
+real data:
+
+1. it writes a 5,000-person synthetic delivery;
+2. it measures the incidence of type 2 diabetes (E11) in primary care in
+   2018-2025;
+3. it compares women with the whole population;
+4. it runs the transfer check on the result.
+
+The core of it:
+
+```python
+from bpskit import transfer
+from bpskit.delivery import normalise_code, parse_date, read_delivery, stream_csv
+from bpskit.icd import expand_spec, prefix_mask
+from bpskit.persontime import day_of, follow_up, to_days
+from bpskit.stats import poisson_rr
+from bpskit.synthetic import make_delivery
+
+cfg = make_delivery("work/delivery", n_persons=5_000)  # files + how to read them
+
+# people: birth and death as days since 1970 (NaN = alive)
+u = cfg["universe"]
+people = read_delivery(f"work/delivery/{u['file']}", [u["id"], u["sex"], u["birth"], u["death"]])
+birth = to_days(parse_date(people[u["birth"]], u["date_format"], ["", "-1"], mid_month_day=15))
+
+# first E11 record per person, streamed block by block
+ap = cfg["sources"]["ap"]
+for block in stream_csv(
+    f"work/delivery/{ap['file']}", [ap["id"], ap["date"], ap["code"]], 16, "utf-8"
+):
+    hits = block[prefix_mask(normalise_code(block[ap["code"]]), expand_spec("E11"))]
+    ...
+
+# follow-up, then observed vs expected with an exact interval
+entry, exit_, in_period = follow_up(birth, death, day_of("2018-01-01"), day_of("2025-12-31"))
+ratio, lo, hi, p = poisson_rr(observed, expected, alpha=0.05)
+
+# nothing leaves the machine that names a patient
+n, problems = transfer.check(
+    ["work/results/out"],
+    ids=set(people[u["id"]]),
+    strict=True,
+    forbidden=transfer.forbidden_columns(cfg),
+)
+```
+
+```
+E11 incidence 1,258 per 100,000 person-years; women vs all: 1.12 (0.98-1.27)
+aggregate table: 1 file checked, 0 problem(s)
+with a leaked file:
+  .../first_cases.csv: contains 3 patient identifier(s)
+  .../first_cases.csv: per-person or identifier column(s) ['patient']
+```
+
+## The transfer check
+
+The rule it enforces: no row carrying a patient identifier leaves the machine
+that holds the delivery. It checks the bytes of every file against the
+identifiers the workflow actually read, not against a guessed id format. A
+workflow writes those identifiers with `transfer.write_ids`:
+
+- `results/work/ids/universe.parquet`: every id in the people file, written
+  before anyone is dropped;
+- `results/work/ids/<source>.parquet`: ids in a diagnosis file but not in the
+  people file.
+
+Where an id file is missing (a run that died early), the check reads the
+delivery file instead. An unreadable file, an unknown suffix or an empty id set
+all refuse. With `strict=True` (for a results folder copied off wholesale),
+only csv/txt/png pass. CSVs whose header holds a per-person key, one of the
+delivery's id or date columns, or an identifier-like name are refused too.
+
+From a workflow directory on the data machine:
+
+```bash
+bpskit pullable --strict results/out   # exit 0 only if results/out/ may leave
+bpskit check-logs .                    # list the logs that may leave
+bpskit synthetic data/synthetic --persons 2000
+```
+
+`pullable` and `check-logs` read the workflow's `config/config.yaml`
+(`--config` to change it). Its `universe` and `sources` sections, in the shape
+`bpskit.synthetic.config()` returns, name the id columns.
+
+## Developing
+
+```bash
+pixi run test       # pytest
+pixi run lint       # ruff check + format check
+pixi run example    # the worked example
+pixi run build      # sdist + wheel, twine check
+```
+
+Releasing: bump `__version__` in `src/bpskit/__init__.py`, add the release to
+`CHANGELOG.md`, commit, then tag `vX.Y.Z` and push the tag.
+`.github/workflows/release.yml` checks that the tag matches the version,
+builds, and publishes to PyPI by trusted publishing.
+
+## Citing
+
+See [`CITATION.cff`](CITATION.cff). MIT licensed.

bpskit-0.2.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+bpskit/__init__.py,sha256=nc8t3V09DcRdGnjvrRohAqCG-SW0FXtF7mZKVoX5uDA,714
+bpskit/cli.py,sha256=oez_8vrj1Ae_QW_gX9r7KDi6pFxInQM1jbFA5OmmsKA,3938
+bpskit/delivery.py,sha256=LWddYwITPR7AqMenxjzvrgltf8rpblMPjsPhZdhvaYM,5940
+bpskit/icd.py,sha256=yWnDUo--9i6KWTC-qfsf_69kQMsKMdLbJ0_rl__do3I,6627
+bpskit/persontime.py,sha256=Rufi_MbyM5geyQmReo4aOwReB0TVIAIWAf7w57aWMEw,4892
+bpskit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bpskit/stats.py,sha256=L4BpDEyRT8kEGJYNisY_4Y9zKTpDb4HTYLnUF90npdk,2876
+bpskit/synthetic.py,sha256=4OPO2Sn6n31onFnQcosrGxFbGp3AclVLMcZ6vprQeZg,6867
+bpskit/transfer.py,sha256=_s44aKadLXmb1ieEsapH4y3udxD5ECrm-2i__gwjOQg,8321
+bpskit-0.2.0.dist-info/METADATA,sha256=ibxTjKKwjbGK--0YL9v4MaJpTpgJgEFZ-PYnYnD_--U,6954
+bpskit-0.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+bpskit-0.2.0.dist-info/entry_points.txt,sha256=NJKaOyKGXbc9J5x_ycMW0AKPrJ5fXoaWBRQSIpGCTKw,43
+bpskit-0.2.0.dist-info/licenses/LICENSE,sha256=O9v5wiTVjzzmwlbvn3s1hN9a0J3R8WXv9Ue8zx_R_Gg,1071
+bpskit-0.2.0.dist-info/RECORD,,

bpskit-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

bpskit-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+bpskit = bpskit.cli:main

bpskit-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Carlos Loucera
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.