bpskit 0.2.0__tar.gz → 0.3.0__tar.gz

{bpskit-0.2.0 → bpskit-0.3.0}/CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to bpskit. The format follows
 [Semantic Versioning](https://semver.org/): until 1.0, a minor release may
 change the public API (the names in each module's `__all__`).
 
+## [0.3.0]
+
+### Added
+- `bpskit.strata`: `Strata` (person-time and expected counts by sex, age band,
+  calendar year and prior-year contact intensity) and `person_pairs`.
+- `bpskit.stats.exact_contrast` and `idtn_from_ratio`: rate ratio, IDTN and rate
+  difference of two groups with exact (Clopper-Pearson) intervals.
+- `bpskit.icd.read_conditions`: read and validate a condition table.
+
+These moved from the BPS sex-differences workflow, so a workflow needs no
+helper modules of its own.
+
 ## [0.2.0]
 
 ### Added
@@ -25,5 +37,6 @@ change the public API (the names in each module's `__all__`).
   BPS sex-differences workflow, and the `bpskit pullable` / `bpskit check-logs`
   command line.
 
+[0.3.0]: https://github.com/loucerac/bpskit/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/loucerac/bpskit/compare/v0.1.0...v0.2.0
 [0.1.0]: https://github.com/loucerac/bpskit/releases/tag/v0.1.0

{bpskit-0.2.0 → bpskit-0.3.0}/CITATION.cff

@@ -8,7 +8,7 @@ authors:
     email: clou@us.es
 license: MIT
 repository-code: "https://github.com/loucerac/bpskit"
-version: 0.2.0
+version: 0.3.0
 keywords:
   - epidemiology
   - electronic health records

{bpskit-0.2.0 → bpskit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: bpskit
-Version: 0.2.0
+Version: 0.3.0
 Summary: Tools for studies on a BPS (Base Poblacional de Salud) health-records delivery, including an identifier check for outputs leaving a secure data environment
 Project-URL: Homepage, https://github.com/loucerac/bpskit
 Project-URL: Changelog, https://github.com/loucerac/bpskit/blob/main/CHANGELOG.md
@@ -42,7 +42,9 @@ health-record extracts in a secure data environment generally.
 - **Person-time.** Dates as days since 1970, age bands, Lexis splitting,
   person-sorted streaming.
 - **Exact tests.** Poisson rate ratios (Garwood intervals), binomial tests,
-  BH and Bonferroni.
+  two-group rate contrasts with exact intervals, BH and Bonferroni.
+- **Indirect standardisation.** Person-time and expected counts by sex, age,
+  calendar year and prior-year contact intensity; within-person record pairs.
 - **An output check for leaving the secure environment.** `bpskit.transfer`
   refuses any file that contains an identifier the workflow read, or a
   per-person column, and fails closed when it cannot read something.
@@ -52,14 +54,14 @@ health-record extracts in a secure data environment generally.
 ## Installing
 
 ```bash
-pip install "bpskit @ git+https://github.com/loucerac/bpskit.git@v0.2.0"
+pip install bpskit
 ```
 
 In a pixi project (`pixi.toml`):
 
 ```toml
 [pypi-dependencies]
-bpskit = { git = "https://github.com/loucerac/bpskit.git", tag = "v0.2.0" }
+bpskit = "==0.3.0"
 ```
 
 In a Snakemake rule environment (`workflow/envs/*.yaml`):
@@ -68,7 +70,7 @@ In a Snakemake rule environment (`workflow/envs/*.yaml`):
 dependencies:
   - pip
   - pip:
-      - git+https://github.com/loucerac/bpskit.git@v0.2.0
+      - bpskit==0.3.0
 ```
 
 Pin a release. Bumping the tag changes the environment file, and Snakemake's

{bpskit-0.2.0 → bpskit-0.3.0}/README.md

@@ -14,7 +14,9 @@ health-record extracts in a secure data environment generally.
 - **Person-time.** Dates as days since 1970, age bands, Lexis splitting,
   person-sorted streaming.
 - **Exact tests.** Poisson rate ratios (Garwood intervals), binomial tests,
-  BH and Bonferroni.
+  two-group rate contrasts with exact intervals, BH and Bonferroni.
+- **Indirect standardisation.** Person-time and expected counts by sex, age,
+  calendar year and prior-year contact intensity; within-person record pairs.
 - **An output check for leaving the secure environment.** `bpskit.transfer`
   refuses any file that contains an identifier the workflow read, or a
   per-person column, and fails closed when it cannot read something.
@@ -24,14 +26,14 @@ health-record extracts in a secure data environment generally.
 ## Installing
 
 ```bash
-pip install "bpskit @ git+https://github.com/loucerac/bpskit.git@v0.2.0"
+pip install bpskit
 ```
 
 In a pixi project (`pixi.toml`):
 
 ```toml
 [pypi-dependencies]
-bpskit = { git = "https://github.com/loucerac/bpskit.git", tag = "v0.2.0" }
+bpskit = "==0.3.0"
 ```
 
 In a Snakemake rule environment (`workflow/envs/*.yaml`):
@@ -40,7 +42,7 @@ In a Snakemake rule environment (`workflow/envs/*.yaml`):
 dependencies:
   - pip
   - pip:
-      - git+https://github.com/loucerac/bpskit.git@v0.2.0
+      - bpskit==0.3.0
 ```
 
 Pin a release. Bumping the tag changes the environment file, and Snakemake's

{bpskit-0.2.0 → bpskit-0.3.0}/src/bpskit/__init__.py

@@ -3,7 +3,9 @@
     bpskit.delivery    read the pipe-delimited delivery: sentinels, dates, streaming
     bpskit.icd         ICD-10 code specs and chapters, ICD-9 -> ICD-10 mapping (CMS GEM)
     bpskit.persontime  dates as numbers, age bands, Lexis splitting, person streaming
-    bpskit.stats       exact Poisson and binomial tests, BH / Bonferroni
+    bpskit.stats       exact Poisson and binomial tests, exact two-group rate contrasts,
+                       BH / Bonferroni
+    bpskit.strata      strata for indirect standardisation, within-person record pairs
     bpskit.transfer    the check that no file leaving the data machine names a patient
     bpskit.synthetic   a synthetic BPS delivery with planted structure, for tests
 
@@ -11,4 +13,4 @@ The command line (`bpskit pullable`, `bpskit check-logs`) runs the transfer
 check on the data machine.
 """
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"

{bpskit-0.2.0 → bpskit-0.3.0}/src/bpskit/icd.py

@@ -5,6 +5,7 @@ Codes are handled in normalised form: upper case, no dot ('E119').
 
 import re
 from collections.abc import Iterable
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
@@ -189,3 +190,27 @@ def harmonise(df: pd.DataFrame, changeover_day: float | None, mapper: Icd9Mapper
         system[is9] = np.where(hit, "icd9_mapped", "icd9_unmapped")
         df.loc[hit[hit].index, "code"] = normalise_code(m[hit])
     return system
+
+
+def read_conditions(path: str | Path) -> pd.DataFrame:
+    """A condition table (TSV): one row per condition, its ICD-10 code spec.
+
+    Columns: condition_id ([a-z0-9_]+, so it can name a file or a Snakemake
+    wildcard), chapter_num, codes (a spec for `expand_spec`), and optionally
+    exclude; any other column is carried along. Lines starting with # are
+    comments.
+
+    Raises:
+        ValueError: a condition_id that cannot be a Snakemake wildcard value, a
+            duplicated id, or a code spec that does not parse.
+    """
+    c = pd.read_csv(path, sep="\t", comment="#", dtype=str, keep_default_na=False)
+    c["chapter_num"] = c["chapter_num"].astype(int)
+    bad = c.loc[~c.condition_id.str.fullmatch(r"[a-z0-9_]+"), "condition_id"]
+    if len(bad):
+        raise ValueError(f"condition_id not wildcard-safe: {list(bad)}")
+    if c.condition_id.duplicated().any():
+        raise ValueError("duplicated condition_id")
+    for spec in c["codes"]:
+        expand_spec(spec)
+    return c

{bpskit-0.2.0 → bpskit-0.3.0}/src/bpskit/stats.py

@@ -9,7 +9,15 @@ import pandas as pd
 from numpy.typing import ArrayLike
 from scipy import stats
 
-__all__ = ["bh", "adjust", "poisson_rr", "binom_half_p", "binom_p"]
+__all__ = [
+    "bh",
+    "adjust",
+    "poisson_rr",
+    "binom_half_p",
+    "binom_p",
+    "idtn_from_ratio",
+    "exact_contrast",
+]
 
 
 def bh(p: pd.Series) -> pd.Series:
@@ -74,3 +82,59 @@ def binom_p(k: ArrayLike, n: ArrayLike, p0: ArrayLike) -> np.ndarray:
     with np.errstate(invalid="ignore"):
         p = np.minimum(1.0, 2 * np.minimum(stats.binom.cdf(k, n, p0), stats.binom.sf(k - 1, n, p0)))
     return np.where((n > 0) & (p0 > 0) & (p0 < 1), p, np.nan)
+
+
+def idtn_from_ratio(r: ArrayLike) -> np.ndarray:
+    """IDTN from a rate ratio: (r - 1) / (r + 1), with an infinite ratio giving 1."""
+    r = np.asarray(r, float)
+    with np.errstate(invalid="ignore", divide="ignore"):
+        out = (r - 1.0) / (r + 1.0)
+    return np.where(np.isinf(r), 1.0, out)
+
+
+def exact_contrast(
+    cf: ArrayLike, pyf: ArrayLike, cm: ArrayLike, pym: ArrayLike, alpha: float
+) -> pd.DataFrame:
+    """Rate ratio, IDTN and rate difference of two groups, with exact intervals.
+
+    Written for female (f) against male (m), the names the output columns carry,
+    but any two groups work.
+
+    Conditional on the total count, female cases are binomial with
+    p = IRR*pyf / (IRR*pyf + pym). A Clopper-Pearson interval on p therefore
+    maps to an exact interval on the IRR, and through the monotone
+    (r-1)/(r+1) onto IDTN. Exact matters here: age bands at the extremes hold
+    a handful of cases, where a Wald interval on log IRR is badly wrong.
+
+    Rates are per 100,000 person-years. Cells with no cases or no person-time
+    in either sex come back NaN rather than as 0 or +-inf.
+    """
+    cf, cm = np.asarray(cf, float), np.asarray(cm, float)
+    pyf, pym = np.asarray(pyf, float), np.asarray(pym, float)
+    n = cf + cm
+    with np.errstate(invalid="ignore", divide="ignore"):
+        rf, rm = cf / pyf, cm / pym
+        irr = rf / rm
+        idtn = (rf - rm) / (rf + rm)
+        p_lo = np.where(cf > 0, stats.beta.ppf(alpha / 2, cf, n - cf + 1), 0.0)
+        p_hi = np.where(cf < n, stats.beta.ppf(1 - alpha / 2, cf + 1, n - cf), 1.0)
+        k = pym / pyf
+        irr_lo = p_lo / (1 - p_lo) * k
+        irr_hi = np.where(p_hi >= 1, np.inf, p_hi / (1 - p_hi) * k)
+    bad = (n == 0) | (pyf <= 0) | (pym <= 0)
+    out = pd.DataFrame(
+        {
+            "rate_f": rf * 1e5,
+            "rate_m": rm * 1e5,
+            "rd_per_100k": (rf - rm) * 1e5,
+            "irr": irr,
+            "irr_lo": irr_lo,
+            "irr_hi": irr_hi,
+            "idtn": idtn,
+            "idtn_lo": idtn_from_ratio(irr_lo),
+            "idtn_hi": idtn_from_ratio(irr_hi),
+        }
+    )
+    out.loc[bad, ["irr", "irr_lo", "irr_hi", "idtn", "idtn_lo", "idtn_hi"]] = np.nan
+    out.loc[(pyf <= 0) | (pym <= 0), ["rate_f", "rate_m", "rd_per_100k"]] = np.nan
+    return out

bpskit-0.3.0/src/bpskit/strata.py

@@ -0,0 +1,198 @@
+"""Strata for indirect standardisation, and every within-person pair of records.
+
+`Strata` splits person-time by sex, age band, calendar year and, optionally,
+prior-year healthcare-contact intensity; expected counts are person-time
+times a reference rate per stratum. `person_pairs` enumerates ordered pairs of
+a person's records in bounded blocks. Both were written for the disease-
+trajectory method (Jensen et al. 2014) and work for any indirect standardisation.
+"""
+
+from collections.abc import Iterator
+
+import numpy as np
+import pandas as pd
+
+from bpskit.persontime import DAYS_PER_YEAR, age_bands, band_index, day_of, lexis, to_days
+
+__all__ = ["Strata", "person_pairs"]
+
+
+class Strata:
+    """Sex x age band x calendar year x (optionally) prior-year contact intensity.
+
+    Stratum index s = ((sex * B + band) * Y + year) * C + level, sex 0 = F,
+    1 = M, so the F strata come first ("sex-major").
+
+    `contact_days` is a dense (persons x years) array of distinct record days in
+    the setting, first column = year `contact_year0`, cut at `contact_bins`
+    into C = len(bins) + 1 levels.
+
+    Why intensity, and why the previous year: people who visit more accumulate
+    more first records, so without it every pair looks associated. A yes/no
+    flag stratifies nothing when nearly everyone has a record every year. A
+    same-year count is partly made by the outcome, since D2's own first record
+    is a contact.
+    """
+
+    BLOCK = 1_000_000  # rows per Lexis split: the (rows x bands) arrays stay under ~200 MB
+
+    def __init__(
+        self,
+        bands: pd.DataFrame,
+        year_starts: np.ndarray,
+        contact_days=None,
+        contact_year0: int = 0,
+        contact_bins=(1,),
+    ):
+        self.bands = bands
+        self.B = len(bands)
+        self.ys = np.asarray(year_starts, float)  # Y + 1 boundaries, in days
+        self.Y = len(self.ys) - 1
+        self.year0 = int(pd.to_datetime(self.ys[0], unit="D").year)
+        self.coff = self.year0 - contact_year0 - 1  # column of the previous year
+        if contact_days is None:
+            self.levels, self.C = None, 1
+        else:
+            if self.coff < 0:
+                raise ValueError(
+                    "contact strata need a calendar year of records before the "
+                    "first stratum year: set index_from a year after washout_from"
+                )
+            bins = np.asarray(sorted(contact_bins))
+            self.levels = np.searchsorted(bins, contact_days, side="right").astype(np.uint8)
+            self.C = len(bins) + 1
+        self.S = 2 * self.B * self.Y * self.C
+
+    @classmethod
+    def for_analysis(
+        cls,
+        traj: dict,
+        ages: dict,
+        washout_from,
+        index_from,
+        last_date,
+        contacts=None,
+        year_starts=None,
+    ):
+        """The strata a trajectory analysis uses, built the same way by every rule.
+
+        `year_starts` defaults to 1 January of each year from `index_from`, with
+        the last boundary at the day after `last_date`.
+        """
+        if year_starts is None:
+            end = day_of(last_date) + 1
+            y0, y1 = pd.Timestamp(index_from).year, pd.Timestamp(last_date).year
+            year_starts = to_days(
+                pd.Series(pd.to_datetime([f"{y}-01-01" for y in range(y0, y1 + 2)]))
+            )
+            year_starts[-1] = end
+        return cls(
+            age_bands(ages["band_width"], ages["open_band_from"]),
+            year_starts,
+            contacts,
+            contact_year0=pd.Timestamp(washout_from).year,
+            contact_bins=traj["contact_bins"],
+        )
+
+    def level_shares(self) -> np.ndarray:
+        """Share of persons at each intensity level, per stratum year (Y x C)."""
+        if self.levels is None:
+            return np.ones((self.Y, 1))
+        cols = self.levels[:, self.coff : self.coff + self.Y]
+        return np.stack([(cols == c).mean(axis=0) for c in range(self.C)], axis=1)
+
+    def _contact(self, person, y):
+        if self.levels is None:
+            return 0
+        return self.levels[person, y + self.coff].astype(np.int64)
+
+    def at(self, day, sex, birth, person) -> np.ndarray:
+        """Stratum of single points in time (days outside the years are clipped)."""
+        y = np.clip(np.searchsorted(self.ys, day, side="right") - 1, 0, self.Y - 1)
+        b = band_index((day - birth) / DAYS_PER_YEAR, self.bands)
+        return ((sex * self.B + b) * self.Y + y) * self.C + self._contact(person, y)
+
+    def accumulate(self, acc: np.ndarray, key, sex, birth, entry, exit_, person) -> None:
+        """acc[key, s] += person-years of [entry, exit) falling in stratum s, in place."""
+        for lo in range(0, len(entry), self.BLOCK):
+            sl = slice(lo, lo + self.BLOCK)
+            self._accumulate(acc, key[sl], sex[sl], birth[sl], entry[sl], exit_[sl], person[sl])
+
+    def _accumulate(self, acc, key, sex, birth, entry, exit_, person) -> None:
+        flat = acc.reshape(-1)
+        for y in range(self.Y):
+            lo = np.maximum(entry, self.ys[y])
+            hi = np.minimum(exit_, self.ys[y + 1])
+            m = hi > lo
+            if not m.any():
+                continue
+            bm = birth[m]
+            pt = lexis((lo[m] - bm) / DAYS_PER_YEAR, (hi[m] - bm) / DAYS_PER_YEAR, self.bands)
+            r, b = np.nonzero(pt)
+            s = ((sex[m][r] * self.B + b) * self.Y + y) * self.C + self._contact(person[m][r], y)
+            flat += np.bincount(key[m][r] * self.S + s, weights=pt[r, b], minlength=flat.size)
+
+    def integrate(self, rate: np.ndarray, col, sex, birth, entry, exit_, person) -> np.ndarray:
+        """Expected count per row: sum over strata of person-years of [entry, exit) in s
+        times rate[s, col[row]].
+
+        Built exactly the way `accumulate` builds the bulk term, so a per-pair
+        correction computed with it can never exceed the bulk and drive an
+        expected count negative.
+        """
+        if len(entry) > self.BLOCK:
+            return np.concatenate(
+                [
+                    self.integrate(
+                        rate,
+                        col[i : i + self.BLOCK],
+                        sex[i : i + self.BLOCK],
+                        birth[i : i + self.BLOCK],
+                        entry[i : i + self.BLOCK],
+                        exit_[i : i + self.BLOCK],
+                        person[i : i + self.BLOCK],
+                    )
+                    for i in range(0, len(entry), self.BLOCK)
+                ]
+            )
+        out = np.zeros(len(entry))
+        for y in range(self.Y):
+            lo = np.maximum(entry, self.ys[y])
+            hi = np.minimum(exit_, self.ys[y + 1])
+            m = np.flatnonzero(hi > lo)
+            if not len(m):
+                continue
+            bm = birth[m]
+            pt = lexis((lo[m] - bm) / DAYS_PER_YEAR, (hi[m] - bm) / DAYS_PER_YEAR, self.bands)
+            r, b = np.nonzero(pt)
+            rows = m[r]
+            s = ((sex[rows] * self.B + b) * self.Y + y) * self.C + self._contact(person[rows], y)
+            out += np.bincount(rows, weights=pt[r, b] * rate[s, col[rows]], minlength=len(out))
+        return out
+
+
+def person_pairs(person: np.ndarray, max_pairs: int) -> Iterator[tuple[np.ndarray, np.ndarray]]:
+    """Every ordered pair of distinct rows of the same person.
+
+    `person` must be sorted. Yields (row_a, row_b) index arrays in blocks of
+    about `max_pairs` pairs; one person's pairs are never split across blocks,
+    so a single person with more than sqrt(max_pairs) records makes a block
+    bigger than `max_pairs`.
+    """
+    if not len(person):
+        return
+    starts = np.flatnonzero(np.r_[True, person[1:] != person[:-1]])
+    sizes = np.diff(np.r_[starts, len(person)]).astype(np.int64)
+    cum = np.cumsum(sizes * sizes)
+    g0 = 0
+    while g0 < len(starts):
+        base = cum[g0 - 1] if g0 else 0
+        g1 = max(int(np.searchsorted(cum, base + max_pairs, side="right")), g0 + 1)
+        st, sz = starts[g0:g1], sizes[g0:g1]
+        rows = np.arange(st[0], st[-1] + sz[-1])
+        rs, rz = np.repeat(st, sz), np.repeat(sz, sz)
+        a = np.repeat(rows, rz)
+        b = np.repeat(rs, rz) + np.arange(len(a)) - np.repeat(np.cumsum(rz) - rz, rz)
+        keep = a != b
+        yield a[keep], b[keep]
+        g0 = g1

bpskit-0.3.0/tests/test_contrast.py

@@ -0,0 +1,25 @@
+import numpy as np
+from scipy import stats
+
+from bpskit.stats import exact_contrast, idtn_from_ratio
+
+
+def test_irr_interval_is_exact_clopper_pearson():
+    cf, cm, pyf, pym = 7.0, 3.0, 1000.0, 2000.0
+    out = exact_contrast([cf], [pyf], [cm], [pym], alpha=0.05).iloc[0]
+    ci = stats.binomtest(int(cf), int(cf + cm)).proportion_ci(0.95, method="exact")
+    to_irr = lambda p: p / (1 - p) * pym / pyf  # noqa: E731
+    np.testing.assert_allclose([out.irr_lo, out.irr_hi], [to_irr(ci.low), to_irr(ci.high)])
+    np.testing.assert_allclose(out.irr, (cf / pyf) / (cm / pym))
+    np.testing.assert_allclose(out.idtn, idtn_from_ratio(out.irr))
+
+
+def test_empty_cells_are_nan_not_infinite():
+    out = exact_contrast([0, 5], [100, 0], [0, 1], [100, 100], alpha=0.05)
+    assert out[["irr", "idtn"]].isna().all().all()
+    assert np.isnan(out.rate_f.iloc[1])
+
+
+def test_idtn_of_infinite_ratio_is_one():
+    assert idtn_from_ratio(np.inf) == 1.0
+    assert idtn_from_ratio(1.0) == 0.0

bpskit-0.3.0/tests/test_strata.py

@@ -0,0 +1,35 @@
+import itertools
+
+import numpy as np
+
+from bpskit.persontime import age_bands
+from bpskit.strata import Strata, person_pairs
+
+
+def test_person_pairs_is_every_ordered_pair_within_a_person():
+    person = np.array([0, 0, 0, 1, 2, 2, 5, 5, 5, 5])
+    got = set()
+    for a, b in person_pairs(person, max_pairs=5):
+        got |= set(zip(a.tolist(), b.tolist()))
+    want = {
+        (i, j) for i, j in itertools.permutations(range(len(person)), 2) if person[i] == person[j]
+    }
+    assert got == want
+
+
+def test_strata_integrate_matches_accumulate():
+    bands = age_bands(5, 95)
+    ys = np.array([0.0, 365.0, 730.0, 1095.0])
+    st = Strata(bands, ys)
+    rng = np.random.default_rng(0)
+    n = 40
+    birth = rng.uniform(-30000, -10000, n)
+    entry = rng.uniform(0, 600, n)
+    exit_ = entry + rng.uniform(0, 600, n)
+    sex = rng.integers(0, 2, n)
+    person = np.arange(n)
+    acc = np.zeros((1, st.S))
+    st.accumulate(acc, np.zeros(n, np.int64), sex, birth, entry, exit_, person)
+    rate = rng.uniform(0, 1, (st.S, 1))
+    got = st.integrate(rate, np.zeros(n, np.int64), sex, birth, entry, exit_, person).sum()
+    np.testing.assert_allclose(got, acc[0] @ rate[:, 0])