sumeds 0.1.0__tar.gz

sumeds-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Robin van de Water
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sumeds-0.1.0/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: sumeds
+Version: 0.1.0
+Summary: Lazy, privacy-aware code occurrence summaries for MEDS datasets
+Keywords: MEDS,healthcare,Polars,privacy
+Author: Robin van de Water
+Author-email: Robin van de Water <r.p.vandewater@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
+Requires-Dist: meds>=0.4.1,<0.5
+Requires-Dist: polars>=1.30
+Requires-Dist: pyarrow>=15
+Requires-Dist: pyyaml>=6
+Requires-Python: >=3.10, <3.15
+Description-Content-Type: text/markdown
+
+# suMEDS
+
+`suMEDS` creates a compact code catalog for defining tasks on very large
+[Medical Event Data Standard (MEDS)](https://github.com/Medical-Event-Data-Standard/meds)
+datasets. It joins canonical descriptions and parent codes to event and unique-subject
+counts, then masks or removes rare codes before writing Parquet, CSV, or JSON.
+
+Polars scans, aggregations, joins, and writes stay lazy. Patient-level rows are never
+collected into Python memory.
+
+## Install and run
+
+```bash
+uv sync
+uv run suMEDS MIMICIV_DEMO/MEDS_cohort \
+  --output code-summary.parquet \
+  --min-subjects 20
+```
+
+Or use YAML:
+
+```yaml
+summary:
+  per_split: false
+  split_columns: false
+privacy:
+  min_subjects: 20
+  min_split_subjects: 1
+  rare_code_action: bucket
+  rare_code_label: __RARE__
+  round_counts_to: 5
+```
+
+```bash
+uv run suMEDS /path/to/MEDS -o summary.parquet -c examples/summary.yaml
+# The suffix selects CSV or JSON instead:
+uv run suMEDS /path/to/MEDS -o summary.json -c examples/summary.yaml
+```
+
+CLI flags override YAML. The defaults bucket codes seen in fewer than 20 unique
+subjects and retain exact counts.
+
+## Python API
+
+```python
+from sumeds import SummaryConfig, summarize
+
+path = summarize(
+    "/path/to/MEDS",
+    "code-summary.parquet",
+    SummaryConfig(min_subjects=20, rare_code_action="bucket"),
+)
+```
+
+The output format is inferred from `.parquet`, `.csv`, or `.json` (with
+`.jsonl`/`.ndjson` also supported). It preserves all code-metadata extension columns and adds
+`event_count`, `subject_count`, and `is_masked`. `per_split=True` emits one row
+per split. `split_columns=True` keeps the total row and adds columns such as
+`event_count_train` and `subject_count_train` from `subject_splits.parquet`.
+Set `min_split_subjects` above 1 to suppress rare split-level cells.
+
+## Documentation
+
+```bash
+uv run mkdocs serve
+uv run mkdocs build --strict
+```
+
+See the [documentation site](docs/index.md) for the data flow, complete CLI and
+YAML reference, output schema, Python API, and privacy limitations.
+
+## Development
+
+```bash
+uv run pre-commit run --all-files
+uv run pytest
+uv run mkdocs build --strict
+```
+
+The included integration test uses `MIMICIV_DEMO/MEDS_cohort` when present.

sumeds-0.1.0/README.md

@@ -0,0 +1,80 @@
+# suMEDS
+
+`suMEDS` creates a compact code catalog for defining tasks on very large
+[Medical Event Data Standard (MEDS)](https://github.com/Medical-Event-Data-Standard/meds)
+datasets. It joins canonical descriptions and parent codes to event and unique-subject
+counts, then masks or removes rare codes before writing Parquet, CSV, or JSON.
+
+Polars scans, aggregations, joins, and writes stay lazy. Patient-level rows are never
+collected into Python memory.
+
+## Install and run
+
+```bash
+uv sync
+uv run suMEDS MIMICIV_DEMO/MEDS_cohort \
+  --output code-summary.parquet \
+  --min-subjects 20
+```
+
+Or use YAML:
+
+```yaml
+summary:
+  per_split: false
+  split_columns: false
+privacy:
+  min_subjects: 20
+  min_split_subjects: 1
+  rare_code_action: bucket
+  rare_code_label: __RARE__
+  round_counts_to: 5
+```
+
+```bash
+uv run suMEDS /path/to/MEDS -o summary.parquet -c examples/summary.yaml
+# The suffix selects CSV or JSON instead:
+uv run suMEDS /path/to/MEDS -o summary.json -c examples/summary.yaml
+```
+
+CLI flags override YAML. The defaults bucket codes seen in fewer than 20 unique
+subjects and retain exact counts.
+
+## Python API
+
+```python
+from sumeds import SummaryConfig, summarize
+
+path = summarize(
+    "/path/to/MEDS",
+    "code-summary.parquet",
+    SummaryConfig(min_subjects=20, rare_code_action="bucket"),
+)
+```
+
+The output format is inferred from `.parquet`, `.csv`, or `.json` (with
+`.jsonl`/`.ndjson` also supported). It preserves all code-metadata extension columns and adds
+`event_count`, `subject_count`, and `is_masked`. `per_split=True` emits one row
+per split. `split_columns=True` keeps the total row and adds columns such as
+`event_count_train` and `subject_count_train` from `subject_splits.parquet`.
+Set `min_split_subjects` above 1 to suppress rare split-level cells.
+
+## Documentation
+
+```bash
+uv run mkdocs serve
+uv run mkdocs build --strict
+```
+
+See the [documentation site](docs/index.md) for the data flow, complete CLI and
+YAML reference, output schema, Python API, and privacy limitations.
+
+## Development
+
+```bash
+uv run pre-commit run --all-files
+uv run pytest
+uv run mkdocs build --strict
+```
+
+The included integration test uses `MIMICIV_DEMO/MEDS_cohort` when present.

sumeds-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "sumeds"
+version = "0.1.0"
+description = "Lazy, privacy-aware code occurrence summaries for MEDS datasets"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = ["MEDS", "healthcare", "Polars", "privacy"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering :: Medical Science Apps.",
+]
+authors = [
+    { name = "Robin van de Water", email = "r.p.vandewater@gmail.com" }
+]
+requires-python = ">=3.10, <3.15"
+dependencies = [
+    "meds>=0.4.1,<0.5",
+    "polars>=1.30",
+    "pyarrow>=15",
+    "pyyaml>=6",
+]
+
+[project.scripts]
+suMEDS = "sumeds.cli:main"
+
+[build-system]
+requires = ["uv_build>=0.11.15,<0.12.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "mkdocs>=1.6",
+    "pre-commit>=4.0",
+    "pytest>=8",
+    "ruff>=0.15.22",
+]

sumeds-0.1.0/src/sumeds/__init__.py

@@ -0,0 +1,26 @@
+"""Compact, lazy summaries for Medical Event Data Standard datasets."""
+
+from .config import SummaryConfig
+from .scan import (
+    code_modifier_columns,
+    dataset_root,
+    event_files,
+    read_dataset_metadata,
+    scan_code_metadata,
+    scan_events,
+    scan_subject_splits,
+)
+from .summary import code_occurrences, summarize
+
+__all__ = [
+    "SummaryConfig",
+    "code_modifier_columns",
+    "code_occurrences",
+    "dataset_root",
+    "event_files",
+    "read_dataset_metadata",
+    "scan_code_metadata",
+    "scan_events",
+    "scan_subject_splits",
+    "summarize",
+]

sumeds-0.1.0/src/sumeds/cli.py

@@ -0,0 +1,87 @@
+"""Command-line interface for :mod:`sumeds`."""
+
+from __future__ import annotations
+
+import argparse
+from importlib.metadata import version
+from pathlib import Path
+from typing import Sequence
+
+from .config import SummaryConfig
+from .summary import summarize
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the public CLI parser."""
+
+    parser = argparse.ArgumentParser(
+        prog="suMEDS",
+        description="Create a lazy, privacy-aware MEDS code occurrence catalog.",
+    )
+    parser.add_argument("dataset_root", type=Path, help="MEDS dataset root")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        required=True,
+        help="output .parquet, .csv, or .json file",
+    )
+    parser.add_argument("-c", "--config", type=Path, help="YAML configuration file")
+    parser.add_argument(
+        "--per-split", action=argparse.BooleanOptionalAction, default=None
+    )
+    parser.add_argument(
+        "--split-columns",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="add per-split count columns alongside totals",
+    )
+    parser.add_argument(
+        "--min-subjects", type=int, help="minimum unique subjects per released code"
+    )
+    parser.add_argument(
+        "--min-split-subjects",
+        type=int,
+        help="minimum unique subjects per visible wide split cell",
+    )
+    parser.add_argument("--rare-code-action", choices=("bucket", "drop"))
+    parser.add_argument("--rare-code-label", help="label used by bucket mode")
+    parser.add_argument(
+        "--round-counts-to", type=int, help="round released counts to this multiple"
+    )
+    parser.add_argument("--version", action="version", version=version("suMEDS"))
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the CLI and return a process exit code."""
+
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        config = (
+            SummaryConfig.from_yaml(args.config) if args.config else SummaryConfig()
+        )
+        config = config.with_overrides(
+            per_split=args.per_split,
+            split_columns=args.split_columns,
+            min_subjects=args.min_subjects,
+            min_split_subjects=args.min_split_subjects,
+            rare_code_action=args.rare_code_action,
+            rare_code_label=args.rare_code_label,
+            round_counts_to=args.round_counts_to,
+        )
+    except (OSError, ValueError) as exc:
+        parser.error(str(exc))
+
+    try:
+        output = summarize(args.dataset_root, args.output, config)
+    except Exception as exc:  # CLI boundary: library callers retain typed exceptions.
+        parser.exit(1, f"suMEDS: error: {exc}\n")
+
+    print(f"Wrote {output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sumeds-0.1.0/src/sumeds/config.py

@@ -0,0 +1,90 @@
+"""Configuration for MEDS summaries."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, fields, replace
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+@dataclass(frozen=True)
+class SummaryConfig:
+    """Options controlling grouping and privacy-aware release.
+
+    ``min_subjects`` is evaluated on unique subjects, not event rows. A value
+    of one disables useful rarity protection and should only be used on public
+    or synthetic data.
+    """
+
+    per_split: bool = False
+    split_columns: bool = False
+    min_subjects: int = 20
+    min_split_subjects: int = 1
+    rare_code_action: str = "bucket"
+    rare_code_label: str = "__RARE__"
+    round_counts_to: int | None = None
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.per_split, bool) or not isinstance(
+            self.split_columns, bool
+        ):
+            raise ValueError("per_split and split_columns must be true or false")
+        if self.per_split and self.split_columns:
+            raise ValueError("per_split and split_columns are mutually exclusive")
+        for name in ("min_subjects", "min_split_subjects"):
+            value = getattr(self, name)
+            if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+                raise ValueError(f"{name} must be a positive integer")
+        if self.rare_code_action not in {"bucket", "drop"}:
+            raise ValueError("rare_code_action must be 'bucket' or 'drop'")
+        if not self.rare_code_label:
+            raise ValueError("rare_code_label must not be empty")
+        if self.round_counts_to is not None and (
+            not isinstance(self.round_counts_to, int)
+            or isinstance(self.round_counts_to, bool)
+            or self.round_counts_to < 1
+        ):
+            raise ValueError("round_counts_to must be null or a positive integer")
+
+    @classmethod
+    def from_yaml(cls, path: str | Path) -> SummaryConfig:
+        """Load a strict ``summary``/``privacy`` YAML configuration file."""
+
+        source = Path(path)
+        raw = yaml.safe_load(source.read_text()) or {}
+        if not isinstance(raw, dict):
+            raise ValueError(f"YAML root must be a mapping: {source}")
+        unknown_sections = set(raw) - {"summary", "privacy"}
+        if unknown_sections:
+            raise ValueError(f"Unknown YAML sections: {sorted(unknown_sections)}")
+
+        summary = _mapping(raw.get("summary"), "summary")
+        privacy = _mapping(raw.get("privacy"), "privacy")
+        allowed = {field.name for field in fields(cls)}
+        summary_options = {"per_split", "split_columns"}
+        unknown = (set(summary) - summary_options) | (
+            set(privacy) - (allowed - summary_options)
+        )
+        if unknown:
+            raise ValueError(
+                f"Unknown configuration options or wrong sections: {sorted(unknown)}"
+            )
+        values: dict[str, Any] = {**summary, **privacy}
+        return cls(**values)
+
+    def with_overrides(self, **values: Any) -> SummaryConfig:
+        """Return a copy with non-``None`` CLI overrides applied."""
+
+        return replace(
+            self, **{key: value for key, value in values.items() if value is not None}
+        )
+
+
+def _mapping(value: Any, name: str) -> dict[str, Any]:
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ValueError(f"YAML section '{name}' must be a mapping")
+    return value

sumeds-0.1.0/src/sumeds/scan.py

@@ -0,0 +1,110 @@
+"""Validated, projection-friendly MEDS scans."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Sequence
+
+import meds
+import polars as pl
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+def dataset_root(path: str | Path) -> Path:
+    """Return a validated MEDS root containing ``data`` and ``metadata``."""
+
+    root = Path(path).expanduser().resolve()
+    for child in (meds.data_subdirectory, Path(meds.dataset_metadata_filepath).parent):
+        if not (root / child).is_dir():
+            raise FileNotFoundError(f"MEDS directory not found: {root / child}")
+    return root
+
+
+def read_dataset_metadata(root: str | Path) -> dict[str, Any]:
+    """Read and validate MEDS ``metadata/dataset.json``."""
+
+    path = dataset_root(root) / meds.dataset_metadata_filepath
+    if not path.is_file():
+        raise FileNotFoundError(f"MEDS dataset metadata not found: {path}")
+    value = json.loads(path.read_text())
+    meds.DatasetMetadataSchema.validate(value)
+    return value
+
+
+def code_modifier_columns(metadata: dict[str, Any]) -> list[str]:
+    """Return declared code modifiers, rejecting duplicates and invalid names."""
+
+    modifiers = metadata.get("code_modifier_columns", [])
+    if not isinstance(modifiers, list) or not all(
+        isinstance(item, str) and item for item in modifiers
+    ):
+        raise ValueError("code_modifier_columns must be a list of non-empty strings")
+    if len(modifiers) != len(set(modifiers)):
+        raise ValueError("code_modifier_columns contains duplicates")
+    if set(modifiers) & {"subject_id", "time", "code"}:
+        raise ValueError("code_modifier_columns contains a core MEDS column")
+    return modifiers
+
+
+def event_files(root: str | Path) -> list[Path]:
+    """List event shards beneath the standard MEDS data directory."""
+
+    data = dataset_root(root) / meds.data_subdirectory
+    files = sorted(data.rglob("*.parquet"))
+    if not files:
+        raise FileNotFoundError(f"No MEDS Parquet shards found below {data}")
+    return files
+
+
+def scan_events(root: str | Path, modifiers: Sequence[str] = ()) -> pl.LazyFrame:
+    """Validate shard footers and lazily project fields needed for counting."""
+
+    files = event_files(root)
+    for path in files:
+        schema = pq.read_schema(path)
+        meds.DataSchema.validate(schema)
+        for column in modifiers:
+            if column not in schema.names:
+                raise ValueError(
+                    f"Declared code modifier '{column}' is missing from {path}"
+                )
+            if not (
+                pa.types.is_string(schema.field(column).type)
+                or pa.types.is_large_string(schema.field(column).type)
+            ):
+                raise ValueError(f"Code modifier '{column}' must be a string in {path}")
+    columns = ["subject_id", "code", *modifiers]
+    return pl.scan_parquet([str(path) for path in files]).select(columns)
+
+
+def scan_code_metadata(root: str | Path) -> pl.LazyFrame:
+    """Validate and lazily scan the canonical code metadata table.
+
+    Optional standard columns are added as typed nulls so the summary output
+    always includes descriptions and parent-code relationships.
+    """
+
+    path = dataset_root(root) / meds.code_metadata_filepath
+    if not path.is_file():
+        raise FileNotFoundError(f"MEDS code metadata not found: {path}")
+    meds.CodeMetadataSchema.validate(pq.read_schema(path))
+    frame = pl.scan_parquet(path)
+    names = set(frame.collect_schema().names())
+    missing = []
+    if "description" not in names:
+        missing.append(pl.lit(None, dtype=pl.String).alias("description"))
+    if "parent_codes" not in names:
+        missing.append(pl.lit(None, dtype=pl.List(pl.String)).alias("parent_codes"))
+    return frame.with_columns(missing) if missing else frame
+
+
+def scan_subject_splits(root: str | Path) -> pl.LazyFrame:
+    """Validate and lazily scan canonical subject-to-split assignments."""
+
+    path = dataset_root(root) / meds.subject_splits_filepath
+    if not path.is_file():
+        raise FileNotFoundError(f"MEDS subject splits not found: {path}")
+    meds.SubjectSplitSchema.validate(pq.read_schema(path))
+    return pl.scan_parquet(path).select("subject_id", "split")

sumeds-0.1.0/src/sumeds/summary.py

@@ -0,0 +1,352 @@
+"""Lazy code occurrence summaries with rare-code masking."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Sequence
+from uuid import uuid4
+
+import polars as pl
+
+from .config import SummaryConfig
+from .scan import (
+    code_modifier_columns,
+    dataset_root,
+    read_dataset_metadata,
+    scan_code_metadata,
+    scan_events,
+    scan_subject_splits,
+)
+
+_COUNT_COLUMNS = ("event_count", "subject_count")
+_RESERVED_COLUMNS = {*_COUNT_COLUMNS, "is_masked", "split"}
+_OUTPUT_FORMATS = {
+    ".parquet": "parquet",
+    ".csv": "csv",
+    ".json": "json",
+    ".jsonl": "ndjson",
+    ".ndjson": "ndjson",
+}
+
+
+def code_occurrences(
+    events: pl.LazyFrame,
+    keys: Sequence[str],
+    splits: Sequence[str] = (),
+) -> pl.LazyFrame:
+    """Count total and optional named-split events and unique subjects lazily."""
+
+    return _code_occurrences(events, keys, _split_columns(splits))
+
+
+def _code_occurrences(
+    events: pl.LazyFrame,
+    keys: Sequence[str],
+    split_columns: Sequence[tuple[str, str, str]],
+) -> pl.LazyFrame:
+    expressions = _count_expressions(split_columns)
+    if split_columns:
+        expressions.append(
+            pl.col("split")
+            .is_null()
+            .sum()
+            .cast(pl.UInt64)
+            .alias("_missing_split_count")
+        )
+    return events.group_by(list(keys)).agg(expressions)
+
+
+def summarize(
+    root: str | Path,
+    output: str | Path,
+    config: SummaryConfig | None = None,
+) -> Path:
+    """Write a privacy-aware MEDS code catalog and return its path.
+
+    Event rows remain in Polars lazy plans and are streamed to bounded-memory
+    outputs. Bucket mode performs a second projected event scan so its subject
+    count is the exact union of subjects across rare codes.
+    """
+
+    config = config or SummaryConfig()
+    root = dataset_root(root)
+    output = Path(output).expanduser().resolve()
+    output_format = _OUTPUT_FORMATS.get(output.suffix.lower())
+    if output_format is None:
+        raise ValueError(
+            f"unsupported output suffix {output.suffix!r}; use Parquet, CSV, or JSON"
+        )
+    if output.is_relative_to(root / "data") or output.is_relative_to(root / "metadata"):
+        raise ValueError("output must not overwrite MEDS data or metadata")
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    metadata_json = read_dataset_metadata(root)
+    modifiers = code_modifier_columns(metadata_json)
+    metadata = scan_code_metadata(root)
+    events = scan_events(root, modifiers)
+    scope: list[str] = []
+    split_columns: list[tuple[str, str, str]] = []
+    if config.per_split or config.split_columns:
+        splits = scan_subject_splits(root)
+        duplicate = (
+            splits.group_by("subject_id").len().filter(pl.col("len") > 1).limit(1)
+        )
+        _raise_if_any(duplicate, "subject_splits contains duplicate subject IDs")
+        if config.split_columns:
+            split_values = (
+                splits.select("split")
+                .unique()
+                .sort("split")
+                .collect(engine="streaming")
+                .get_column("split")
+                .to_list()
+            )
+            if not split_values:
+                raise ValueError("subject_splits contains no splits")
+            split_columns = _split_columns(split_values)
+        events = events.join(splits, on="subject_id", how="left")
+        if config.per_split:
+            scope.append("split")
+
+    base_keys = ["code", *modifiers]
+    count_keys = [*scope, *base_keys]
+    split_count_names = _split_count_names(split_columns)
+    _validate_metadata(metadata, modifiers, config, split_count_names)
+    token = uuid4().hex
+    counts_path = output.with_name(f".{output.name}.{token}.counts.parquet")
+    staged_path = output.with_name(f".{output.name}.{token}.tmp{output.suffix}")
+
+    try:
+        _code_occurrences(events, count_keys, split_columns).sink_parquet(counts_path)
+        counts = pl.scan_parquet(counts_path)
+        if config.per_split:
+            missing_split = counts.filter(pl.col("split").is_null()).limit(1)
+        elif config.split_columns:
+            missing_split = counts.filter(pl.col("_missing_split_count") > 0).limit(1)
+            counts = counts.drop("_missing_split_count")
+        else:
+            missing_split = None
+        if missing_split is not None:
+            _raise_if_any(
+                missing_split,
+                "subject_splits does not cover every event subject",
+            )
+        safe = counts.filter(pl.col("subject_count") >= config.min_subjects)
+        if split_columns:
+            safe = _mask_split_counts(safe, split_columns, config.min_split_subjects)
+        common = safe.join(
+            metadata, on=base_keys, how="left", nulls_equal=True
+        ).with_columns(pl.lit(False).alias("is_masked"))
+        count_columns = [*_COUNT_COLUMNS, *split_count_names]
+        columns = [
+            *scope,
+            *metadata.collect_schema().names(),
+            *count_columns,
+            "is_masked",
+        ]
+        common = _round_counts(common, config.round_counts_to, count_columns).select(
+            columns
+        )
+
+        frames = [common]
+        if config.rare_code_action == "bucket":
+            rare_keys = counts.filter(
+                pl.col("subject_count") < config.min_subjects
+            ).select(count_keys)
+            rare_events = events.join(
+                rare_keys, on=count_keys, how="semi", nulls_equal=True
+            )
+            bucket = _rare_bucket(
+                rare_events, metadata, scope, split_columns, config
+            ).select(columns)
+            frames.append(bucket)
+
+        result = pl.concat(frames, how="vertical_relaxed").sort(
+            [*scope, "code", *modifiers]
+        )
+        _sink_output(result, staged_path, output_format)
+        staged_path.replace(output)
+    finally:
+        counts_path.unlink(missing_ok=True)
+        staged_path.unlink(missing_ok=True)
+
+    return output
+
+
+def _split_columns(values: Sequence[str]) -> list[tuple[str, str, str]]:
+    result = []
+    used = set()
+    for value in values:
+        suffix = re.sub(r"\W+", "_", value).strip("_").lower() or "unnamed"
+        names = (f"event_count_{suffix}", f"subject_count_{suffix}")
+        if used & set(names):
+            raise ValueError("split names produce duplicate output columns")
+        used.update(names)
+        result.append((value, *names))
+    return result
+
+
+def _split_count_names(split_columns: Sequence[tuple[str, str, str]]) -> list[str]:
+    return [name for columns in split_columns for name in columns[1:]]
+
+
+def _count_expressions(
+    split_columns: Sequence[tuple[str, str, str]],
+) -> list[pl.Expr]:
+    expressions = [
+        pl.len().cast(pl.UInt64).alias("event_count"),
+        pl.col("subject_id").n_unique().cast(pl.UInt64).alias("subject_count"),
+    ]
+    for value, event_name, subject_name in split_columns:
+        in_split = pl.col("split") == value
+        expressions.extend(
+            [
+                in_split.sum().cast(pl.UInt64).alias(event_name),
+                pl.col("subject_id")
+                .filter(in_split)
+                .n_unique()
+                .cast(pl.UInt64)
+                .alias(subject_name),
+            ]
+        )
+    return expressions
+
+
+def _mask_split_counts(
+    frame: pl.LazyFrame,
+    split_columns: Sequence[tuple[str, str, str]],
+    min_subjects: int,
+) -> pl.LazyFrame:
+    if min_subjects == 1:
+        return frame
+    return frame.with_columns(
+        expression
+        for _, event_name, subject_name in split_columns
+        for expression in (
+            pl.when(pl.col(subject_name) >= min_subjects)
+            .then(pl.col(event_name))
+            .otherwise(None)
+            .alias(event_name),
+            pl.when(pl.col(subject_name) >= min_subjects)
+            .then(pl.col(subject_name))
+            .otherwise(None)
+            .alias(subject_name),
+        )
+    )
+
+
+def _sink_output(frame: pl.LazyFrame, path: Path, output_format: str) -> None:
+    if output_format == "parquet":
+        frame.sink_parquet(path)
+    elif output_format == "csv":
+        schema = frame.collect_schema()
+        nested = []
+        for name, dtype in schema.items():
+            column = pl.col(name)
+            if isinstance(dtype, pl.Array):
+                column = column.arr.to_list()
+                dtype = pl.List(dtype.inner)
+            if isinstance(dtype, pl.List):
+                nested.append(
+                    column.list.eval(pl.element().cast(pl.String))
+                    .list.join("|")
+                    .alias(name)
+                )
+            elif isinstance(dtype, pl.Struct):
+                nested.append(column.struct.json_encode().alias(name))
+        frame.with_columns(nested).sink_csv(path)
+    elif output_format == "ndjson":
+        frame.sink_ndjson(path)
+    else:
+        lines = path.with_suffix(path.suffix + ".ndjson")
+        try:
+            frame.sink_ndjson(lines)
+            with lines.open("rb") as source, path.open("wb") as target:
+                target.write(b"[")
+                separator = b""
+                for line in source:
+                    line = line.strip()
+                    if line:
+                        target.write(separator + line)
+                        separator = b",\n"
+                target.write(b"]\n")
+        finally:
+            lines.unlink(missing_ok=True)
+
+
+def _validate_metadata(
+    metadata: pl.LazyFrame,
+    modifiers: Sequence[str],
+    config: SummaryConfig,
+    extra_reserved: Sequence[str] = (),
+) -> None:
+    names = set(metadata.collect_schema().names())
+    missing = set(modifiers) - names
+    if missing:
+        raise ValueError(
+            f"Code metadata is missing declared modifiers: {sorted(missing)}"
+        )
+    collisions = names & (_RESERVED_COLUMNS | set(extra_reserved))
+    if collisions:
+        raise ValueError(
+            f"Code metadata uses reserved output columns: {sorted(collisions)}"
+        )
+    keys = ["code", *modifiers]
+    duplicate = metadata.group_by(keys).len().filter(pl.col("len") > 1).limit(1)
+    _raise_if_any(duplicate, f"Code metadata keys are not unique: {keys}")
+    if config.rare_code_action == "bucket":
+        collision = metadata.filter(pl.col("code") == config.rare_code_label).limit(1)
+        _raise_if_any(
+            collision, f"rare_code_label already exists: {config.rare_code_label!r}"
+        )
+
+
+def _rare_bucket(
+    events: pl.LazyFrame,
+    metadata: pl.LazyFrame,
+    scope: Sequence[str],
+    split_columns: Sequence[tuple[str, str, str]],
+    config: SummaryConfig,
+) -> pl.LazyFrame:
+    expressions = _count_expressions(split_columns)
+    if scope:
+        result = events.group_by(list(scope)).agg(expressions)
+    else:
+        result = events.select(expressions)
+    result = result.filter(pl.col("subject_count") >= config.min_subjects)
+    if split_columns:
+        result = _mask_split_counts(result, split_columns, config.min_split_subjects)
+
+    schema = metadata.collect_schema()
+    values = [
+        pl.lit(config.rare_code_label, dtype=dtype).alias(name)
+        if name == "code"
+        else pl.lit(None, dtype=dtype).alias(name)
+        for name, dtype in schema.items()
+    ]
+    return _round_counts(
+        result.with_columns(*values, pl.lit(True).alias("is_masked")),
+        config.round_counts_to,
+        [*_COUNT_COLUMNS, *_split_count_names(split_columns)],
+    )
+
+
+def _round_counts(
+    frame: pl.LazyFrame,
+    multiple: int | None,
+    columns: Sequence[str] = _COUNT_COLUMNS,
+) -> pl.LazyFrame:
+    if not multiple or multiple == 1:
+        return frame
+    return frame.with_columns(
+        (((pl.col(name) + multiple // 2) // multiple) * multiple)
+        .cast(pl.UInt64)
+        .alias(name)
+        for name in columns
+    )
+
+
+def _raise_if_any(frame: pl.LazyFrame, message: str) -> None:
+    if frame.collect(engine="streaming").height:
+        raise ValueError(message)