calm-benchmark 0.1.3__py3-none-any.whl

calm/__init__.py

@@ -0,0 +1,41 @@
+"""Public API for CALM's reproducible menopausal-status benchmark engine."""
+
+from calm.cli import app
+from calm.config.loading import load_config
+from calm.config.models import RunConfig
+from calm.pipeline.attribution import FeatureAttribution, PredictionAttribution, attribute
+from calm.pipeline.calibrate import CalibratedModel, PredictionResult, calibrate, predict
+from calm.pipeline.evaluate import EvaluationResult, Subgroup, evaluate
+from calm.pipeline.features import select_features
+from calm.pipeline.labels import LabelResult, build_labels, get_label_rule
+from calm.pipeline.report import ReportArtifacts, ReportWriter
+from calm.pipeline.split import FrozenSplit, stratified_split
+from calm.pipeline.train import TrainedModel, train
+
+__version__ = "0.1.3"
+
+__all__ = [
+    "CalibratedModel",
+    "FeatureAttribution",
+    "EvaluationResult",
+    "FrozenSplit",
+    "LabelResult",
+    "PredictionResult",
+    "PredictionAttribution",
+    "ReportArtifacts",
+    "ReportWriter",
+    "RunConfig",
+    "Subgroup",
+    "TrainedModel",
+    "app",
+    "attribute",
+    "build_labels",
+    "calibrate",
+    "evaluate",
+    "get_label_rule",
+    "load_config",
+    "predict",
+    "select_features",
+    "stratified_split",
+    "train",
+]

calm/adapters/__init__.py

@@ -0,0 +1,6 @@
+"""Concrete adapters for external systems."""
+
+from calm.adapters.nhanes import NhanesSource
+from calm.adapters.synthetic import SyntheticSource
+
+__all__ = ["NhanesSource", "SyntheticSource"]

calm/adapters/codebook.py

@@ -0,0 +1,30 @@
+"""Official NCHS HTML codebook metadata ingestion."""
+
+from bs4 import BeautifulSoup
+
+from calm.domain.entities import ValueLabel, VariableCode, VariableMetadata
+
+
+def parse_codebook(html: str) -> tuple[VariableMetadata, ...]:
+    """Extract variable labels and coded response labels from an NCHS codebook page."""
+    soup = BeautifulSoup(html, "html.parser")
+    variables: list[VariableMetadata] = []
+    for heading in soup.find_all("h3"):
+        text = heading.get_text(" ", strip=True)
+        code, separator, label = text.partition(" - ")
+        if not separator or not code.isupper():
+            continue
+        table = heading.find_next("table")
+        value_labels: list[ValueLabel] = []
+        if table is not None:
+            for row in table.find_all("tr")[1:]:
+                cells = row.find_all("td")
+                if len(cells) >= 2:
+                    value_labels.append(
+                        ValueLabel(
+                            cells[0].get_text(" ", strip=True),
+                            cells[1].get_text(" ", strip=True),
+                        )
+                    )
+        variables.append(VariableMetadata(VariableCode(code), label, tuple(value_labels)))
+    return tuple(variables)

calm/adapters/nhanes.py

@@ -0,0 +1,70 @@
+"""NHANES raw-file acquisition adapter."""
+
+from collections.abc import Callable
+from datetime import datetime
+from pathlib import Path
+from typing import Protocol, cast
+from urllib.request import urlopen
+
+from calm.domain.entities import CycleSpec, RawBundle, RawFile
+from calm.observability.hashing import hash_bytes
+
+
+class ByteFetcher(Protocol):
+    """Fetch the bytes at a configured source URL."""
+
+    def __call__(self, url: str) -> bytes:
+        """Return source bytes for a URL."""
+
+
+def fetch_url_bytes(url: str) -> bytes:
+    """Fetch raw bytes from a configured URL.
+
+    Args:
+        url: Direct source URL explicitly supplied by orchestration.
+
+    Returns:
+        Response bytes.
+    """
+    with urlopen(url) as response:  # noqa: S310
+        return cast(bytes, response.read())
+
+
+class NhanesSource:
+    """Acquire configured NHANES XPT components and record their lineage."""
+
+    def __init__(
+        self,
+        destination: Path,
+        components: tuple[str, ...],
+        url_for: Callable[[CycleSpec, str], str],
+        clock: Callable[[], datetime],
+        fetcher: ByteFetcher = fetch_url_bytes,
+    ) -> None:
+        """Initialize acquisition dependencies supplied by orchestration."""
+        self._destination = destination
+        self._components = components
+        self._url_for = url_for
+        self._clock = clock
+        self._fetcher = fetcher
+
+    def fetch(self, cycle: CycleSpec) -> RawBundle:
+        """Fetch configured components for a cycle and persist their exact bytes."""
+        files: list[RawFile] = []
+        cycle_directory = self._destination / cycle.identifier
+        cycle_directory.mkdir(parents=True, exist_ok=True)
+        for component in self._components:
+            source_url = self._url_for(cycle, component)
+            payload = self._fetcher(source_url)
+            local_path = cycle_directory / f"{component}.xpt"
+            local_path.write_bytes(payload)
+            files.append(
+                RawFile(
+                    component=component,
+                    source_url=source_url,
+                    checksum=str(hash_bytes(payload)),
+                    local_path=local_path,
+                    fetched_at=self._clock(),
+                )
+            )
+        return RawBundle(cycle=cycle, files=tuple(files))

calm/adapters/nhanes_2017_2018.py

@@ -0,0 +1,118 @@
+"""Pinned local adapter and metadata contract for public NHANES 2017--2018."""
+
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from pathlib import Path
+
+from calm.domain.entities import (
+    CycleSpec,
+    RawBundle,
+    RawFile,
+    ValueLabel,
+    VariableCode,
+    VariableMetadata,
+)
+from calm.domain.errors import DomainVerificationError
+from calm.observability.hashing import hash_bytes
+
+_BASE_URL = "https://wwwn.cdc.gov/Nchs/Data/Nhanes/Public/2017/DataFiles"
+
+
+class Nhanes2017_2018Source:
+    """Read the pinned public 2017--2018 files already present on local storage."""
+
+    _files = {
+        "demographics": (
+            "DEMO_J.XPT",
+            "c0b46e0345ea19404928656277c8b0d10b0cca348a9b2fe4fc3c67e8b7ee73ec",
+        ),  # noqa: E501
+        "questionnaire": (
+            "RHQ_J.XPT",
+            "c919896216154ea207470dd84e428f5fdd65ced39735148c78fb6032e0b77a7b",
+        ),  # noqa: E501
+        "body_measures": (
+            "BMX_J.XPT",
+            "8d675e42d8826ac98714b2c3dd4c5138a5e353fb4424f7eff5e6db4a01ce838a",
+        ),  # noqa: E501
+        "blood_pressure": (
+            "BPX_J.XPT",
+            "f98f1d9a4c18173e715c749b4294f6f1525140d5ad5751316bb17cc038fb3423",
+        ),  # noqa: E501
+        "total_cholesterol": (
+            "TCHOL_J.XPT",
+            "0291a6a4f6d82dac8392c8c992b519946824dfeda01647935520cf506ed9f4ab",
+        ),  # noqa: E501
+        "hdl": ("HDL_J.XPT", "9f4eb41b89f0c9f3d6262f873e671eeb2f2eb40581a8426b2a3c8fd4fc19439e"),  # noqa: E501
+        "glycated_hemoglobin": (
+            "GHB_J.XPT",
+            "35f07094573a0061a03ed609a5a363b34eb1b1c7065d1623b43d72e132a8a654",
+        ),  # noqa: E501
+        "complete_blood_count": (
+            "CBC_J.XPT",
+            "00f964098dc91272e415344554cbf1b627ecaeb2eda201518527446a7d81e742",
+        ),  # noqa: E501
+    }
+
+    def __init__(self, directory: Path) -> None:
+        """Initialize the directory containing the downloaded official XPT files."""
+        self._directory = directory
+
+    def fetch(self, cycle: CycleSpec) -> RawBundle:
+        """Validate and expose the immutable local files for the supported cycle."""
+        if cycle.identifier != "2017-2018":
+            raise DomainVerificationError(f"unsupported NHANES cycle: {cycle.identifier}")
+        files: list[RawFile] = []
+        for component, (filename, expected_checksum) in self._files.items():
+            path = self._directory / filename
+            if not path.is_file():
+                raise DomainVerificationError(f"missing pinned NHANES input: {path}")
+            checksum = str(hash_bytes(path.read_bytes()))
+            if checksum != expected_checksum:
+                raise DomainVerificationError(
+                    f"checksum mismatch for pinned NHANES input: {filename}"
+                )
+            files.append(
+                RawFile(
+                    component=component,
+                    source_url=f"{_BASE_URL}/{filename}",
+                    checksum=checksum,
+                    local_path=path,
+                    fetched_at=datetime.fromtimestamp(path.stat().st_mtime, UTC),
+                )
+            )
+        return RawBundle(cycle, tuple(files))
+
+
+def questionnaire_metadata_2017_2018() -> tuple[VariableMetadata, ...]:
+    """Return the versioned RHQ codebook contract needed by the public label rule."""
+    return (
+        VariableMetadata(VariableCode("SEQN"), "Respondent sequence number"),
+        VariableMetadata(
+            VariableCode("RHQ031"),
+            "Had regular periods in past 12 months",
+            (ValueLabel("1", "Yes"), ValueLabel("2", "No")),
+        ),
+        VariableMetadata(
+            VariableCode("RHD043"),
+            "Reason not having regular periods",
+            (
+                ValueLabel("1", "Pregnancy"),
+                ValueLabel("2", "Breast feeding"),
+                ValueLabel("3", "Hysterectomy"),
+                ValueLabel("7", "Menopause/Change of life"),
+            ),
+        ),
+    )
+
+
+def validate_2017_2018_schema(tables: Mapping[str, tuple[str, ...]]) -> None:
+    """Fail before cohort construction unless required source fields are exactly available."""
+    required = {
+        "demographics": {"SEQN", "RIAGENDR", "RIDAGEYR"},
+        "questionnaire": {"SEQN", "RHQ031", "RHD043"},
+    }
+    for table, columns in required.items():
+        available = set(tables.get(table, ()))
+        missing = sorted(columns - available)
+        if missing:
+            raise DomainVerificationError(f"2017-2018 {table} schema mismatch: missing {missing}")

calm/adapters/synthetic.py

@@ -0,0 +1,109 @@
+"""Deterministic offline NHANES-like source for tests and demonstrations."""
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pandas as pd
+import pyreadstat
+
+from calm.domain.entities import CycleSpec, RawBundle, RawFile
+from calm.observability.hashing import hash_bytes
+
+
+class SyntheticSource:
+    """Produce a metadata-labelled benchmark bundle without network access."""
+
+    def __init__(self, destination: Path) -> None:
+        """Set the destination for generated fixture data."""
+        self._destination = destination
+
+    def fetch(self, cycle: CycleSpec) -> RawBundle:
+        """Write the component bundle needed by the complete synthetic pipeline."""
+        directory = self._destination / cycle.identifier
+        directory.mkdir(parents=True, exist_ok=True)
+        identifiers = list(range(1, 81))
+        postmenopausal = [identifier % 2 == 0 for identifier in identifiers]
+        paths = {
+            "questionnaire": directory / "questionnaire.sav",
+            "demographics": directory / "demographics.sav",
+            "laboratory": directory / "laboratory.sav",
+        }
+        pyreadstat.write_sav(
+            pd.DataFrame(
+                {
+                    "SEQN": identifiers,
+                    "SEX": [2] * 80,
+                    "AGE": [40 + index % 20 for index in range(80)],
+                }
+            ),
+            paths["demographics"],
+            column_labels={"SEQN": "Respondent sequence number", "SEX": "Sex", "AGE": "Age"},
+            variable_value_labels={"SEX": {1: "Male", 2: "Female"}},
+        )
+        pyreadstat.write_sav(
+            pd.DataFrame(
+                {
+                    "SEQN": identifiers,
+                    "STATUS": [2 if value else 1 for value in postmenopausal],
+                    "RHQ031": [2 if value else 1 for value in postmenopausal],
+                    "RHD043": [7 if value else None for value in postmenopausal],
+                    "RHD280": [2] * 80,
+                    "RHQ305": [2] * 80,
+                    "RHD143": [2] * 80,
+                    "RHQ200": [2] * 80,
+                    "RHQ540": [2] * 80,
+                }
+            ),
+            paths["questionnaire"],
+            column_labels={
+                column: column
+                for column in (
+                    "SEQN",
+                    "STATUS",
+                    "RHQ031",
+                    "RHD043",
+                    "RHD280",
+                    "RHQ305",
+                    "RHD143",
+                    "RHQ200",
+                    "RHQ540",
+                )
+            },
+            variable_value_labels={
+                "STATUS": {1: "Yes", 2: "No"},
+                "RHQ031": {1: "Yes", 2: "No"},
+                "RHD043": {3: "Hysterectomy", 7: "Menopause/Change of life"},
+                **{
+                    column: {1: "Yes", 2: "No"}
+                    for column in ("RHD280", "RHQ305", "RHD143", "RHQ200", "RHQ540")
+                },
+            },
+        )
+        pyreadstat.write_sav(
+            pd.DataFrame(
+                {
+                    "SEQN": identifiers,
+                    "marker_one": [float(index % 2) for index in range(80)],
+                    "marker_two": [float((index * 7) % 11) for index in range(80)],
+                }
+            ),
+            paths["laboratory"],
+            column_labels={
+                "SEQN": "Respondent sequence number",
+                "marker_one": "Synthetic marker one",
+                "marker_two": "Synthetic marker two",
+            },
+        )
+        return RawBundle(
+            cycle=cycle,
+            files=tuple(
+                RawFile(
+                    component=component,
+                    source_url=f"synthetic://{component}",
+                    checksum=str(hash_bytes(path.read_bytes())),
+                    local_path=path,
+                    fetched_at=datetime(2026, 1, 1, tzinfo=UTC),
+                )
+                for component, path in paths.items()
+            ),
+        )

calm/cli.py

@@ -0,0 +1,86 @@
+"""Command-line interface for CALM."""
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from calm.adapters.nhanes_2017_2018 import Nhanes2017_2018Source
+from calm.adapters.synthetic import SyntheticSource
+from calm.config.loading import load_config
+from calm.config.models import (
+    CalibrationConfig,
+    CohortConfig,
+    CycleConfig,
+    FeatureConfig,
+    LabelConfig,
+    ModelConfig,
+    PathsConfig,
+    RunConfig,
+    SplitConfig,
+)
+from calm.orchestration.benchmark import run_benchmark
+
+app = typer.Typer(
+    name="calm",
+    help="Build reproducible menopausal-status prediction benchmarks.",
+    no_args_is_help=True,
+)
+
+
+@app.callback()
+def main() -> None:
+    """Run the CALM command-line application."""
+
+
+@app.command()
+def demo(
+    output: Annotated[
+        Path, typer.Option(help="Directory in which to write demo artifacts.")
+    ] = Path("runs"),
+    run_id: Annotated[str, typer.Option(help="Stable name for the demo run.")] = "synthetic-demo",
+) -> None:
+    """Run the complete offline synthetic pipeline and write its audit artifacts."""
+    config = RunConfig(
+        seed=31,
+        cycles=(CycleConfig(identifier="2017-2018"),),
+        cohort=CohortConfig(age_policy="restricted", minimum_age=40, maximum_age=60),
+        label=LabelConfig(
+            rule_name="public_cessation",
+            perimenopause_policy="exclude",
+            include_early_natural_menopause=False,
+        ),
+        features=FeatureConfig(allowlist=("marker_one", "marker_two")),
+        split=SplitConfig(validation_fraction=0.2, test_fraction=0.2),
+        model=ModelConfig(estimator="logistic_regression", options={"max_iter": 200}),
+        calibration=CalibrationConfig(method="sigmoid", no_call_lower=0.45, no_call_upper=0.55),
+        paths=PathsConfig(raw_data=output / ".raw", cache=output / ".cache", runs=output),
+    )
+    result = run_benchmark(config, SyntheticSource(config.paths.raw_data), run_id)
+    typer.echo(f"Wrote benchmark artifacts to {result.artifacts.metrics_path.parent}")
+
+
+@app.command("run-synthetic")
+def run_synthetic(
+    config_path: Annotated[Path, typer.Option("--config", exists=True, readable=True)],
+    run_id: Annotated[
+        str, typer.Option(help="Stable name for the benchmark run.")
+    ] = "synthetic-run",
+) -> None:
+    """Execute the complete synthetic benchmark from a validated TOML configuration."""
+    config = load_config(config_path)
+    result = run_benchmark(config, SyntheticSource(config.paths.raw_data), run_id)
+    typer.echo(f"Wrote benchmark artifacts to {result.artifacts.metrics_path.parent}")
+
+
+@app.command("run-nhanes")
+def run_nhanes(
+    config_path: Annotated[Path, typer.Option("--config", exists=True, readable=True)],
+    run_id: Annotated[str, typer.Option(help="Stable name for the benchmark run.")] = (
+        "nhanes-2017-2018"
+    ),
+) -> None:
+    """Run the pinned public NHANES 2017--2018 benchmark from downloaded inputs."""
+    config = load_config(config_path)
+    result = run_benchmark(config, Nhanes2017_2018Source(config.paths.raw_data), run_id)
+    typer.echo(f"Wrote benchmark artifacts to {result.artifacts.metrics_path.parent}")

calm/config/__init__.py

@@ -0,0 +1,6 @@
+"""Configuration contracts and loading."""
+
+from calm.config.loading import load_config
+from calm.config.models import RunConfig
+
+__all__ = ["RunConfig", "load_config"]

calm/config/loading.py

@@ -0,0 +1,19 @@
+"""TOML configuration loading at the application boundary."""
+
+import tomllib
+from pathlib import Path
+
+from calm.config.models import RunConfig
+
+
+def load_config(path: Path) -> RunConfig:
+    """Load and validate a run configuration from a TOML file.
+
+    Args:
+        path: Location of the user-supplied TOML configuration.
+
+    Returns:
+        A validated immutable run configuration.
+    """
+    with path.open("rb") as config_file:
+        return RunConfig.model_validate(tomllib.load(config_file))

calm/config/models.py

@@ -0,0 +1,132 @@
+"""Validated, immutable configuration contracts for CALM runs."""
+
+from pathlib import Path
+from typing import Literal, Self
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class ConfigModel(BaseModel):
+    """Base class for immutable, strict CALM configuration models."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class CycleConfig(ConfigModel):
+    """A continuous NHANES two-year cycle selected for a run."""
+
+    identifier: str = Field(pattern=r"^\d{4}-\d{4}$")
+    components: tuple[
+        Literal["demographics", "examination", "laboratory", "questionnaire"], ...
+    ] = (
+        "demographics",
+        "examination",
+        "laboratory",
+        "questionnaire",
+    )
+
+
+class CohortConfig(ConfigModel):
+    """Rules for selecting the analytical cohort."""
+
+    age_policy: Literal["restricted", "full_range"]
+    minimum_age: int = Field(ge=0, le=120)
+    maximum_age: int = Field(ge=0, le=120)
+
+    @model_validator(mode="after")
+    def validate_age_window(self) -> Self:
+        """Require a nonempty configured age interval."""
+        if self.minimum_age >= self.maximum_age:
+            msg = "minimum_age must be less than maximum_age"
+            raise ValueError(msg)
+        return self
+
+
+class LabelConfig(ConfigModel):
+    """Policies governing metadata-verified outcome construction."""
+
+    rule_name: str = "natural_menopause"
+    rule_version: str = "1"
+    perimenopause_policy: Literal["exclude", "premenopausal"]
+    include_early_natural_menopause: bool = True
+
+
+class FeatureConfig(ConfigModel):
+    """An explicit allowlist of candidate non-label feature names."""
+
+    allowlist: tuple[str, ...] = Field(min_length=1)
+
+
+class SplitConfig(ConfigModel):
+    """Deterministic partition fractions for model development."""
+
+    test_fraction: float = Field(gt=0, lt=1)
+    validation_fraction: float = Field(gt=0, lt=1)
+
+    @model_validator(mode="after")
+    def validate_training_fraction(self) -> Self:
+        """Retain a nonempty training partition."""
+        if self.test_fraction + self.validation_fraction >= 1:
+            msg = "test_fraction and validation_fraction must sum to less than 1"
+            raise ValueError(msg)
+        return self
+
+
+class ModelConfig(ConfigModel):
+    """Configured estimator selection and stable scalar options."""
+
+    estimator: Literal["logistic_regression", "gradient_boosting"]
+    options: dict[str, str | int | float | bool] = Field(default_factory=dict)
+
+
+class CalibrationConfig(ConfigModel):
+    """Probability calibration and abstention-band settings."""
+
+    method: Literal["sigmoid", "isotonic"]
+    no_call_lower: float = Field(ge=0, le=1)
+    no_call_upper: float = Field(ge=0, le=1)
+    minimum_validation_samples: int = Field(default=10, ge=2)
+    minimum_validation_per_class: int = Field(default=5, ge=2)
+    selection_protocol: Literal["preselected"] = "preselected"
+
+    @model_validator(mode="after")
+    def validate_no_call_band(self) -> Self:
+        """Require a properly ordered abstention interval."""
+        if self.no_call_lower >= self.no_call_upper:
+            msg = "no_call_lower must be less than no_call_upper"
+            raise ValueError(msg)
+        return self
+
+
+class EvaluationConfig(ConfigModel):
+    """Uncertainty and reporting policies fixed before frozen-test evaluation."""
+
+    bootstrap_resamples: int = Field(default=200, ge=100)
+    confidence_level: float = Field(default=0.95, gt=0, lt=1)
+    survey_weight_decision: Literal["not_applied_individual_level"] = "not_applied_individual_level"
+    generalization_protocol: Literal["within_cycle_frozen_test", "held_out_cycle_required"] = (
+        "within_cycle_frozen_test"
+    )
+
+
+class PathsConfig(ConfigModel):
+    """Filesystem locations supplied by the application boundary."""
+
+    raw_data: Path
+    cache: Path
+    runs: Path
+
+
+class RunConfig(ConfigModel):
+    """The complete validated and immutable intent of one CALM run."""
+
+    seed: int = Field(ge=0)
+    cycles: tuple[CycleConfig, ...] = Field(min_length=1)
+    cohort: CohortConfig
+    label: LabelConfig
+    features: FeatureConfig
+    split: SplitConfig
+    model: ModelConfig
+    calibration: CalibrationConfig
+    evaluation: EvaluationConfig = EvaluationConfig()
+    paths: PathsConfig

calm/domain/__init__.py

@@ -0,0 +1,5 @@
+"""Pure domain entities and invariants."""
+
+from calm.domain.entities import Cohort, LabelSet, VariableCode
+
+__all__ = ["Cohort", "LabelSet", "VariableCode"]