eval-toolkit 0.27.1__py3-none-any.whl

eval_toolkit/__init__.py

@@ -0,0 +1,238 @@
+"""eval-toolkit — reusable evaluation contracts for binary classification.
+
+Public API remains available from ``eval_toolkit`` and from submodules:
+
+    from eval_toolkit import pr_auc, bootstrap_ci, BootstrapCI
+    from eval_toolkit.metrics import pr_auc
+
+The package root uses lazy exports so importing ``eval_toolkit`` does not
+eagerly import optional-heavy modules such as plotting, loaders, or harnesses.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import Any
+
+from eval_toolkit._version import __version__
+
+_EXPORTS: dict[str, str] = {
+    "CsvPredictionReader": "eval_toolkit.analysis",
+    "JsonlPredictionReader": "eval_toolkit.analysis",
+    "PredictionArrays": "eval_toolkit.analysis",
+    "bootstrap_metric_from_predictions": "eval_toolkit.analysis",
+    "load_prediction_arrays": "eval_toolkit.analysis",
+    "paired_diff_from_prediction_refs": "eval_toolkit.analysis",
+    "MetricState": "eval_toolkit.artifacts",
+    "PredictionArtifactRef": "eval_toolkit.artifacts",
+    "PredictionColumns": "eval_toolkit.artifacts",
+    "error_metric": "eval_toolkit.artifacts",
+    "sanitize_for_json": "eval_toolkit.artifacts",
+    "skipped_metric": "eval_toolkit.artifacts",
+    "validate_manifest": "eval_toolkit.artifacts",
+    "validate_payload": "eval_toolkit.artifacts",
+    "validate_prediction_artifact_ref": "eval_toolkit.artifacts",
+    "validate_results": "eval_toolkit.artifacts",
+    "write_json_strict": "eval_toolkit.artifacts",
+    "DEFAULT_CONFIDENCE": "eval_toolkit.bootstrap",
+    "DEFAULT_METHOD": "eval_toolkit.bootstrap",
+    "DEFAULT_N_RESAMPLES": "eval_toolkit.bootstrap",
+    "DEFAULT_SEED": "eval_toolkit.bootstrap",
+    "BootstrapCI": "eval_toolkit.bootstrap",
+    "DeLongResult": "eval_toolkit.bootstrap",
+    "MDEEstimate": "eval_toolkit.bootstrap",
+    "MetricFn": "eval_toolkit.bootstrap",
+    "PairedBootstrapCI": "eval_toolkit.bootstrap",
+    "ThresholdedMetricFn": "eval_toolkit.bootstrap",
+    "ThresholdFn": "eval_toolkit.bootstrap",
+    "bootstrap_ci": "eval_toolkit.bootstrap",
+    "cross_validate_metric": "eval_toolkit.bootstrap",
+    "cv_clt_ci": "eval_toolkit.bootstrap",
+    "delong_roc_variance": "eval_toolkit.bootstrap",
+    "mde_from_ci": "eval_toolkit.bootstrap",
+    "paired_bootstrap_diff": "eval_toolkit.bootstrap",
+    "paired_bootstrap_ece_diff": "eval_toolkit.bootstrap",
+    "paired_bootstrap_op_point_diff": "eval_toolkit.bootstrap",
+    "paired_mde": "eval_toolkit.bootstrap",
+    "DEFAULT_FN_COST": "eval_toolkit.calibration",
+    "DEFAULT_FP_COST": "eval_toolkit.calibration",
+    "DEFAULT_N_BINS": "eval_toolkit.calibration",
+    "DEFAULT_PRIOR": "eval_toolkit.calibration",
+    "DEFAULT_STRATEGY": "eval_toolkit.calibration",
+    "CostMatrix": "eval_toolkit.calibration",
+    "bayes_optimal_threshold": "eval_toolkit.calibration",
+    "fit_beta_calibrator": "eval_toolkit.calibration",
+    "fit_isotonic_calibrator": "eval_toolkit.calibration",
+    "fit_platt_calibrator": "eval_toolkit.calibration",
+    "fit_temperature": "eval_toolkit.calibration",
+    "fit_temperature_oracle": "eval_toolkit.calibration",
+    "reliability_curve": "eval_toolkit.calibration",
+    "reliability_diagram_data": "eval_toolkit.calibration",
+    "ClaimReport": "eval_toolkit.claims",
+    "ClaimSpec": "eval_toolkit.claims",
+    "EvidenceGate": "eval_toolkit.claims",
+    "GateResult": "eval_toolkit.claims",
+    "evaluate_claims": "eval_toolkit.claims",
+    "external_diagnostic_gate": "eval_toolkit.claims",
+    "headline_present_gate": "eval_toolkit.claims",
+    "low_fpr_feasibility_gate": "eval_toolkit.claims",
+    "metric_threshold_gate": "eval_toolkit.claims",
+    "minimum_slice_size_gate": "eval_toolkit.claims",
+    "no_leakage_errors_gate": "eval_toolkit.claims",
+    "no_scorer_errors_gate": "eval_toolkit.claims",
+    "paired_diff_present_gate": "eval_toolkit.claims",
+    "required_metric_gate": "eval_toolkit.claims",
+    "required_scorer_gate": "eval_toolkit.claims",
+    "required_slice_gate": "eval_toolkit.claims",
+    "source_role_gate": "eval_toolkit.claims",
+    "strict_artifact_gate": "eval_toolkit.claims",
+    "from_yaml": "eval_toolkit.config",
+    "frozen_config": "eval_toolkit.config",
+    "ANCHOR_RE": "eval_toolkit.docs",
+    "DEFAULT_FORMATTERS": "eval_toolkit.docs",
+    "render_files": "eval_toolkit.docs",
+    "render_text": "eval_toolkit.docs",
+    "walk_path": "eval_toolkit.docs",
+    "AggregateEvidence": "eval_toolkit.evidence",
+    "EvidenceAxis": "eval_toolkit.evidence",
+    "PairingMetadata": "eval_toolkit.evidence",
+    "RECOMMENDED_SOURCE_ROLES": "eval_toolkit.evidence",
+    "DEFAULT_BOOTSTRAP_RESAMPLES": "eval_toolkit.harness",
+    "RUN_RESULT_SCHEMA_VERSION": "eval_toolkit.harness",
+    "EvalSlice": "eval_toolkit.harness",
+    "RunResult": "eval_toolkit.harness",
+    "evaluate": "eval_toolkit.harness",
+    "evaluate_folded": "eval_toolkit.harness",
+    "evaluate_scorer_on_slice": "eval_toolkit.harness",
+    "with_claim_report": "eval_toolkit.harness",
+    "write_run_result": "eval_toolkit.harness",
+    "CrossSplitLeakageCheck": "eval_toolkit.leakage",
+    "ExactDuplicateCheck": "eval_toolkit.leakage",
+    "GroupLeakageCheck": "eval_toolkit.leakage",
+    "LabelConflictCheck": "eval_toolkit.leakage",
+    "LeakageCheck": "eval_toolkit.leakage",
+    "LeakageFinding": "eval_toolkit.leakage",
+    "LeakageReport": "eval_toolkit.leakage",
+    "NearDuplicateCheck": "eval_toolkit.leakage",
+    "NormalizedFormLeakageCheck": "eval_toolkit.leakage",
+    "TemporalLeakageCheck": "eval_toolkit.leakage",
+    "run_leakage_checks": "eval_toolkit.leakage",
+    "DataFrameLoader": "eval_toolkit.loaders",
+    "DatasetLoader": "eval_toolkit.loaders",
+    "HFDatasetsLoader": "eval_toolkit.loaders",
+    "ParquetGlobLoader": "eval_toolkit.loaders",
+    "SingleSliceLoader": "eval_toolkit.loaders",
+    "MANIFEST_SCHEMA_VERSION": "eval_toolkit.manifest",
+    "RunManifest": "eval_toolkit.manifest",
+    "SourceRoleRecord": "eval_toolkit.manifest",
+    "build_manifest": "eval_toolkit.manifest",
+    "validate_source_roles": "eval_toolkit.manifest",
+    "write_manifest": "eval_toolkit.manifest",
+    "DEFAULT_ASSUMED_PRIORS": "eval_toolkit.metrics",
+    "ThresholdResult": "eval_toolkit.metrics",
+    "brier_decomposition": "eval_toolkit.metrics",
+    "brier_score": "eval_toolkit.metrics",
+    "expected_calibration_error": "eval_toolkit.metrics",
+    "expected_calibration_error_debiased": "eval_toolkit.metrics",
+    "expected_calibration_error_equal_mass": "eval_toolkit.metrics",
+    "expected_calibration_error_l2": "eval_toolkit.metrics",
+    "expected_calibration_error_l2_debiased": "eval_toolkit.metrics",
+    "headline_metrics": "eval_toolkit.metrics",
+    "metrics_at_threshold": "eval_toolkit.metrics",
+    "pr_auc": "eval_toolkit.metrics",
+    "precision_at_prior": "eval_toolkit.metrics",
+    "quantile_stratified_pr_auc": "eval_toolkit.metrics",
+    "quantile_stratified_report": "eval_toolkit.metrics",
+    "roc_auc": "eval_toolkit.metrics",
+    "score_distribution_summary": "eval_toolkit.metrics",
+    "single_class_threshold_metrics": "eval_toolkit.metrics",
+    "stratified_recall": "eval_toolkit.metrics",
+    "FittedOperatingPoint": "eval_toolkit.operating_points",
+    "OperatingPointSpec": "eval_toolkit.operating_points",
+    "apply_operating_points": "eval_toolkit.operating_points",
+    "fit_operating_points": "eval_toolkit.operating_points",
+    "path_for_config": "eval_toolkit.paths",
+    "resolve_repo_path": "eval_toolkit.paths",
+    "split_provenance_config": "eval_toolkit.paths",
+    "DEFAULT_FIGSIZE": "eval_toolkit.plotting",
+    "PALETTE": "eval_toolkit.plotting",
+    "PLOT_STYLE": "eval_toolkit.plotting",
+    "make_palette": "eval_toolkit.plotting",
+    "plot_bootstrap_distribution": "eval_toolkit.plotting",
+    "plot_confusion_matrix_grid": "eval_toolkit.plotting",
+    "plot_lift_ci": "eval_toolkit.plotting",
+    "plot_metric_bars": "eval_toolkit.plotting",
+    "plot_pr_curve": "eval_toolkit.plotting",
+    "plot_reliability_diagram": "eval_toolkit.plotting",
+    "plot_score_histograms": "eval_toolkit.plotting",
+    "save_figure": "eval_toolkit.plotting",
+    "set_plot_style": "eval_toolkit.plotting",
+    "FileHash": "eval_toolkit.provenance",
+    "FileHashMissing": "eval_toolkit.provenance",
+    "capture_git_sha": "eval_toolkit.provenance",
+    "compute_file_hash": "eval_toolkit.provenance",
+    "figure_metadata": "eval_toolkit.provenance",
+    "file_sha256": "eval_toolkit.provenance",
+    "make_run_dir": "eval_toolkit.provenance",
+    "EvalSliceLike": "eval_toolkit.protocols",
+    "PredictionReader": "eval_toolkit.protocols",
+    "Scorer": "eval_toolkit.protocols",
+    "SliceAwareScorer": "eval_toolkit.protocols",
+    "Versioned": "eval_toolkit.protocols",
+    "set_global_seeds": "eval_toolkit.seeds",
+    "GroupKFoldSplitter": "eval_toolkit.splits",
+    "HoldoutSplitter": "eval_toolkit.splits",
+    "PoolBuilder": "eval_toolkit.splits",
+    "SourceDisjointKFoldSplitter": "eval_toolkit.splits",
+    "Splitter": "eval_toolkit.splits",
+    "StratifiedKFoldSplitter": "eval_toolkit.splits",
+    "TimeSeriesSplitter": "eval_toolkit.splits",
+    "iter_folds_with_pool": "eval_toolkit.splits",
+    "DEFAULT_DEDUP_THRESHOLD": "eval_toolkit.text_dedup",
+    "DedupReport": "eval_toolkit.text_dedup",
+    "EmbeddingCosineStrategy": "eval_toolkit.text_dedup",
+    "ExactNormalizedHashStrategy": "eval_toolkit.text_dedup",
+    "JaccardNgramStrategy": "eval_toolkit.text_dedup",
+    "MinHashLSHStrategy": "eval_toolkit.text_dedup",
+    "SimilarityAuditFinding": "eval_toolkit.text_dedup",
+    "SimilarityAuditReport": "eval_toolkit.text_dedup",
+    "SimilarityStrategy": "eval_toolkit.text_dedup",
+    "TfidfCosineStrategy": "eval_toolkit.text_dedup",
+    "audit_source_label_similarity": "eval_toolkit.text_dedup",
+    "cross_dedup": "eval_toolkit.text_dedup",
+    "near_dedup": "eval_toolkit.text_dedup",
+    "normalize_text_for_dedup": "eval_toolkit.text_dedup",
+    "sha256_text": "eval_toolkit.text_dedup",
+    "CISafeThresholdSelector": "eval_toolkit.thresholds",
+    "CostSensitiveSelector": "eval_toolkit.thresholds",
+    "MaxF1Selector": "eval_toolkit.thresholds",
+    "TargetFPRSelector": "eval_toolkit.thresholds",
+    "TargetPrecisionSelector": "eval_toolkit.thresholds",
+    "TargetRecallSelector": "eval_toolkit.thresholds",
+    "ThresholdPolicyMetadata": "eval_toolkit.thresholds",
+    "ThresholdSelector": "eval_toolkit.thresholds",
+    "WilsonInterval": "eval_toolkit.thresholds",
+    "YoudenJSelector": "eval_toolkit.thresholds",
+    "select_threshold": "eval_toolkit.thresholds",
+    "wilson_interval": "eval_toolkit.thresholds",
+}
+
+__all__ = ["__version__", *_EXPORTS.keys()]
+
+
+def __getattr__(name: str) -> Any:
+    """Resolve public symbols lazily."""
+    if name == "__version__":
+        return __version__
+    module_name = _EXPORTS.get(name)
+    if module_name is None:
+        raise AttributeError(f"module 'eval_toolkit' has no attribute {name!r}")
+    module = import_module(module_name)
+    value = getattr(module, name)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    """Expose lazy public symbols to introspection."""
+    return sorted(__all__)

eval_toolkit/__main__.py

@@ -0,0 +1,156 @@
+"""eval_toolkit CLI entry: ``python -m eval_toolkit ...``.
+
+Subcommands:
+  schemas list                       List bundled JSON schema names.
+  schemas show <name>                Pretty-print a single schema.
+  schemas check                      Meta-validate every bundled schema against
+                                     Draft 2020-12. Used by CI to assert
+                                     schema integrity as a package invariant.
+  validate <file> <schema-name>      Validate a JSON file against a bundled schema
+                                     (requires [validation] extra).
+
+Exit codes:
+  0 — ok
+  1 — validation failed (payload or schema meta-validation)
+  2 — bad arg (file or schema not found; empty schemas directory)
+  3 — missing optional dependency (jsonschema for validate)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import eval_toolkit
+
+
+def _schemas_dir() -> Path:
+    return Path(eval_toolkit.__file__).parent / "schemas"
+
+
+def _cmd_schemas_list(_args: argparse.Namespace) -> int:
+    for f in sorted(_schemas_dir().glob("*.json")):
+        print(f.stem)
+    return 0
+
+
+def _cmd_schemas_show(args: argparse.Namespace) -> int:
+    name: str = args.name
+    candidate = _schemas_dir() / f"{name}.json"
+    if not candidate.exists():
+        # Tolerate users typing the full filename (e.g., results.v1.json).
+        candidate_with_ext = _schemas_dir() / name
+        if candidate_with_ext.exists() and candidate_with_ext.suffix == ".json":
+            candidate = candidate_with_ext
+        else:
+            print(f"unknown schema: {name}", file=sys.stderr)
+            return 2
+    print(json.dumps(json.loads(candidate.read_text()), indent=2, sort_keys=True))
+    return 0
+
+
+def _cmd_schemas_check(_args: argparse.Namespace) -> int:
+    """Meta-validate every bundled schema against Draft 2020-12.
+
+    Exits non-zero on empty schemas directory, malformed JSON, or
+    schema-meta-validation failure. ``jsonschema`` is a hard dep
+    (see ``pyproject.toml``), so no optional-extra degrade path needed.
+    """
+    from jsonschema import (  # type: ignore[import-untyped]  # noqa: PLC0415
+        Draft202012Validator,
+    )
+    from jsonschema.exceptions import (  # type: ignore[import-untyped]  # noqa: PLC0415
+        SchemaError,
+    )
+
+    schemas_dir = _schemas_dir()
+    files = sorted(schemas_dir.glob("*.json"))
+    if not files:
+        print(f"ERROR: no schemas found in {schemas_dir}", file=sys.stderr)
+        return 2
+    failures: list[str] = []
+    for f in files:
+        try:
+            schema = json.loads(f.read_text())
+            Draft202012Validator.check_schema(schema)
+            print(f"  {f.name}: OK")
+        except (json.JSONDecodeError, SchemaError) as exc:
+            failures.append(f"{f.name}: {exc}")
+    if failures:
+        print("\nSchema validation failures:", file=sys.stderr)
+        for line in failures:
+            print(f"  {line}", file=sys.stderr)
+        return 1
+    return 0
+
+
+def _cmd_validate(args: argparse.Namespace) -> int:
+    try:
+        import jsonschema  # noqa: F401
+    except ImportError:
+        print(
+            "validate requires the [validation] extra: " "pip install 'eval-toolkit[validation]'",
+            file=sys.stderr,
+        )
+        return 3
+    schema_name: str = args.schema
+    schema_path = _schemas_dir() / f"{schema_name}.json"
+    if not schema_path.exists():
+        # Tolerate users typing the full filename.
+        alt = _schemas_dir() / schema_name
+        if alt.exists() and alt.suffix == ".json":
+            schema_path = alt
+        else:
+            print(f"unknown schema: {schema_name}", file=sys.stderr)
+            return 2
+    file_path = Path(args.file)
+    if not file_path.exists():
+        print(f"file not found: {args.file}", file=sys.stderr)
+        return 2
+    schema = json.loads(schema_path.read_text())
+    payload = json.loads(file_path.read_text())
+    import jsonschema as _js  # noqa: PLC0415
+
+    try:
+        _js.validate(payload, schema)
+    except _js.ValidationError as exc:
+        loc = "/".join(str(p) for p in exc.absolute_path) or "(root)"
+        print(f"VALIDATION ERROR at {loc}: {exc.message}", file=sys.stderr)
+        return 1
+    print(f"{args.file}: OK against {schema_name}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Entry point for ``python -m eval_toolkit`` and the ``eval-toolkit`` script."""
+    parser = argparse.ArgumentParser(
+        prog="eval-toolkit",
+        description="eval-toolkit CLI: schema discovery + payload validation",
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    schemas = sub.add_parser("schemas", help="bundled JSON schema discovery")
+    schemas_sub = schemas.add_subparsers(dest="schemas_cmd", required=True)
+    schemas_list = schemas_sub.add_parser("list", help="list bundled schema names")
+    schemas_list.set_defaults(func=_cmd_schemas_list)
+    schemas_show = schemas_sub.add_parser("show", help="pretty-print a single schema")
+    schemas_show.add_argument("name", help="schema name (e.g., 'results.v1')")
+    schemas_show.set_defaults(func=_cmd_schemas_show)
+    schemas_check = schemas_sub.add_parser(
+        "check", help="meta-validate every bundled schema against Draft 2020-12"
+    )
+    schemas_check.set_defaults(func=_cmd_schemas_check)
+
+    validate = sub.add_parser("validate", help="validate a JSON file against a bundled schema")
+    validate.add_argument("file", help="path to JSON file to validate")
+    validate.add_argument("schema", help="schema name (e.g., 'results.v1')")
+    validate.set_defaults(func=_cmd_validate)
+
+    args = parser.parse_args(argv)
+    return int(args.func(args))
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

eval_toolkit/_version.py

@@ -0,0 +1,5 @@
+"""Single lightweight version source."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.27.1"

eval_toolkit/analysis.py

@@ -0,0 +1,196 @@
+"""Post-run analysis over retained prediction artifacts."""
+
+from __future__ import annotations
+
+import csv
+import json
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from eval_toolkit.bootstrap import bootstrap_ci, paired_bootstrap_diff
+from eval_toolkit.metrics import pr_auc
+from eval_toolkit.protocols import PredictionReader
+
+__all__ = [
+    "CsvPredictionReader",
+    "JsonlPredictionReader",
+    "PredictionArrays",
+    "bootstrap_metric_from_predictions",
+    "load_prediction_arrays",
+    "paired_diff_from_prediction_refs",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class PredictionArrays:
+    """Numeric arrays loaded from a prediction artifact."""
+
+    labels: np.ndarray
+    scores: np.ndarray
+    row_ids: tuple[str, ...] = ()
+    content_hashes: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        """Validate array shape."""
+        if self.labels.shape != self.scores.shape:
+            raise ValueError("labels and scores must have identical shape")
+
+
+class CsvPredictionReader:
+    """Read CSV prediction files into a column-oriented mapping."""
+
+    def read_predictions(
+        self,
+        uri: str,
+        *,
+        columns: Mapping[str, str],
+    ) -> Mapping[str, Sequence[object]]:
+        """Read a local CSV file."""
+        wanted = set(columns.values())
+        out: dict[str, list[object]] = {col: [] for col in wanted}
+        with Path(uri).open(newline="") as fh:
+            reader = csv.DictReader(fh)
+            for row in reader:
+                for col in wanted:
+                    out[col].append(row.get(col, ""))
+        return out
+
+
+class JsonlPredictionReader:
+    """Read JSON Lines prediction files into a column-oriented mapping."""
+
+    def read_predictions(
+        self,
+        uri: str,
+        *,
+        columns: Mapping[str, str],
+    ) -> Mapping[str, Sequence[object]]:
+        """Read a local JSONL file."""
+        wanted = set(columns.values())
+        out: dict[str, list[object]] = {col: [] for col in wanted}
+        with Path(uri).open() as fh:
+            for line in fh:
+                if not line.strip():
+                    continue
+                row = json.loads(line)
+                for col in wanted:
+                    out[col].append(row.get(col))
+        return out
+
+
+def load_prediction_arrays(
+    ref: Mapping[str, Any],
+    *,
+    reader: PredictionReader | None = None,
+) -> PredictionArrays:
+    """Load labels and scores from a prediction artifact reference.
+
+    Raises
+    ------
+    ValueError
+        If ``ref`` lacks a ``columns`` mapping, lacks a non-empty ``uri``,
+        or its ``columns`` mapping is missing the ``label`` / ``score``
+        keys (re-raised from :func:`_required_column`).
+    """
+    columns = ref.get("columns")
+    if not isinstance(columns, Mapping):
+        raise ValueError("prediction ref must include a columns mapping")
+    label_col = _required_column(columns, "label")
+    score_col = _required_column(columns, "score")
+    uri = ref.get("uri")
+    if not isinstance(uri, str) or not uri:
+        raise ValueError("prediction ref must include a non-empty uri")
+    selected_reader = reader or _reader_for_ref(ref)
+    reader_columns = {str(k): str(v) for k, v in columns.items() if isinstance(v, str)}
+    table = selected_reader.read_predictions(uri, columns=reader_columns)
+    labels = np.asarray(table[label_col], dtype=int)
+    scores = np.asarray(table[score_col], dtype=float)
+    row_id_col = columns.get("row_id")
+    hash_col = columns.get("content_hash")
+    row_ids = tuple(str(v) for v in table.get(str(row_id_col), ())) if row_id_col else ()
+    hashes = tuple(str(v) for v in table.get(str(hash_col), ())) if hash_col else ()
+    return PredictionArrays(labels=labels, scores=scores, row_ids=row_ids, content_hashes=hashes)
+
+
+def bootstrap_metric_from_predictions(
+    ref: Mapping[str, Any],
+    *,
+    reader: PredictionReader | None = None,
+    n_resamples: int = 1000,
+    seed: int = 42,
+) -> dict[str, object]:
+    """Compute a PR-AUC bootstrap CI from one prediction ref."""
+    arrays = load_prediction_arrays(ref, reader=reader)
+    return bootstrap_ci(
+        arrays.labels,
+        arrays.scores,
+        pr_auc,
+        n_resamples=n_resamples,
+        seed=seed,
+    ).to_dict()
+
+
+def paired_diff_from_prediction_refs(
+    baseline_ref: Mapping[str, Any],
+    candidate_ref: Mapping[str, Any],
+    *,
+    baseline_reader: PredictionReader | None = None,
+    candidate_reader: PredictionReader | None = None,
+    n_resamples: int = 1000,
+    seed: int = 42,
+) -> dict[str, object]:
+    """Compute paired PR-AUC delta from two prediction refs.
+
+    Raises
+    ------
+    ValueError
+        If the two refs disagree on row count, label values, ``row_ids``,
+        or ``content_hashes``; or if either ref is malformed (re-raised
+        from :func:`load_prediction_arrays`).
+    """
+    baseline = load_prediction_arrays(baseline_ref, reader=baseline_reader)
+    candidate = load_prediction_arrays(candidate_ref, reader=candidate_reader)
+    if baseline.labels.shape != candidate.labels.shape:
+        raise ValueError("prediction refs must have the same number of rows")
+    if not np.array_equal(baseline.labels, candidate.labels):
+        raise ValueError("prediction refs must have identical labels for paired comparison")
+    if baseline.row_ids and candidate.row_ids and baseline.row_ids != candidate.row_ids:
+        raise ValueError("prediction refs must have identical row_ids for paired comparison")
+    if (
+        baseline.content_hashes
+        and candidate.content_hashes
+        and baseline.content_hashes != candidate.content_hashes
+    ):
+        raise ValueError("prediction refs must have identical content_hashes for paired comparison")
+    return paired_bootstrap_diff(
+        baseline.labels,
+        baseline.scores,
+        candidate.scores,
+        pr_auc,
+        n_resamples=n_resamples,
+        seed=seed,
+    ).to_dict()
+
+
+def _reader_for_ref(ref: Mapping[str, Any]) -> PredictionReader:
+    media_type = str(ref.get("media_type", ""))
+    uri = str(ref.get("uri", ""))
+    if media_type in {"text/csv", "application/csv"} or uri.endswith(".csv"):
+        return CsvPredictionReader()
+    if media_type in {"application/jsonl", "application/x-ndjson"} or uri.endswith(".jsonl"):
+        return JsonlPredictionReader()
+    raise ValueError(
+        "no built-in prediction reader for this artifact; pass a PredictionReader "
+        "or use CSV/JSONL"
+    )
+
+
+def _required_column(columns: Mapping[str, object], key: str) -> str:
+    value = columns.get(key)
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"prediction columns must include {key!r}")
+    return value