avow 0.1.0__py3-none-any.whl

assay/__init__.py

@@ -0,0 +1,17 @@
+"""Assay: the scoring engine that refuses to lie — the score face of the Avow envelope.
+
+The scoring surface needs the heavy scientific stack (scikit-learn / scipy / numpy),
+installed via the ``avow[assay]`` extra. If it is missing we fail with a coded
+``ScoringExtraMissing`` instead of a raw ``ModuleNotFoundError`` traceback."""
+
+from __future__ import annotations
+
+from assay.errors import ScoringExtraMissing
+from avow import __version__
+
+try:
+    from assay.api import composite_score, replay, score, verify
+except ModuleNotFoundError as exc:  # pragma: no cover - only without the [assay] extra
+    raise ScoringExtraMissing("install avow[assay] to use the scoring face") from exc
+
+__all__ = ["__version__", "composite_score", "replay", "score", "verify"]

assay/api.py

@@ -0,0 +1,169 @@
+"""The Assay facade: score, composite_score, verify, replay.
+
+This is the only module most callers touch. It wires the reused primitives into a
+signed, reproducible receipt and offers offline verification and replay."""
+
+from __future__ import annotations
+
+from nacl.signing import SigningKey
+
+from assay.calibration import CalibrationReport, calibration_report
+from assay.composite import SubScore, composite
+from assay.errors import (
+    InsufficientSamples,
+    ReplayMismatch,
+    SignatureInvalid,
+    UnknownMetric,
+)
+from assay.metrics import ClassificationScores, binary_scores, correctness
+from assay.models import CompositeRequest, ScoreRequest, SubScoreInput
+from assay.receipt import (
+    CalibrationDetail,
+    ClassificationDetail,
+    CompositeDetail,
+    ReceiptPayload,
+    ReliabilityPoint,
+    ScoreReceipt,
+    SubScorePart,
+    payload_digest,
+    sign_payload,
+)
+from assay.settings import AssaySettings
+from assay.uncertainty import Abstention, Estimate, mean_interval
+from avow import __version__
+from avow.canonical import content_hash
+from avow.verify import verify_receipt
+
+# The metric label a caller names is not free text: it selects which computation
+# Assay performs and is signed into the receipt. Only registered metrics are
+# accepted, so a receipt's `metric` field is verified, never merely asserted.
+_CLASSIFICATION_METRICS = frozenset({"binary"})
+_COMPOSITE_METRICS = frozenset({"weighted_composite"})
+
+
+def _require_metric(name: str, allowed: frozenset[str]) -> None:
+    if name not in allowed:
+        raise UnknownMetric(f"unknown metric {name!r}")
+
+
+def _classification_detail(scores: ClassificationScores) -> ClassificationDetail:
+    return ClassificationDetail(
+        precision=scores.precision,
+        recall=scores.recall,
+        f1=scores.f1,
+        pr_auc=scores.pr_auc,
+        roc_auc=scores.roc_auc,
+    )
+
+
+def _calibration_detail(report: CalibrationReport) -> CalibrationDetail:
+    points = tuple(
+        ReliabilityPoint(
+            mean_predicted=b.mean_predicted,
+            fraction_positive=b.fraction_positive,
+            count=b.count,
+        )
+        for b in report.bins
+    )
+    return CalibrationDetail(ece=report.ece, brier=report.brier, reliability=points)
+
+
+def _headline(
+    estimate: Estimate,
+) -> tuple[float | None, float | None, float | None, bool]:
+    if isinstance(estimate, Abstention):
+        return None, None, None, True
+    return estimate.point, estimate.low, estimate.high, False
+
+
+def _estimate(request: ScoreRequest, settings: AssaySettings) -> Estimate:
+    hits = correctness(request.y_true, request.y_score, threshold=request.threshold)
+    return mean_interval(
+        hits,
+        min_samples=settings.min_samples,
+        n_resamples=settings.bootstrap_resamples,
+        confidence_level=settings.confidence_level,
+        seed=settings.bootstrap_seed,
+    )
+
+
+def _classification_payload(request: ScoreRequest, settings: AssaySettings) -> ReceiptPayload:
+    scores = binary_scores(request.y_true, request.y_score, threshold=request.threshold)
+    report = calibration_report(request.y_true, request.y_score, n_bins=settings.ece_bins)
+    point, low, high, abstained = _headline(_estimate(request, settings))
+    return ReceiptPayload(
+        assay_version=__version__,
+        metric=request.metric,
+        metric_version=request.metric_version,
+        inputs_hash=content_hash(request.model_dump(mode="json")),
+        score=point,
+        interval_low=low,
+        interval_high=high,
+        abstained=abstained,
+        abstain_reason=InsufficientSamples.code if abstained else None,
+        classification=_classification_detail(scores),
+        calibration=_calibration_detail(report),
+    )
+
+
+def score(
+    request: ScoreRequest, *, signing_key: SigningKey, settings: AssaySettings
+) -> ScoreReceipt:
+    """Score a classification request into a signed, verifiable receipt."""
+    _require_metric(request.metric, _CLASSIFICATION_METRICS)
+    return sign_payload(_classification_payload(request, settings), signing_key)
+
+
+def _as_subscore(s: SubScoreInput) -> SubScore:
+    return SubScore(
+        name=s.name,
+        value=s.value,
+        low=s.low,
+        high=s.high,
+        scale_min=s.scale_min,
+        scale_max=s.scale_max,
+        weight=s.weight,
+    )
+
+
+def _composite_payload(request: CompositeRequest) -> ReceiptPayload:
+    result = composite([_as_subscore(s) for s in request.subscores])
+    parts = tuple(
+        SubScorePart(name=p.name, normalized_value=p.normalized_value, weight=p.weight)
+        for p in result.parts
+    )
+    return ReceiptPayload(
+        assay_version=__version__,
+        metric=request.metric,
+        metric_version=request.metric_version,
+        inputs_hash=content_hash(request.model_dump(mode="json")),
+        score=result.value,
+        interval_low=result.low,
+        interval_high=result.high,
+        composite=CompositeDetail(parts=parts),
+    )
+
+
+def composite_score(request: CompositeRequest, *, signing_key: SigningKey) -> ScoreReceipt:
+    """Score a weighted multi-scale composite into a signed receipt."""
+    _require_metric(request.metric, _COMPOSITE_METRICS)
+    return sign_payload(_composite_payload(request), signing_key)
+
+
+def verify(receipt: ScoreReceipt, *, expected_public_key: str) -> bool:
+    """Return whether a receipt verifies offline against a **pinned** signer.
+
+    ``expected_public_key`` is the signer's public key the caller trusts, obtained
+    out-of-band (e.g. the ``.pub`` file from ``keygen``) — never read from the
+    receipt itself, whose embedded key an attacker could swap."""
+    try:
+        verify_receipt(receipt, expected_public_key=expected_public_key)
+    except (SignatureInvalid, ReplayMismatch):
+        return False
+    return True
+
+
+def replay(request: ScoreRequest, receipt: ScoreReceipt, *, settings: AssaySettings) -> bool:
+    """Recompute the payload from inputs and confirm it reproduces the signed hash."""
+    recomputed = _classification_payload(request, settings)
+    return payload_digest(recomputed) == receipt.payload_hash

assay/calibration.py

@@ -0,0 +1,70 @@
+"""Calibration evidence: Expected Calibration Error (ECE), a reliability diagram,
+and the Brier score.
+
+Reliability points come from ``sklearn.calibration.calibration_curve`` (which drops
+empty bins), bin populations from ``numpy.histogram`` over the same uniform edges,
+and Brier from ``sklearn.metrics.brier_score_loss``. ECE is the population-weighted
+gap between predicted confidence and observed frequency."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+import numpy as np
+from sklearn.calibration import calibration_curve
+from sklearn.metrics import brier_score_loss
+
+
+@dataclass(frozen=True)
+class ReliabilityBin:
+    """One reliability-diagram point."""
+
+    mean_predicted: float
+    fraction_positive: float
+    count: int
+
+
+@dataclass(frozen=True)
+class CalibrationReport:
+    """ECE, Brier, and the reliability diagram for a set of predictions."""
+
+    ece: float
+    brier: float
+    bins: tuple[ReliabilityBin, ...]
+
+
+def _bin_populations(score_arr: np.ndarray, n_bins: int) -> list[int]:
+    # Bin with sklearn's own scheme (searchsorted on interior edges) rather than a
+    # parallel np.histogram: at an exact bin edge the two disagree, which would
+    # misalign these counts with calibration_curve's non-empty bins.
+    edges = np.linspace(0.0, 1.0, n_bins + 1)
+    bin_ids = np.searchsorted(edges[1:-1], score_arr)
+    counts = np.bincount(bin_ids, minlength=n_bins)
+    return [int(c) for c in counts if c > 0]
+
+
+def _bins(prob_pred: np.ndarray, prob_true: np.ndarray, weights: list[int]) -> list[ReliabilityBin]:
+    return [
+        ReliabilityBin(mean_predicted=float(p), fraction_positive=float(t), count=w)
+        for p, t, w in zip(prob_pred, prob_true, weights, strict=True)
+    ]
+
+
+def _ece(bins: list[ReliabilityBin], total: int) -> float:
+    return sum(b.count / total * abs(b.mean_predicted - b.fraction_positive) for b in bins)
+
+
+def calibration_report(
+    y_true: Sequence[int], y_score: Sequence[float], *, n_bins: int
+) -> CalibrationReport:
+    """Build the ECE / Brier / reliability report for binary predictions."""
+    true_arr = np.asarray(y_true, dtype=float)
+    score_arr = np.asarray(y_score, dtype=float)
+    prob_true, prob_pred = calibration_curve(true_arr, score_arr, n_bins=n_bins, strategy="uniform")
+    bins = _bins(prob_pred, prob_true, _bin_populations(score_arr, n_bins))
+    return CalibrationReport(
+        ece=_ece(bins, total=len(score_arr)),
+        brier=float(brier_score_loss(true_arr, score_arr)),
+        bins=tuple(bins),
+    )

assay/cli.py

@@ -0,0 +1,97 @@
+"""Typer CLI. Thin command layer: parse files into typed models, delegate to the
+facade, translate results to exit codes. No business logic lives here."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from assay.api import composite_score
+from assay.api import score as score_receipt
+from assay.api import verify as verify_receipt_bool
+from assay.models import CompositeRequest, ScoreRequest
+from assay.receipt import ScoreReceipt
+from assay.settings import AssaySettings
+from avow.keys import (
+    generate_signing_key,
+    load_signing_key,
+    read_public_key,
+    save_public_key,
+    save_signing_key,
+)
+from avow.ledger import append
+
+app = typer.Typer(help="Assay — the scoring engine that refuses to lie.")
+
+
+@app.command()
+def keygen(
+    out: Annotated[Path, typer.Option(help="Where to write the signing key.")] = Path(
+        "signing.key"
+    ),
+    pub: Annotated[
+        Path | None,
+        typer.Option(help="Where to write the public key (default: <out>.pub)."),
+    ] = None,
+) -> None:
+    """Generate a new Ed25519 signing key (private seed 0600) plus its public key.
+
+    The public key is written separately so a verifier can pin it out-of-band and
+    never has to trust the key embedded in a receipt."""
+    key = generate_signing_key()
+    save_signing_key(key, path=out)
+    pub_path = pub if pub is not None else Path(f"{out}.pub")
+    save_public_key(key, path=pub_path)
+    typer.echo(f"wrote signing key: {out}")
+    typer.echo(f"wrote public key: {pub_path}")
+
+
+@app.command()
+def score(
+    request: Annotated[Path, typer.Option("--request", help="ScoreRequest JSON.")],
+    key: Annotated[Path, typer.Option("--key", help="Signing key file.")],
+    out: Annotated[Path, typer.Option("--out", help="Receipt output path.")],
+    ledger: Annotated[Path, typer.Option("--ledger", help="Ledger JSONL path.")],
+) -> None:
+    """Score a classification request and write a signed receipt."""
+    parsed = ScoreRequest.model_validate_json(request.read_text(encoding="utf-8"))
+    receipt = score_receipt(parsed, signing_key=load_signing_key(key), settings=AssaySettings())
+    out.write_text(receipt.model_dump_json(indent=2), encoding="utf-8")
+    append(receipt, path=ledger)
+    typer.echo(f"wrote receipt: {out}")
+
+
+@app.command()
+def composite(
+    request: Annotated[Path, typer.Option("--request", help="CompositeRequest JSON.")],
+    key: Annotated[Path, typer.Option("--key", help="Signing key file.")],
+    out: Annotated[Path, typer.Option("--out", help="Receipt output path.")],
+) -> None:
+    """Score a weighted multi-scale composite and write a signed receipt."""
+    parsed = CompositeRequest.model_validate_json(request.read_text(encoding="utf-8"))
+    receipt = composite_score(parsed, signing_key=load_signing_key(key))
+    out.write_text(receipt.model_dump_json(indent=2), encoding="utf-8")
+    typer.echo(f"wrote receipt: {out}")
+
+
+@app.command()
+def verify(
+    receipt: Annotated[Path, typer.Option("--receipt", help="Receipt JSON to verify.")],
+    public_key: Annotated[
+        Path,
+        typer.Option("--public-key", help="Pinned signer public-key file (the .pub from keygen)."),
+    ],
+) -> None:
+    """Verify a receipt offline against a pinned signer; exit non-zero if it fails.
+
+    The expected public key is read from ``--public-key`` (out-of-band), never from
+    the receipt's own field, so a re-signed forgery cannot authenticate itself."""
+    parsed = ScoreReceipt.model_validate_json(receipt.read_text(encoding="utf-8"))
+    expected = read_public_key(public_key)
+    if verify_receipt_bool(parsed, expected_public_key=expected):
+        typer.echo("OK: receipt verified")
+        return
+    typer.echo("FAIL: receipt did not verify")
+    raise typer.Exit(code=1)

assay/composite.py

@@ -0,0 +1,101 @@
+"""Weighted multi-scale composite with propagated uncertainty.
+
+Each sub-score is normalized to [0,1] by its own scale, then combined as a
+positive-weighted mean. Because the mean is monotone in each input, the composite
+interval is the same weighted mean applied to the sub-scores' lows and highs —
+exact interval arithmetic, no fabricated tightening."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Final
+
+from assay.errors import InvalidScoreRequest
+
+MIN_SUBSCORES: Final[int] = 3
+
+
+@dataclass(frozen=True)
+class SubScore:
+    """One input sub-score with its native scale and an uncertainty interval."""
+
+    name: str
+    value: float
+    low: float
+    high: float
+    scale_min: float
+    scale_max: float
+    weight: float
+
+
+@dataclass(frozen=True)
+class NormalizedSubScore:
+    """A sub-score after normalization to [0,1]."""
+
+    name: str
+    normalized_value: float
+    weight: float
+
+
+@dataclass(frozen=True)
+class CompositeScore:
+    """The composite value with its propagated interval and normalized parts."""
+
+    value: float
+    low: float
+    high: float
+    parts: tuple[NormalizedSubScore, ...]
+
+
+def _require_min_count(subscores: Sequence[SubScore]) -> None:
+    if len(subscores) < MIN_SUBSCORES:
+        raise InvalidScoreRequest("composite needs at least three sub-scores")
+
+
+def _require_increasing_scales(subscores: Sequence[SubScore]) -> None:
+    if any(s.scale_max <= s.scale_min for s in subscores):
+        raise InvalidScoreRequest("scale_max must exceed scale_min")
+
+
+def _require_positive_weight(subscores: Sequence[SubScore]) -> None:
+    if any(s.weight <= 0 for s in subscores):
+        raise InvalidScoreRequest("each sub-score weight must be positive")
+
+
+def _require_ordered_intervals(subscores: Sequence[SubScore]) -> None:
+    if any(not (s.low <= s.value <= s.high) for s in subscores):
+        raise InvalidScoreRequest("each sub-score must satisfy low <= value <= high")
+
+
+def _validate(subscores: Sequence[SubScore]) -> None:
+    _require_min_count(subscores)
+    _require_increasing_scales(subscores)
+    _require_positive_weight(subscores)
+    _require_ordered_intervals(subscores)
+
+
+def _normalize(x: float, s: SubScore) -> float:
+    return min(1.0, max(0.0, (x - s.scale_min) / (s.scale_max - s.scale_min)))
+
+
+def _weighted(
+    subscores: Sequence[SubScore], total_w: float, pick: Callable[[SubScore], float]
+) -> float:
+    return sum(s.weight * _normalize(pick(s), s) for s in subscores) / total_w
+
+
+def _part(s: SubScore) -> NormalizedSubScore:
+    return NormalizedSubScore(s.name, _normalize(s.value, s), s.weight)
+
+
+def composite(subscores: Sequence[SubScore]) -> CompositeScore:
+    """Combine >= 3 multi-scale sub-scores into one interval-carrying composite."""
+    _validate(subscores)
+    total_w = sum(s.weight for s in subscores)
+    return CompositeScore(
+        value=_weighted(subscores, total_w, lambda s: s.value),
+        low=_weighted(subscores, total_w, lambda s: s.low),
+        high=_weighted(subscores, total_w, lambda s: s.high),
+        parts=tuple(_part(s) for s in subscores),
+    )

assay/errors.py

@@ -0,0 +1,61 @@
+"""Coded domain-error catalog for the *scoring* face. Every failure raises a typed
+``AssayError`` with a stable string ``code`` so callers (and the CLI) branch on cause
+without string-matching messages.
+
+The trust-envelope failures (``SignatureInvalid``, ``ReplayMismatch``,
+``CanonicalizationFailed``, ``LedgerIntegrityError``) belong to the shared envelope and
+live in ``avow.errors``; they are re-exported here so the score face's callers keep a
+single import site."""
+
+from __future__ import annotations
+
+from typing import ClassVar
+
+from avow.errors import (
+    CanonicalizationFailed,
+    LedgerIntegrityError,
+    ReplayMismatch,
+    SignatureInvalid,
+)
+
+__all__ = [
+    "AssayError",
+    "CanonicalizationFailed",
+    "InsufficientSamples",
+    "InvalidScoreRequest",
+    "LedgerIntegrityError",
+    "ReplayMismatch",
+    "ScoringExtraMissing",
+    "SignatureInvalid",
+    "UnknownMetric",
+]
+
+
+class AssayError(Exception):
+    """Base class for every Assay scoring-face domain error."""
+
+    code: ClassVar[str] = "assay.error"
+
+
+class InvalidScoreRequest(AssayError):
+    """Inputs are malformed (length mismatch, empty, single-class)."""
+
+    code: ClassVar[str] = "assay.invalid_request"
+
+
+class UnknownMetric(AssayError):
+    """Requested metric name is not registered."""
+
+    code: ClassVar[str] = "assay.unknown_metric"
+
+
+class InsufficientSamples(AssayError):
+    """Sample count is below the abstention floor."""
+
+    code: ClassVar[str] = "assay.insufficient_samples"
+
+
+class ScoringExtraMissing(AssayError):
+    """The scoring face was imported without its heavy extra installed."""
+
+    code: ClassVar[str] = "assay.scoring_extra_missing"

assay/metrics.py

@@ -0,0 +1,78 @@
+"""Deterministic classification metrics — thin wrappers over scikit-learn.
+
+Assay reimplements no metric math; it validates inputs, delegates to sklearn, and
+returns an immutable, fully-typed result."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from sklearn.metrics import (
+    accuracy_score,
+    average_precision_score,
+    precision_recall_fscore_support,
+    roc_auc_score,
+)
+
+from assay.errors import InvalidScoreRequest
+
+_MIN_CLASSES = 2
+
+
+@dataclass(frozen=True)
+class ClassificationScores:
+    """Immutable bundle of binary-classification metrics."""
+
+    accuracy: float
+    precision: float
+    recall: float
+    f1: float
+    pr_auc: float
+    roc_auc: float
+
+
+def _validate(y_true: Sequence[int], y_score: Sequence[float]) -> None:
+    if len(y_true) != len(y_score):
+        raise InvalidScoreRequest("y_true and y_score length mismatch")
+    if len(y_true) == 0:
+        raise InvalidScoreRequest("inputs are empty")
+    if len(set(y_true)) < _MIN_CLASSES:
+        raise InvalidScoreRequest("need both classes present for AUC metrics")
+
+
+def _threshold(y_score: Sequence[float], threshold: float) -> list[int]:
+    return [1 if s >= threshold else 0 for s in y_score]
+
+
+def _prf(y_true: Sequence[int], y_pred: Sequence[int]) -> tuple[float, float, float]:
+    precision, recall, f1, _ = precision_recall_fscore_support(
+        y_true, y_pred, average="binary", zero_division=0.0
+    )
+    return float(precision), float(recall), float(f1)
+
+
+def binary_scores(
+    y_true: Sequence[int], y_score: Sequence[float], *, threshold: float = 0.5
+) -> ClassificationScores:
+    """Compute accuracy, precision, recall, F1, PR-AUC and ROC-AUC."""
+    _validate(y_true, y_score)
+    y_pred = _threshold(y_score, threshold)
+    precision, recall, f1 = _prf(y_true, y_pred)
+    return ClassificationScores(
+        accuracy=float(accuracy_score(y_true, y_pred)),
+        precision=precision,
+        recall=recall,
+        f1=f1,
+        pr_auc=float(average_precision_score(y_true, y_score)),
+        roc_auc=float(roc_auc_score(y_true, y_score)),
+    )
+
+
+def correctness(
+    y_true: Sequence[int], y_score: Sequence[float], *, threshold: float = 0.5
+) -> tuple[float, ...]:
+    """Return a per-example 1.0/0.0 correctness vector (for bootstrapping)."""
+    _validate(y_true, y_score)
+    y_pred = _threshold(y_score, threshold)
+    return tuple(float(int(p == t)) for p, t in zip(y_pred, y_true, strict=True))

assay/models.py

@@ -0,0 +1,42 @@
+"""Typed request models at the input boundary. Frozen and ``extra="forbid"`` so a
+malformed or ambiguous request is rejected before any computation."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ScoreRequest(BaseModel):
+    """A classification scoring request."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    metric: str
+    metric_version: str
+    y_true: tuple[int, ...]
+    y_score: tuple[float, ...]
+    threshold: float = 0.5
+
+
+class SubScoreInput(BaseModel):
+    """One sub-score with its native scale and interval, for a composite."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    name: str
+    value: float
+    low: float
+    high: float
+    scale_min: float
+    scale_max: float
+    weight: float
+
+
+class CompositeRequest(BaseModel):
+    """A weighted multi-scale composite request."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    metric: str = "weighted_composite"
+    metric_version: str
+    subscores: tuple[SubScoreInput, ...]