modelcertify 0.1.0__tar.gz → 0.2.0__tar.gz

{modelcertify-0.1.0/src/modelcertify.egg-info → modelcertify-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modelcertify
-Version: 0.1.0
+Version: 0.2.0
 Summary: Supply-chain certification for trained models
 License: Apache-2.0
 Classifier: Development Status :: 3 - Alpha
@@ -38,6 +38,13 @@ makes that run auditable and certifies it against a written policy.
 * A **certificate** is a policy evaluated against a run's evidence: it
   records every gate, how each one came out, the proof root it is bound to,
   and the issuer who signed it.
+* An **evaluation suite** (P4.x) is a private set of scored cases. Running it
+  produces a report whose **suite digest** is committed to the proof, so a
+  policy can gate on exactly which suite was used without shipping the suite.
+  The runner supports two modes: `quick` (deterministic subsample, for
+  iteration) and `full` (every case, for the attestable result). Scoring is a
+  dependency-free port of the decision-index primitives (accuracy, macro/binary
+  F1, NDCG@10, MRR, chance baselines).
 
 The certificate never overstates. It carries the gate results, so a reader can
 see *why* something was certified, not just that it was.
@@ -70,6 +77,7 @@ modelcertify policy --file policy.json --json
 modelcertify certify --evidence evidence.json --policy policy.json \
     --issuer scai --key "$MODELCERTIFY_KEY" --out certificate.json
 modelcertify verify --certificate certificate.json --key "$MODELCERTIFY_KEY"
+modelcertify eval --suite suite.json --mode full --run-id run-1 --out certificate.json
 ```
 
 Every command accepts `--json`.

{modelcertify-0.1.0 → modelcertify-0.2.0}/README.md

@@ -13,6 +13,13 @@ makes that run auditable and certifies it against a written policy.
 * A **certificate** is a policy evaluated against a run's evidence: it
   records every gate, how each one came out, the proof root it is bound to,
   and the issuer who signed it.
+* An **evaluation suite** (P4.x) is a private set of scored cases. Running it
+  produces a report whose **suite digest** is committed to the proof, so a
+  policy can gate on exactly which suite was used without shipping the suite.
+  The runner supports two modes: `quick` (deterministic subsample, for
+  iteration) and `full` (every case, for the attestable result). Scoring is a
+  dependency-free port of the decision-index primitives (accuracy, macro/binary
+  F1, NDCG@10, MRR, chance baselines).
 
 The certificate never overstates. It carries the gate results, so a reader can
 see *why* something was certified, not just that it was.
@@ -45,6 +52,7 @@ modelcertify policy --file policy.json --json
 modelcertify certify --evidence evidence.json --policy policy.json \
     --issuer scai --key "$MODELCERTIFY_KEY" --out certificate.json
 modelcertify verify --certificate certificate.json --key "$MODELCERTIFY_KEY"
+modelcertify eval --suite suite.json --mode full --run-id run-1 --out certificate.json
 ```
 
 Every command accepts `--json`.

{modelcertify-0.1.0 → modelcertify-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "modelcertify"
-version = "0.1.0"
+version = "0.2.0"
 description = "Supply-chain certification for trained models"
 readme = "README.md"
 license = {text = "Apache-2.0"}

{modelcertify-0.1.0 → modelcertify-0.2.0}/src/modelcertify/cli.py

@@ -6,6 +6,7 @@ Subcommands:
 ``policy``    show a certification policy
 ``certify``   evaluate a policy against a run's evidence
 ``verify``    verify a certificate's signature
+``eval``      run a private evaluation suite and certify the report
 
 Every command supports ``--json``.
 """
@@ -20,6 +21,7 @@ from pathlib import Path
 from typing import Any
 
 from .certify import Certificate, Policy, certify, load_evidence
+from .eval import EvalRunner, load_suite
 from .proof import Artifact, RunProof
 
 __all__ = ["build_parser", "main"]
@@ -76,6 +78,28 @@ def build_parser() -> argparse.ArgumentParser:
     verify.add_argument("--key", help="signing key (hex)")
     verify.add_argument("--json", action="store_true", help="emit JSON on stdout")
 
+    eval_cmd = sub.add_parser("eval", help="run a suite and certify the report")
+    eval_cmd.add_argument("--suite", required=True, help="suite JSON file")
+    eval_cmd.add_argument(
+        "--mode",
+        default="full",
+        choices=["quick", "full"],
+        help="quick = deterministic subsample; full = every case",
+    )
+    eval_cmd.add_argument("--metric", default="accuracy", help="report metric")
+    eval_cmd.add_argument(
+        "--model",
+        default="identity",
+        choices=["identity", "always-yes"],
+        help="built-in model function (for CLI demos)",
+    )
+    eval_cmd.add_argument("--run-id", default="run", help="run id for the proof")
+    eval_cmd.add_argument("--issuer", default="modelcertify", help="certificate issuer")
+    eval_cmd.add_argument("--key", help="signing key (hex)")
+    eval_cmd.add_argument("--out", help="write the certificate here")
+    eval_cmd.add_argument("--report-out", help="write the eval report JSON here")
+    eval_cmd.add_argument("--json", action="store_true", help="emit JSON on stdout")
+
     return parser
 
 
@@ -166,6 +190,52 @@ def _cmd_verify(args: argparse.Namespace) -> int:
     return 0 if verified else 1
 
 
+def _cmd_eval(args: argparse.Namespace) -> int:
+    suite = load_suite(args.suite)
+
+    def model_fn(question: dict[str, Any]) -> dict[str, Any]:
+        qtype = question.get("type", "choice")
+        if qtype == "noul":
+            return {"noul": 1.0 if args.model == "always-yes" else 0.0}
+        criteria = question.get("criteria") or {}
+        if qtype == "ranking":
+            return {"ranking": sorted(criteria, key=lambda k: str(k))}
+        first = sorted(criteria)[0] if criteria else "a"
+        return {"choice": first}
+
+    runner = EvalRunner(model_fn)
+    result = runner.run(suite, mode=args.mode, metric=args.metric)
+
+    if args.report_out:
+        Path(args.report_out).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.report_out).write_text(json.dumps(result.to_dict(), indent=2) + "\n")
+
+    key = _key(args.key) if args.key or os.environ.get("MODELCERTIFY_KEY") else None
+    certificate = runner.certify(
+        suite,
+        result,
+        run_id=args.run_id,
+        issuer=args.issuer,
+        key=key,
+    )
+    if args.out:
+        certificate.write(args.out)
+
+    payload = {
+        "eval": result.to_dict(),
+        "certificate": certificate.to_dict(),
+    }
+    if args.json:
+        _emit_json(payload)
+        return 0 if certificate.certified else 1
+    print(f"suite: {suite.name} v{suite.version} digest={suite.digest[:16]}…")
+    print(f"mode: {result.mode}  answered: {result.report.answered}/{result.report.total}")
+    print(f"metric: {result.report.metric} = {result.report.score:.4f} "
+          f"(chance {result.report.chance if result.report.chance is not None else 'n/a'})")
+    print(f"certified: {certificate.certified}")
+    return 0 if certificate.certified else 1
+
+
 def _emit_json(payload: Any) -> None:
     sys.stdout.write(json.dumps(payload, indent=2) + "\n")
 
@@ -182,6 +252,8 @@ def main(argv: Sequence[str] | None = None) -> int:
             return _cmd_certify(args)
         if args.command == "verify":
             return _cmd_verify(args)
+        if args.command == "eval":
+            return _cmd_eval(args)
     except (FileNotFoundError, ValueError, RuntimeError) as exc:
         if getattr(args, "json", False):
             _emit_json({"error": str(exc)})

modelcertify-0.2.0/src/modelcertify/eval.py

@@ -0,0 +1,302 @@
+"""Evaluation suites and the two-mode runner (P4.2/P4.3).
+
+A :class:`Suite` is a *private* set of evaluation cases: questions (with
+gold answers) bundled into a checksummed manifest. The checksum lets a
+runner prove *which* suite was used without shipping the suite itself — the
+manifest digest becomes an artifact in the run proof, so a certificate's
+artifact gates can require exactly the suite that was evaluated.
+
+The :class:`EvalRunner` runs a suite in one of two modes:
+
+* ``quick`` — a deterministic subsample of the suite (plus a mode field in
+  the report), for smoke checks and iteration.
+* ``full`` — every case, for the attestable result.
+
+Either way the report is JSON-serializable, can be attached to a
+:class:`RunProof` as an artifact, and feeds the metric gates in
+:mod:`modelcertify.certify`.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .proof import Artifact, RunProof
+from .scoring import ScoreReport, score_answers
+
+__all__ = [
+    "Case",
+    "Suite",
+    "EvalResult",
+    "EvalRunner",
+    "SuiteError",
+    "suite_manifest_digest",
+    "load_suite",
+]
+
+QUICK_FRACTION = 0.25
+QUICK_MIN = 2
+
+
+class SuiteError(ValueError):
+    """Raised when a suite is malformed."""
+
+
+@dataclass(frozen=True)
+class Case:
+    """One evaluation case: a question plus its gold answer."""
+
+    field: str
+    question: dict[str, Any]
+    gold: Any
+    weight: float = 1.0
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> Case:
+        return cls(
+            field=str(raw["field"]),
+            question=dict(raw["question"]),
+            gold=raw["gold"],
+            weight=float(raw.get("weight", 1.0)),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "field": self.field,
+            "question": self.question,
+            "gold": self.gold,
+            "weight": self.weight,
+        }
+
+
+@dataclass(frozen=True)
+class Suite:
+    """A private evaluation suite."""
+
+    name: str
+    version: str = "1"
+    cases: tuple[Case, ...] = ()
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def digest(self) -> str:
+        """The manifest digest: proves which suite ran, without shipping it."""
+        return suite_manifest_digest(self)
+
+    @property
+    def questions(self) -> dict[str, dict[str, Any]]:
+        questions: dict[str, dict[str, Any]] = {}
+        for case in self.cases:
+            entry = dict(case.question)
+            entry["gold"] = case.gold
+            questions[case.field] = entry
+        return questions
+
+    @property
+    def golds(self) -> dict[str, Any]:
+        return {case.field: case.gold for case in self.cases}
+
+    def sample(self, fraction: float = QUICK_FRACTION, minimum: int = QUICK_MIN) -> Suite:
+        """A deterministic subsample for the ``quick`` mode."""
+        cases = self.cases
+        if not cases:
+            return self
+        size = max(minimum, int(len(cases) * fraction))
+        size = min(size, len(cases))
+        step = max(1, len(cases) // size)
+        chosen = cases[::step][:size]
+        return Suite(
+            name=self.name,
+            version=self.version,
+            cases=chosen,
+            metadata={**self.metadata, "sampled": True, "sample_size": len(chosen)},
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "version": self.version,
+            "digest": self.digest,
+            "cases": [case.to_dict() for case in self.cases],
+            "metadata": dict(self.metadata),
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> Suite:
+        return cls(
+            name=str(raw["name"]),
+            version=str(raw.get("version", "1")),
+            cases=tuple(Case.from_dict(entry) for entry in raw.get("cases", [])),
+            metadata=dict(raw.get("metadata") or {}),
+        )
+
+    @classmethod
+    def read(cls, path: str | Path) -> Suite:
+        raw = json.loads(Path(path).read_text())
+        return cls.from_dict(raw)
+
+
+def suite_manifest_digest(suite: Suite) -> str:
+    """A digest over the suite's cases (not its metadata)."""
+    payload = {
+        "name": suite.name,
+        "version": suite.version,
+        "cases": [case.to_dict() for case in suite.cases],
+    }
+    body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+    return hashlib.sha256(body).hexdigest()
+
+
+def load_suite(path: str | Path) -> Suite:
+    return Suite.read(path)
+
+
+@dataclass(frozen=True)
+class EvalResult:
+    """The outcome of running a suite in one mode."""
+
+    suite_name: str
+    suite_version: str
+    suite_digest: str
+    mode: str
+    report: ScoreReport
+    answered: int = 0
+    started_at: str = ""
+    finished_at: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "suite": self.suite_name,
+            "suite_version": self.suite_version,
+            "suite_digest": self.suite_digest,
+            "mode": self.mode,
+            "answered": self.answered,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "report": self.report.to_dict(),
+        }
+
+    def as_artifact(self, name: str = "eval-report.json") -> Artifact:
+        return Artifact.from_bytes(
+            name,
+            json.dumps(self.to_dict()).encode(),
+            media_type="application/json",
+        )
+
+
+class EvalRunner:
+    """Runs a suite in ``quick`` or ``full`` mode."""
+
+    def __init__(
+        self,
+        model_fn: Callable[[dict[str, Any]], dict[str, Any]],
+        *,
+        quick_fraction: float = QUICK_FRACTION,
+        quick_min: int = QUICK_MIN,
+    ) -> None:
+        self.model_fn = model_fn
+        self.quick_fraction = quick_fraction
+        self.quick_min = quick_min
+
+    def run(self, suite: Suite, mode: str = "full", metric: str = "accuracy") -> EvalResult:
+        import datetime as _dt
+
+        if mode not in ("quick", "full"):
+            raise ValueError("mode must be 'quick' or 'full'")
+        chosen = suite.sample(self.quick_fraction, self.quick_min) if mode == "quick" else suite
+        started = _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
+        answers: dict[str, dict[str, Any]] = {}
+        answered = 0
+        for case in chosen.cases:
+            try:
+                answers[case.field] = dict(self.model_fn(dict(case.question)))
+                answered += 1
+            except Exception:
+                # A failed case is recorded as unanswered; it does not count
+                # as correct, so it still lowers the score.
+                continue
+        report = score_answers(chosen.questions, answers, metric=metric)
+        finished = _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
+        return EvalResult(
+            suite_name=suite.name,
+            suite_version=suite.version,
+            suite_digest=suite.digest,
+            mode=mode,
+            report=report,
+            answered=answered,
+            started_at=started,
+            finished_at=finished,
+        )
+
+    def certify(
+        self,
+        suite: Suite,
+        result: EvalResult,
+        *,
+        run_id: str,
+        issuer: str,
+        key: bytes | None = None,
+        policy: Any | None = None,
+        extra_artifacts: tuple[Artifact, ...] = (),
+    ) -> Any:
+        """Attach an eval result to a run proof and certify it.
+
+        The suite digest is recorded as an artifact too, so a policy can gate
+        on the exact suite used. Returns the certificate.
+        """
+        from .certify import certify as _certify
+
+        artifacts = list(extra_artifacts) + [
+            result.as_artifact(),
+            Artifact.from_bytes(
+                "suite-manifest.json",
+                json.dumps(
+                    {"name": suite.name, "version": suite.version, "digest": suite.digest}
+                ).encode(),
+                media_type="application/json",
+            ),
+        ]
+        proof = RunProof.build(run_id, artifacts)
+        from .certify import Evidence
+
+        evidence = Evidence(
+            proof=proof,
+            metrics={
+                "score": result.report.score,
+                "metric": result.report.metric,
+                "coverage": result.report.coverage,
+                "answered": result.report.answered,
+                "mode": result.mode,
+            },
+            provenance={
+                "suite": suite.name,
+                "suite_version": suite.version,
+                "suite_digest": suite.digest,
+            },
+        )
+        policy = policy or _default_policy(suite)
+        return _certify(policy, evidence, issuer=issuer, subject=run_id, key=key)
+
+
+def _default_policy(suite: Suite) -> Any:
+    from .certify import Gate, Policy
+
+    return Policy(
+        policy_id="eval-baseline",
+        name="Evaluation baseline policy",
+        version="1",
+        required_artifacts=("eval-report.json", "suite-manifest.json"),
+        gates=(
+            Gate(name="score_above_chance", kind="metric", field="score", op=">", threshold=0.5),
+            Gate(
+                name="suite_digest_present",
+                kind="provenance",
+                field="suite_digest",
+                threshold=None,
+            ),
+        ),
+    )

modelcertify-0.2.0/src/modelcertify/scoring.py

@@ -0,0 +1,220 @@
+"""Scoring primitives for model evaluation.
+
+This module is a dependency-free port of the scoring core used by the
+decision-index benchmarks (accuracy, macro/binary F1, NDCG@10, MRR, chance
+baselines). It is deliberately generic: a scorer looks at a question's
+accepted answers (gold) and a model's answer and produces a per-field score,
+and an aggregate report is computed over a whole run. The report is what a
+certificate's metric gates can read.
+"""
+from __future__ import annotations
+
+import json
+import math
+import statistics
+from dataclasses import dataclass, field
+from typing import Any
+
+__all__ = [
+    "ScoreReport",
+    "score_answers",
+    "score_choice",
+    "score_noul",
+    "score_ranking",
+    "macro_f1",
+    "binary_f1",
+    "chance_for",
+    "mean",
+    "avg",
+]
+
+
+def mean(values: list[float]) -> float | None:
+    return statistics.mean(values) if values else None
+
+
+def avg(values: list[float]) -> float:
+    return statistics.mean(values) if values else 0.0
+
+
+def macro_f1(pairs: list[tuple[str, str]]) -> float | None:
+    """Macro-averaged F1 over the class labels seen in ``pairs``."""
+    labels: set[str] = set()
+    for gold, pred in pairs:
+        labels.add(gold)
+        labels.add(pred)
+    per_label: list[float] = []
+    for label in sorted(labels):
+        tp = sum(g == label and p == label for g, p in pairs)
+        fp = sum(g != label and p == label for g, p in pairs)
+        fn = sum(g == label and p != label for g, p in pairs)
+        denom = 2 * tp + fp + fn
+        per_label.append(2 * tp / denom if denom else 0.0)
+    return mean(per_label)
+
+
+def binary_f1(pairs: list[tuple[str, str]], positive: str = "yes") -> float:
+    tp = sum(g == positive and p == positive for g, p in pairs)
+    fp = sum(g != positive and p == positive for g, p in pairs)
+    fn = sum(g == positive and p != positive for g, p in pairs)
+    denom = 2 * tp + fp + fn
+    return 2 * tp / denom if denom else 0.0
+
+
+def chance_for(question: dict[str, Any]) -> float:
+    """The expected score of a random guess for a question."""
+    qtype = question.get("type", "choice")
+    if qtype == "noul":
+        return 0.5
+    criteria = question.get("criteria") or {}
+    n = len(criteria)
+    return 1.0 / n if n else 0.0
+
+
+@dataclass(frozen=True)
+class ScoreReport:
+    """Aggregate scores over one evaluation run."""
+
+    metric: str = "accuracy"
+    score: float = 0.0
+    chance: float | None = None
+    answered: int = 0
+    total: int = 0
+    per_field: dict[str, dict[str, Any]] = field(default_factory=dict)
+    details: list[dict[str, Any]] = field(default_factory=list)
+
+    @property
+    def coverage(self) -> float:
+        return self.answered / self.total if self.total else 1.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "metric": self.metric,
+            "score": self.score,
+            "chance": self.chance,
+            "answered": self.answered,
+            "total": self.total,
+            "coverage": self.coverage,
+            "per_field": dict(self.per_field),
+            "details": list(self.details),
+        }
+
+
+def score_choice(question: dict[str, Any], answer: dict[str, Any], gold: Any) -> dict[str, Any]:
+    """Score a discrete-choice question."""
+    pred = answer.get("choice")
+    correct = pred == gold
+    return {
+        "correct": correct,
+        "pred": pred,
+        "gold": gold,
+        "chance": chance_for(question),
+    }
+
+
+def score_noul(question: dict[str, Any], answer: dict[str, Any], gold: Any) -> dict[str, Any]:
+    """Score a "no-unsupported-label" (boolean) question."""
+    value = answer.get("noul", 0.0)
+    pred = value >= 0.5
+    return {
+        "correct": pred == gold,
+        "pred": pred,
+        "gold": gold,
+        "confidence": max(float(value), 1.0 - float(value)),
+        "chance": 0.5,
+    }
+
+
+def score_ranking(question: dict[str, Any], answer: dict[str, Any], gold: Any) -> dict[str, Any]:
+    """Score a ranking question with NDCG@10 and MRR.
+
+    ``answer["ranking"]`` is a list of candidate ids in the model's order,
+    most relevant first. ``gold`` maps candidate id -> relevance grade.
+    """
+    ranking = answer.get("ranking") or []
+    qrels = dict(gold)
+    graded = [qrels.get(d, 0) for d in ranking]
+    ideal = sorted(qrels.values(), reverse=True)[:10]
+    dcg = sum((2**v - 1) / math.log2(i + 2) for i, v in enumerate(graded[:10]))
+    idcg = sum((2**v - 1) / math.log2(i + 2) for i, v in enumerate(ideal))
+    positives = sum(v > 0 for v in qrels.values())
+    mrr = next((1 / (i + 1) for i, v in enumerate(graded) if v > 0), 0.0)
+    recall10 = sum(v > 0 for v in graded[:10]) / positives if positives else 0.0
+    return {
+        "ndcg_at_10": dcg / idcg if idcg else 0.0,
+        "mrr": mrr,
+        "recall_at_10": recall10,
+        "chance": 0.0,
+        "correct": mrr > 0,
+    }
+
+
+_SCORERS = {
+    "choice": score_choice,
+    "noul": score_noul,
+    "ranking": score_ranking,
+}
+
+
+def score_answers(
+    questions: dict[str, dict[str, Any]],
+    answers: dict[str, dict[str, Any]],
+    *,
+    metric: str = "accuracy",
+) -> ScoreReport:
+    """Score a model's answers against gold for a set of questions.
+
+    ``questions`` maps field key -> question dict (with ``type`` and, for
+    choice questions, ``criteria``). ``answers`` maps field key -> the model's
+    answer: ``{"choice": ...}``, ``{"noul": <probability>}`` or
+    ``{"ranking": [...]}``. Gold truth lives in ``question["gold"]``.
+    """
+    correct = 0
+    answered = 0
+    per_field: dict[str, dict[str, Any]] = {}
+    details: list[dict[str, Any]] = []
+    for key, question in questions.items():
+        if key not in answers:
+            per_field[key] = {"answered": False, "correct": False, "chance": chance_for(question)}
+            continue
+        qtype = question.get("type", "choice")
+        scorer = _SCORERS.get(qtype, score_choice)
+        gold = question.get("gold")
+        scored = scorer(question, answers[key], gold)
+        answered += 1
+        correct += 1 if scored.get("correct") else 0
+        per_field[key] = {"answered": True, **scored}
+        details.append({"field": key, **scored})
+    chance = mean([chance_for(question) for question in questions.values()]) if questions else None
+    score = correct / answered if answered else 0.0
+    report = ScoreReport(
+        metric=metric,
+        score=score,
+        chance=chance,
+        answered=answered,
+        total=len(questions),
+        per_field=per_field,
+        details=details,
+    )
+    if metric == "macro_f1":
+        pairs = [
+            (str(d["gold"]), str(d["pred"]))
+            for d in details
+            if "pred" in d and "gold" in d
+        ]
+        f1 = macro_f1(pairs)
+        if f1 is not None:
+            report = ScoreReport(
+                metric=metric,
+                score=f1,
+                chance=chance,
+                answered=answered,
+                total=len(questions),
+                per_field=per_field,
+                details=details,
+            )
+    return report
+
+
+def report_to_json(report: ScoreReport) -> str:
+    return json.dumps(report.to_dict(), indent=2, sort_keys=True) + "\n"

{modelcertify-0.1.0 → modelcertify-0.2.0/src/modelcertify.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modelcertify
-Version: 0.1.0
+Version: 0.2.0
 Summary: Supply-chain certification for trained models
 License: Apache-2.0
 Classifier: Development Status :: 3 - Alpha
@@ -38,6 +38,13 @@ makes that run auditable and certifies it against a written policy.
 * A **certificate** is a policy evaluated against a run's evidence: it
   records every gate, how each one came out, the proof root it is bound to,
   and the issuer who signed it.
+* An **evaluation suite** (P4.x) is a private set of scored cases. Running it
+  produces a report whose **suite digest** is committed to the proof, so a
+  policy can gate on exactly which suite was used without shipping the suite.
+  The runner supports two modes: `quick` (deterministic subsample, for
+  iteration) and `full` (every case, for the attestable result). Scoring is a
+  dependency-free port of the decision-index primitives (accuracy, macro/binary
+  F1, NDCG@10, MRR, chance baselines).
 
 The certificate never overstates. It carries the gate results, so a reader can
 see *why* something was certified, not just that it was.
@@ -70,6 +77,7 @@ modelcertify policy --file policy.json --json
 modelcertify certify --evidence evidence.json --policy policy.json \
     --issuer scai --key "$MODELCERTIFY_KEY" --out certificate.json
 modelcertify verify --certificate certificate.json --key "$MODELCERTIFY_KEY"
+modelcertify eval --suite suite.json --mode full --run-id run-1 --out certificate.json
 ```
 
 Every command accepts `--json`.

{modelcertify-0.1.0 → modelcertify-0.2.0}/src/modelcertify.egg-info/SOURCES.txt

@@ -5,8 +5,10 @@ src/modelcertify/__init__.py
 src/modelcertify/attestation.py
 src/modelcertify/certify.py
 src/modelcertify/cli.py
+src/modelcertify/eval.py
 src/modelcertify/proof.py
 src/modelcertify/py.typed
+src/modelcertify/scoring.py
 src/modelcertify.egg-info/PKG-INFO
 src/modelcertify.egg-info/SOURCES.txt
 src/modelcertify.egg-info/dependency_links.txt
@@ -16,4 +18,6 @@ src/modelcertify.egg-info/top_level.txt
 tests/test_attestation.py
 tests/test_certify.py
 tests/test_cli.py
-tests/test_proof.py
+tests/test_eval.py
+tests/test_proof.py
+tests/test_scoring.py

modelcertify-0.2.0/tests/test_eval.py

@@ -0,0 +1,119 @@
+"""Tests for evaluation suites and the two-mode runner (P4.2/P4.3)."""
+from __future__ import annotations
+
+import pytest
+
+from modelcertify.eval import Case, EvalResult, EvalRunner, Suite, load_suite, suite_manifest_digest
+
+
+def make_suite(name: str = "toy", count: int = 8, gold: str = "a") -> Suite:
+    cases = []
+    for i in range(count):
+        cases.append(
+            Case(
+                field=f"q{i}",
+                question={"type": "choice", "criteria": {"a": "A", "b": "B"}},
+                gold=gold,
+            )
+        )
+    return Suite(name=name, version="1", cases=tuple(cases))
+
+
+def test_suite_digest_stable():
+    a, b = make_suite(), make_suite()
+    assert a.digest == b.digest
+    assert suite_manifest_digest(a) == a.digest
+    changed = Suite(name=a.name, cases=a.cases[:1])
+    assert changed.digest != a.digest
+
+
+def test_suite_roundtrip(tmp_path):
+    suite = make_suite("roundtrip")
+    path = tmp_path / "suite.json"
+    path.write_text(__import__("json").dumps(suite.to_dict(), indent=2))
+    loaded = load_suite(path)
+    assert loaded.name == suite.name
+    assert loaded.digest == suite.digest
+    assert len(loaded.cases) == len(suite.cases)
+
+
+def test_quick_mode_subsamples():
+    suite = make_suite(count=40)
+    quick = suite.sample()
+    assert len(quick.cases) < len(suite.cases)
+    assert quick.metadata.get("sampled") is True
+    # Deterministic: same suite samples the same way.
+    assert suite.sample().cases == suite.sample().cases
+
+
+def _always_first(question: dict) -> dict:
+    if question.get("type") == "noul":
+        return {"noul": 1.0}
+    return {"choice": "a"}
+
+
+def test_runner_full_mode_perfect_score():
+    suite = make_suite(count=6, gold="a")
+    result = EvalRunner(_always_first).run(suite, mode="full")
+    assert isinstance(result, EvalResult)
+    assert result.mode == "full"
+    assert result.report.answered == 6
+    assert result.report.score == 1.0
+
+
+def test_runner_quick_mode_records_mode():
+    suite = make_suite(count=40)
+    result = EvalRunner(_always_first).run(suite, mode="quick")
+    assert result.mode == "quick"
+    assert result.report.answered < 40
+    assert result.suite_digest == suite.digest
+
+
+def test_runner_unknown_mode_rejected():
+    with pytest.raises(ValueError):
+        EvalRunner(_always_first).run(make_suite(), mode="slow")
+
+
+def test_runner_failed_cases_lower_score():
+    suite = make_suite(count=4, gold="a")
+
+    def flaky_on_q0(question: dict) -> dict:
+        raise RuntimeError("boom")
+
+    result = EvalRunner(flaky_on_q0).run(suite, mode="full")
+    assert result.report.answered == 0
+    assert result.report.score == 0.0
+
+
+def test_eval_result_artifact_and_certify():
+    from modelcertify.certify import Certificate
+
+    suite = make_suite(count=4, gold="a")
+    runner = EvalRunner(_always_first)
+    result = runner.run(suite, mode="full")
+    artifact = result.as_artifact()
+    assert artifact.name == "eval-report.json"
+    assert artifact.size_bytes > 0
+    certificate = runner.certify(
+        suite, result, run_id="run-1", issuer="acme", key=b"k" * 32
+    )
+    assert isinstance(certificate, Certificate)
+    assert certificate.certified
+    assert certificate.attestation.verify_signature(b"k" * 32)
+    # The suite digest is committed to the proof provenance.
+    policy_artifacts = certificate.policy.required_artifacts
+    assert "eval-report.json" in policy_artifacts
+    assert "suite-manifest.json" in policy_artifacts
+
+
+def test_certify_fails_below_chance():
+    # The baseline policy requires score > 0.5; an all-wrong model fails.
+    suite = make_suite(count=4, gold="a")
+
+    def wrong(question: dict) -> dict:
+        return {"choice": "b"}
+
+    runner = EvalRunner(wrong)
+    result = runner.run(suite, mode="full")
+    certificate = runner.certify(suite, result, run_id="run-2", issuer="acme")
+    assert not certificate.certified

modelcertify-0.2.0/tests/test_scoring.py

@@ -0,0 +1,76 @@
+"""Tests for the scoring primitives (P4.1 port)."""
+from __future__ import annotations
+
+from modelcertify.scoring import binary_f1, macro_f1, score_answers, score_ranking
+
+
+def test_macro_f1():
+    pairs = [
+        ("a", "a"),
+        ("a", "b"),
+        ("b", "b"),
+        ("c", "c"),
+        ("c", "d"),
+    ]
+    f1 = macro_f1(pairs)
+    assert f1 is not None
+    assert 0.0 < f1 < 1.0
+    # Perfect predictions give 1.0.
+    assert macro_f1([("a", "a"), ("b", "b")]) == 1.0
+    assert macro_f1([]) is None
+
+
+def test_binary_f1():
+    pairs = [("yes", "yes"), ("yes", "no"), ("no", "no")]
+    assert binary_f1(pairs, positive="yes") == 2 / 3
+    assert binary_f1([("no", "no")], positive="yes") == 0.0
+
+
+def test_score_answers_accuracy():
+    questions = {
+        "q1": {"type": "choice", "criteria": {"a": "A", "b": "B"}, "gold": "a"},
+        "q2": {"type": "choice", "criteria": {"a": "A", "b": "B"}, "gold": "b"},
+        "q3": {"type": "noul", "gold": True},
+    }
+    answers = {
+        "q1": {"choice": "a"},
+        "q2": {"choice": "a"},  # wrong
+        "q3": {"noul": 0.9},
+    }
+    report = score_answers(questions, answers)
+    assert report.answered == 3
+    assert report.total == 3
+    assert report.score == 2 / 3
+    assert report.per_field["q1"]["correct"] is True
+    assert report.per_field["q2"]["correct"] is False
+
+
+def test_score_answers_macro_f1():
+    questions = {
+        "q1": {"type": "choice", "criteria": {"a": "A", "b": "B"}, "gold": "a"},
+        "q2": {"type": "choice", "criteria": {"a": "A", "b": "B"}, "gold": "b"},
+    }
+    answers = {"q1": {"choice": "a"}, "q2": {"choice": "b"}}
+    report = score_answers(questions, answers, metric="macro_f1")
+    assert report.metric == "macro_f1"
+    assert report.score == 1.0
+
+
+def test_score_answers_unanswered_field():
+    questions = {"q1": {"type": "choice", "criteria": {"a": "A"}, "gold": "a"}}
+    report = score_answers(questions, {})
+    assert report.answered == 0
+    assert report.score == 0.0
+    assert report.per_field["q1"]["answered"] is False
+
+
+def test_score_ranking_ndcg():
+    question = {
+        "type": "ranking",
+        "gold": {"d1": 2, "d2": 1, "d3": 0},
+    }
+    answer = {"ranking": ["d1", "d2", "d3"]}
+    scored = score_ranking(question, answer, question["gold"])
+    assert scored["ndcg_at_10"] == 1.0
+    assert scored["mrr"] == 1.0
+    assert scored["correct"] is True