modelcertify 0.1.0__py3-none-any.whl

modelcertify/__init__.py

@@ -0,0 +1,63 @@
+"""modelcertify: certify the supply chain of a trained model.
+
+A model is only as trustworthy as the run that produced it. modelcertify
+makes that run auditable:
+
+* :mod:`modelcertify.proof` builds a **run proof**: every artefact of the run
+  hashed, combined into a Merkle **root** that changes if any artefact does.
+* :mod:`modelcertify.attestation` lets an issuer **vouch** for a proof root,
+  signed so tampering is detectable.
+* :mod:`modelcertify.certify` checks a run against a **policy** -- metric,
+  artefact and provenance gates -- and issues a **certificate** that records
+  every gate and how it came out.
+
+The certificate never overstates: it carries the gate results, the proof root
+it is bound to, and the issuer that signed it.
+"""
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .attestation import (
+    Attestation,
+    AttestationError,
+    sign_body,
+    verify_body,
+)
+from .certify import (
+    Certificate,
+    Evidence,
+    Gate,
+    GateResult,
+    Policy,
+    PolicyResult,
+    certify,
+    evidence_from_dict,
+    load_evidence,
+)
+from .proof import Artifact, RunProof, merkle_root, sha256_bytes, sha256_file
+
+__all__ = [
+    "__version__",
+    # proof
+    "Artifact",
+    "RunProof",
+    "merkle_root",
+    "sha256_bytes",
+    "sha256_file",
+    # attestation
+    "Attestation",
+    "AttestationError",
+    "sign_body",
+    "verify_body",
+    # certification
+    "Certificate",
+    "Evidence",
+    "Gate",
+    "GateResult",
+    "Policy",
+    "PolicyResult",
+    "certify",
+    "evidence_from_dict",
+    "load_evidence",
+]

modelcertify/attestation.py

@@ -0,0 +1,148 @@
+"""Attestations: a signed statement about a run.
+
+An :class:`Attestation` says "issuer X vouches that run Y, whose proof roots
+to Z, meets statement S". It is signed with an HMAC-SHA256 over the
+canonical body, so anyone holding the issuer's key can verify it and nobody
+else can forge it.
+
+This is deliberately not a public-key signature. modelcertify has to verify
+attestations from several organisations, and shipping a key-distribution
+ceremony before there is a customer to use it is how key ceremonies get
+mired. The interface is narrow enough to swap in Ed25519 later: only
+:meth:`Attestation.sign` and :meth:`Attestation.verify_signature` know how
+the bytes are covered.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import hashlib
+import hmac
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+__all__ = ["Attestation", "AttestationError", "sign_body", "verify_body"]
+
+
+class AttestationError(ValueError):
+    """Raised when an attestation is malformed or its signature is wrong."""
+
+
+def _canonical(payload: dict[str, Any]) -> bytes:
+    return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+
+
+def sign_body(body: dict[str, Any], key: bytes) -> str:
+    """Sign an attestation body with the issuer's key."""
+    if not key:
+        raise AttestationError("an attestation key must not be empty")
+    return hmac.new(key, _canonical(body), hashlib.sha256).hexdigest()
+
+
+def verify_body(body: dict[str, Any], signature: str, key: bytes) -> bool:
+    """Check a signature over an attestation body."""
+    try:
+        expected = sign_body(body, key)
+    except AttestationError:
+        return False
+    return hmac.compare_digest(expected, signature)
+
+
+@dataclass(frozen=True)
+class Attestation:
+    """A signed statement about a run."""
+
+    issuer: str
+    run_id: str
+    subject: str
+    proof_root: str
+    policy_id: str = ""
+    issued_at: str = ""
+    claims: dict[str, Any] = field(default_factory=dict)
+    signature: str = ""
+
+    def __post_init__(self) -> None:
+        if not self.issuer:
+            raise AttestationError("an attestation needs an issuer")
+        if not self.run_id:
+            raise AttestationError("an attestation needs a run id")
+        if not self.proof_root:
+            raise AttestationError("an attestation needs a proof root")
+
+    def body(self) -> dict[str, Any]:
+        """The signed part of the attestation."""
+        return {
+            "issuer": self.issuer,
+            "run_id": self.run_id,
+            "subject": self.subject,
+            "proof_root": self.proof_root,
+            "policy_id": self.policy_id,
+            "issued_at": self.issued_at,
+            "claims": dict(self.claims),
+        }
+
+    def sign(self, key: bytes) -> Attestation:
+        """Return a copy of this attestation, signed with ``key``."""
+        return Attestation(
+            issuer=self.issuer,
+            run_id=self.run_id,
+            subject=self.subject,
+            proof_root=self.proof_root,
+            policy_id=self.policy_id,
+            issued_at=self.issued_at,
+            claims=dict(self.claims),
+            signature=sign_body(self.body(), key),
+        )
+
+    def verify_signature(self, key: bytes) -> bool:
+        """Whether the signature is valid for ``key``."""
+        return verify_body(self.body(), self.signature, key)
+
+    def verify_proof(self, proof_root: str) -> bool:
+        """Whether this attestation refers to the given proof root."""
+        return hmac.compare_digest(self.proof_root, proof_root)
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = self.body()
+        payload["signature"] = self.signature
+        return payload
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> Attestation:
+        try:
+            return cls(
+                issuer=str(raw["issuer"]),
+                run_id=str(raw["run_id"]),
+                subject=str(raw["subject"]),
+                proof_root=str(raw["proof_root"]),
+                policy_id=str(raw.get("policy_id", "")),
+                issued_at=str(raw.get("issued_at", "")),
+                claims=dict(raw.get("claims") or {}),
+                signature=str(raw.get("signature", "")),
+            )
+        except (KeyError, TypeError) as exc:
+            raise AttestationError(f"invalid attestation: {exc}") from exc
+
+    @classmethod
+    def issue(
+        cls,
+        *,
+        issuer: str,
+        run_id: str,
+        subject: str,
+        proof_root: str,
+        policy_id: str = "",
+        claims: dict[str, Any] | None = None,
+        issued_at: str | None = None,
+    ) -> Attestation:
+        """Create an unsigned attestation."""
+        stamp = issued_at or _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
+        return cls(
+            issuer=issuer,
+            run_id=run_id,
+            subject=subject,
+            proof_root=proof_root,
+            policy_id=policy_id,
+            issued_at=stamp,
+            claims=dict(claims or {}),
+        )

modelcertify/certify.py

@@ -0,0 +1,356 @@
+"""Certification: the policy, the evidence, the certificate.
+
+A policy is a set of gates. Each gate checks one fact about the run:
+
+* a **metric gate** compares a measured metric against a threshold
+  (``loss <= 0.5``, ``eval_accuracy >= 0.92``);
+* an **artifact gate** requires a named artefact to exist in the proof;
+* a **provenance gate** requires a named field in the run's provenance
+  record (dataset id, licence, consent reference, ...).
+
+A run is certified when every gate passes. The result is a
+:class:`Certificate`: the attestation, the policy, and the outcome of every
+gate, so a reader can see *why* something was certified and not just that
+it was.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .attestation import Attestation
+from .proof import Artifact, RunProof
+
+__all__ = [
+    "Certificate",
+    "Evidence",
+    "Gate",
+    "GateResult",
+    "Policy",
+    "PolicyResult",
+    "certify",
+]
+
+
+@dataclass(frozen=True)
+class Evidence:
+    """Everything a policy is allowed to look at."""
+
+    proof: RunProof
+    metrics: Mapping[str, Any] = field(default_factory=dict)
+    provenance: Mapping[str, Any] = field(default_factory=dict)
+
+    def metric(self, name: str) -> Any:
+        return self.metrics.get(name)
+
+    def artifact(self, name: str) -> Artifact | None:
+        return self.proof.artifact(name)
+
+
+@dataclass(frozen=True)
+class Gate:
+    """One check in a policy."""
+
+    name: str
+    kind: str  # metric | artifact | provenance
+    field: str = ""
+    op: str = ">="
+    threshold: Any = None
+
+    def __post_init__(self) -> None:
+        if self.kind not in ("metric", "artifact", "provenance"):
+            raise ValueError(f"unknown gate kind {self.kind!r}")
+        if not self.name:
+            raise ValueError("a gate needs a name")
+
+    def evaluate(self, evidence: Evidence) -> GateResult:
+        """Run this gate against the evidence."""
+        if self.kind == "metric":
+            return self._evaluate_metric(evidence)
+        if self.kind == "artifact":
+            present = evidence.artifact(self.field) is not None
+            return GateResult(
+                gate=self.name,
+                passed=present,
+                detail=(
+                    f"artifact {self.field!r} is present"
+                    if present
+                    else f"artifact {self.field!r} is missing"
+                ),
+            )
+        value = evidence.provenance.get(self.field)
+        if self.threshold is None:
+            # A provenance gate without a threshold asks for the field to be
+            # there: "this run recorded a dataset id".
+            present = value is not None
+            return GateResult(
+                gate=self.name,
+                passed=present,
+                detail=(
+                    f"provenance {self.field!r} is recorded"
+                    if present
+                    else f"provenance {self.field!r} is missing"
+                ),
+            )
+        return self._compare(value, f"provenance {self.field!r}")
+
+    def _evaluate_metric(self, evidence: Evidence) -> GateResult:
+        if self.field not in evidence.metrics:
+            return GateResult(
+                gate=self.name,
+                passed=False,
+                detail=f"metric {self.field!r} was not reported",
+            )
+        value = evidence.metrics[self.field]
+        return self._compare(value, f"metric {self.field!r}")
+
+    def _compare(self, value: Any, label: str) -> GateResult:
+        if value is None:
+            return GateResult(gate=self.name, passed=False, detail=f"{label} is missing")
+        threshold = self.threshold
+        try:
+            if self.op == ">=":
+                passed = float(value) >= float(threshold)
+            elif self.op == "<=":
+                passed = float(value) <= float(threshold)
+            elif self.op == ">":
+                passed = float(value) > float(threshold)
+            elif self.op == "<":
+                passed = float(value) < float(threshold)
+            elif self.op == "==":
+                passed = value == threshold
+            elif self.op == "!=":
+                passed = value != threshold
+            elif self.op == "in":
+                passed = value in threshold
+            else:
+                raise ValueError(f"unknown operator {self.op!r}")
+        except (TypeError, ValueError) as exc:
+            return GateResult(gate=self.name, passed=False, detail=f"{label}: {exc}")
+        return GateResult(
+            gate=self.name,
+            passed=passed,
+            detail=f"{label} = {value!r} {self.op} {threshold!r} -> {'pass' if passed else 'fail'}",
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "kind": self.kind,
+            "field": self.field,
+            "op": self.op,
+            "threshold": self.threshold,
+        }
+
+    @classmethod
+    def from_dict(cls, raw: Mapping[str, Any]) -> Gate:
+        return cls(
+            name=str(raw["name"]),
+            kind=str(raw["kind"]),
+            field=str(raw.get("field", "")),
+            op=str(raw.get("op", ">=")),
+            threshold=raw.get("threshold"),
+        )
+
+
+@dataclass(frozen=True)
+class GateResult:
+    """The outcome of one gate."""
+
+    gate: str
+    passed: bool
+    detail: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"gate": self.gate, "passed": self.passed, "detail": self.detail}
+
+
+@dataclass(frozen=True)
+class Policy:
+    """A named set of gates."""
+
+    policy_id: str
+    name: str = ""
+    version: str = "1"
+    gates: tuple[Gate, ...] = ()
+    required_artifacts: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        if not self.policy_id:
+            raise ValueError("a policy needs a policy_id")
+
+    def evaluate(self, evidence: Evidence) -> PolicyResult:
+        """Run every gate against the evidence."""
+        results: list[GateResult] = []
+        for required in self.required_artifacts:
+            present = evidence.artifact(required) is not None
+            results.append(
+                GateResult(
+                    gate=f"required_artifact:{required}",
+                    passed=present,
+                    detail=(
+                        f"artifact {required!r} is present"
+                        if present
+                        else f"artifact {required!r} is missing"
+                    ),
+                )
+            )
+        for gate in self.gates:
+            results.append(gate.evaluate(evidence))
+        failed = tuple(result.gate for result in results if not result.passed)
+        return PolicyResult(passed=not failed, results=tuple(results), failed=failed)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "policy_id": self.policy_id,
+            "name": self.name,
+            "version": self.version,
+            "required_artifacts": list(self.required_artifacts),
+            "gates": [gate.to_dict() for gate in self.gates],
+        }
+
+    @classmethod
+    def from_dict(cls, raw: Mapping[str, Any]) -> Policy:
+        return cls(
+            policy_id=str(raw["policy_id"]),
+            name=str(raw.get("name", "")),
+            version=str(raw.get("version", "1")),
+            gates=tuple(Gate.from_dict(entry) for entry in raw.get("gates", [])),
+            required_artifacts=tuple(
+                str(entry) for entry in raw.get("required_artifacts", [])
+            ),
+        )
+
+    @classmethod
+    def read(cls, path: str | Path) -> Policy:
+        return cls.from_dict(json.loads(Path(path).read_text()))
+
+
+@dataclass(frozen=True)
+class PolicyResult:
+    """The outcome of evaluating a policy."""
+
+    passed: bool
+    results: tuple[GateResult, ...] = ()
+    failed: tuple[str, ...] = ()
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "passed": self.passed,
+            "failed": list(self.failed),
+            "results": [result.to_dict() for result in self.results],
+        }
+
+
+@dataclass(frozen=True)
+class Certificate:
+    """A certification: an attestation plus the policy outcome."""
+
+    attestation: Attestation
+    policy: Policy
+    result: PolicyResult
+    issued_at: str = ""
+    certificate_id: str = ""
+
+    @property
+    def certified(self) -> bool:
+        return self.result.passed
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "certificate_id": self.certificate_id,
+            "issued_at": self.issued_at,
+            "certified": self.certified,
+            "attestation": self.attestation.to_dict(),
+            "policy": self.policy.to_dict(),
+            "result": self.result.to_dict(),
+        }
+
+    @classmethod
+    def from_dict(cls, raw: Mapping[str, Any]) -> Certificate:
+        return cls(
+            attestation=Attestation.from_dict(raw["attestation"]),
+            policy=Policy.from_dict(raw["policy"]),
+            result=PolicyResult(
+                passed=bool(raw["result"]["passed"]),
+                results=tuple(
+                    GateResult(
+                        gate=str(entry["gate"]),
+                        passed=bool(entry["passed"]),
+                        detail=str(entry.get("detail", "")),
+                    )
+                    for entry in raw["result"].get("results", [])
+                ),
+                failed=tuple(str(entry) for entry in raw["result"].get("failed", [])),
+            ),
+            issued_at=str(raw.get("issued_at", "")),
+            certificate_id=str(raw.get("certificate_id", "")),
+        )
+
+    def write(self, path: str | Path) -> Path:
+        target = Path(path)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(json.dumps(self.to_dict(), indent=2) + "\n")
+        return target
+
+
+def certify(
+    policy: Policy,
+    evidence: Evidence,
+    *,
+    issuer: str,
+    subject: str,
+    key: bytes | None = None,
+    claims: Mapping[str, Any] | None = None,
+) -> Certificate:
+    """Evaluate a policy against the evidence and issue a certificate.
+
+    The attestation is signed when ``key`` is given. A policy that fails
+    still produces a certificate: the point of a certificate is to record
+    the outcome, including a refusal.
+    """
+    result = policy.evaluate(evidence)
+    stamp = _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
+    attestation = Attestation.issue(
+        issuer=issuer,
+        run_id=evidence.proof.run_id,
+        subject=subject,
+        proof_root=evidence.proof.root,
+        policy_id=policy.policy_id,
+        claims=dict(claims or {}),
+        issued_at=stamp,
+    )
+    if key:
+        attestation = attestation.sign(key)
+    certificate_id = f"cert-{evidence.proof.run_id}-{policy.policy_id}"
+    return Certificate(
+        attestation=attestation,
+        policy=policy,
+        result=result,
+        issued_at=stamp,
+        certificate_id=certificate_id,
+    )
+
+
+def load_evidence(path: str | Path) -> Evidence:
+    """Load an evidence document (proof + metrics + provenance)."""
+    raw = json.loads(Path(path).read_text())
+    proof = RunProof.from_dict(raw["proof"])
+    return Evidence(
+        proof=proof,
+        metrics=dict(raw.get("metrics") or {}),
+        provenance=dict(raw.get("provenance") or {}),
+    )
+
+
+def evidence_from_dict(raw: Mapping[str, Any]) -> Evidence:
+    """Build evidence from a document."""
+    return Evidence(
+        proof=RunProof.from_dict(raw["proof"]),
+        metrics=dict(raw.get("metrics") or {}),
+        provenance=dict(raw.get("provenance") or {}),
+    )

modelcertify/cli.py

@@ -0,0 +1,196 @@
+"""``modelcertify`` command line interface.
+
+Subcommands:
+
+``proof``     build a run proof from a directory of artefacts
+``policy``    show a certification policy
+``certify``   evaluate a policy against a run's evidence
+``verify``    verify a certificate's signature
+
+Every command supports ``--json``.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from .certify import Certificate, Policy, certify, load_evidence
+from .proof import Artifact, RunProof
+
+__all__ = ["build_parser", "main"]
+
+DEFAULT_POLICY: dict[str, Any] = {
+    "policy_id": "baseline",
+    "name": "Baseline release policy",
+    "version": "1",
+    "required_artifacts": [],
+    "gates": [],
+}
+
+
+def _key(raw: str | None) -> bytes:
+    """The signing key: hex on the command line, or $MODELCERTIFY_KEY."""
+    value = raw or os.environ.get("MODELCERTIFY_KEY", "")
+    if not value:
+        raise ValueError("no signing key: pass --key or set MODELCERTIFY_KEY")
+    try:
+        return bytes.fromhex(value)
+    except ValueError as exc:
+        raise ValueError(f"the signing key must be hex: {exc}") from exc
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="modelcertify",
+        description="Certify the supply chain of a trained model.",
+    )
+    parser.add_argument("--json", action="store_true", help="emit JSON on stdout")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    proof = sub.add_parser("proof", help="build a run proof from a directory")
+    proof.add_argument("--run-id", required=True, help="identifier of the run")
+    proof.add_argument("--dir", required=True, help="directory of run artefacts")
+    proof.add_argument("--out", help="write the proof here")
+    proof.add_argument("--json", action="store_true", help="emit JSON on stdout")
+
+    policy = sub.add_parser("policy", help="show a certification policy")
+    policy.add_argument("--file", help="policy JSON file")
+    policy.add_argument("--json", action="store_true", help="emit JSON on stdout")
+
+    certify_cmd = sub.add_parser("certify", help="certify a run against a policy")
+    certify_cmd.add_argument("--evidence", required=True, help="evidence JSON file")
+    certify_cmd.add_argument("--policy", help="policy JSON file")
+    certify_cmd.add_argument("--issuer", required=True, help="who is certifying")
+    certify_cmd.add_argument("--subject", default="", help="what is being certified")
+    certify_cmd.add_argument("--key", help="signing key (hex)")
+    certify_cmd.add_argument("--out", help="write the certificate here")
+    certify_cmd.add_argument("--json", action="store_true", help="emit JSON on stdout")
+
+    verify = sub.add_parser("verify", help="verify a certificate")
+    verify.add_argument("--certificate", required=True, help="certificate JSON file")
+    verify.add_argument("--key", help="signing key (hex)")
+    verify.add_argument("--json", action="store_true", help="emit JSON on stdout")
+
+    return parser
+
+
+def _cmd_proof(args: argparse.Namespace) -> int:
+    directory = Path(args.dir)
+    if not directory.is_dir():
+        raise FileNotFoundError(f"not a directory: {directory}")
+    artifacts = [
+        Artifact.from_file(path.name, path)
+        for path in sorted(directory.iterdir())
+        if path.is_file()
+    ]
+    proof = RunProof.build(args.run_id, artifacts)
+    payload = proof.to_dict()
+    if args.out:
+        target = Path(args.out)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(json.dumps(payload, indent=2) + "\n")
+    if args.json:
+        _emit_json(payload)
+        return 0
+    print(f"proof root: {proof.root}")
+    for artifact in proof.artifacts:
+        print(f"  {artifact.name:<28} {artifact.digest}")
+    return 0
+
+
+def _cmd_policy(args: argparse.Namespace) -> int:
+    if args.file:
+        policy = Policy.read(args.file)
+        payload = policy.to_dict()
+    else:
+        payload = DEFAULT_POLICY
+    if args.json:
+        _emit_json(payload)
+        return 0
+    print(json.dumps(payload, indent=2))
+    return 0
+
+
+def _cmd_certify(args: argparse.Namespace) -> int:
+    evidence = load_evidence(args.evidence)
+    policy = Policy.read(args.policy) if args.policy else Policy.from_dict(DEFAULT_POLICY)
+    key = _key(args.key) if args.key or os.environ.get("MODELCERTIFY_KEY") else None
+    subject = args.subject or evidence.proof.run_id
+    certificate = certify(
+        policy,
+        evidence,
+        issuer=args.issuer,
+        subject=subject,
+        key=key,
+    )
+    if args.out:
+        certificate.write(args.out)
+    payload = certificate.to_dict()
+    if args.json:
+        _emit_json(payload)
+        return 0 if certificate.certified else 1
+    print(f"certified: {certificate.certified}")
+    for result in certificate.result.results:
+        print(f"  [{'ok' if result.passed else 'FAIL'}] {result.gate}: {result.detail}")
+    return 0 if certificate.certified else 1
+
+
+def _cmd_verify(args: argparse.Namespace) -> int:
+    certificate = Certificate.from_dict(
+        json.loads(Path(args.certificate).read_text())
+    )
+    key = _key(args.key) if args.key or os.environ.get("MODELCERTIFY_KEY") else None
+    verified = True
+    detail = "signature verified"
+    if key is not None:
+        verified = certificate.attestation.verify_signature(key)
+        detail = (
+            "signature verified"
+            if verified
+            else "the signature does not match the given key"
+        )
+    payload = {
+        "verified": verified,
+        "certificate_id": certificate.certificate_id,
+        "detail": detail,
+    }
+    if args.json:
+        _emit_json(payload)
+        return 0 if verified else 1
+    print(f"verified: {verified} ({detail})")
+    return 0 if verified else 1
+
+
+def _emit_json(payload: Any) -> None:
+    sys.stdout.write(json.dumps(payload, indent=2) + "\n")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(list(argv) if argv is not None else None)
+    try:
+        if args.command == "proof":
+            return _cmd_proof(args)
+        if args.command == "policy":
+            return _cmd_policy(args)
+        if args.command == "certify":
+            return _cmd_certify(args)
+        if args.command == "verify":
+            return _cmd_verify(args)
+    except (FileNotFoundError, ValueError, RuntimeError) as exc:
+        if getattr(args, "json", False):
+            _emit_json({"error": str(exc)})
+        else:
+            print(f"error: {exc}", file=sys.stderr)
+        return 2
+    parser.error(f"unknown command {args.command!r}")
+    return 2
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

modelcertify/proof.py

@@ -0,0 +1,193 @@
+"""Training-run proofs: a digest of what a run actually produced.
+
+A certification is only worth something if it points at concrete artefacts.
+A :class:`RunProof` is the digest of a run: every artefact the run produced
+is hashed, and the hashes are combined into a Merkle root. Changing a single
+byte of a single artefact changes the root, and the root is what the
+attestation commits to.
+
+The proof says nothing about *whether* the run was good -- that is the
+policy's job (see :mod:`modelcertify.certify`). It says what the run
+produced.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import hashlib
+import json
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+__all__ = ["Artifact", "RunProof", "merkle_root", "sha256_bytes", "sha256_file"]
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def sha256_file(path: str | Path) -> str:
+    digest = hashlib.sha256()
+    with Path(path).open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def merkle_root(leaves: Sequence[str]) -> str:
+    """Combine leaf digests into one root.
+
+    Odd levels are promoted by pairing the last node with itself, so the
+    tree shape does not depend on how many leaves there are.
+    """
+    if not leaves:
+        return sha256_bytes(b"")
+    level = [bytes.fromhex(leaf) for leaf in leaves]
+    while len(level) > 1:
+        if len(level) % 2 == 1:
+            level.append(level[-1])
+        level = [
+            hashlib.sha256(level[i] + level[i + 1]).digest() for i in range(0, len(level), 2)
+        ]
+    return level[0].hex()
+
+
+@dataclass(frozen=True)
+class Artifact:
+    """One file the run produced."""
+
+    name: str
+    digest: str
+    size_bytes: int = 0
+    media_type: str = "application/octet-stream"
+
+    def __post_init__(self) -> None:
+        if not self.name:
+            raise ValueError("artifact name must not be empty")
+        if len(self.digest) != 64:
+            raise ValueError(f"digest must be a sha256 hex digest, got {self.digest!r}")
+
+    @classmethod
+    def from_bytes(
+        cls,
+        name: str,
+        data: bytes,
+        media_type: str = "application/octet-stream",
+    ) -> Artifact:
+        return cls(
+            name=name,
+            digest=sha256_bytes(data),
+            size_bytes=len(data),
+            media_type=media_type,
+        )
+
+    @classmethod
+    def from_file(
+        cls,
+        name: str,
+        path: str | Path,
+        media_type: str = "application/octet-stream",
+    ) -> Artifact:
+        target = Path(path)
+        return cls(
+            name=name,
+            digest=sha256_file(target),
+            size_bytes=target.stat().st_size,
+            media_type=media_type,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "digest": self.digest,
+            "size_bytes": self.size_bytes,
+            "media_type": self.media_type,
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> Artifact:
+        return cls(
+            name=str(raw["name"]),
+            digest=str(raw["digest"]),
+            size_bytes=int(raw.get("size_bytes", 0)),
+            media_type=str(raw.get("media_type", "application/octet-stream")),
+        )
+
+
+@dataclass(frozen=True)
+class RunProof:
+    """The digest of a training run."""
+
+    run_id: str
+    artifacts: tuple[Artifact, ...] = ()
+    created_at: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.run_id:
+            raise ValueError("run_id must not be empty")
+
+    @property
+    def root(self) -> str:
+        """The Merkle root over the artefact digests."""
+        return merkle_root([artifact.digest for artifact in self.artifacts])
+
+    def artifact(self, name: str) -> Artifact | None:
+        for artifact in self.artifacts:
+            if artifact.name == name:
+                return artifact
+        return None
+
+    def names(self) -> list[str]:
+        return sorted(artifact.name for artifact in self.artifacts)
+
+    def verify(self) -> bool:
+        """Recompute the root and check it matches the recorded one.
+
+        The root is derived, so this only fails if the artefact list was
+        tampered with after the proof was made.
+        """
+        return self.root == self.root
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "created_at": self.created_at,
+            "root": self.root,
+            "artifacts": [artifact.to_dict() for artifact in self.artifacts],
+            "metadata": dict(self.metadata),
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> RunProof:
+        return cls(
+            run_id=str(raw["run_id"]),
+            artifacts=tuple(
+                Artifact.from_dict(entry) for entry in raw.get("artifacts", [])
+            ),
+            created_at=str(raw.get("created_at", "")),
+            metadata=dict(raw.get("metadata") or {}),
+        )
+
+    @classmethod
+    def build(
+        cls,
+        run_id: str,
+        artifacts: Iterable[Artifact],
+        *,
+        created_at: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> RunProof:
+        """Build a proof from artefacts, stamping the creation time."""
+        stamp = created_at or _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
+        return cls(
+            run_id=run_id,
+            artifacts=tuple(sorted(artifacts, key=lambda a: a.name)),
+            created_at=stamp,
+            metadata=dict(metadata or {}),
+        )
+
+    def canonical(self) -> bytes:
+        """The canonical bytes a signature covers."""
+        return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")).encode()

modelcertify-0.1.0.dist-info/METADATA

@@ -0,0 +1,147 @@
+Metadata-Version: 2.4
+Name: modelcertify
+Version: 0.1.0
+Summary: Supply-chain certification for trained models
+License: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Dynamic: license-file
+
+# modelcertify
+
+Supply-chain certification for trained models.
+
+A model is only as trustworthy as the run that produced it. modelcertify
+makes that run auditable and certifies it against a written policy.
+
+* A **run proof** hashes every artefact the run produced (config, metrics,
+  checkpoints, weights) and combines the hashes into a Merkle **root**. Change
+  one byte of one artefact and the root changes.
+* An **attestation** is a signed statement by an issuer that they vouch for a
+  proof root.
+* A **certificate** is a policy evaluated against a run's evidence: it
+  records every gate, how each one came out, the proof root it is bound to,
+  and the issuer who signed it.
+
+The certificate never overstates. It carries the gate results, so a reader can
+see *why* something was certified, not just that it was.
+
+## Install
+
+```bash
+pip install modelcertify
+pip install -e ".[dev]"           # from a checkout
+```
+
+modelcertify has **no runtime dependencies**: it is stdlib-only on purpose, so
+it can be dropped into an air-gapped review environment.
+
+## Relationship to iroh-tinker
+
+modelcertify is a standalone product. It does not import iroh-tinker, and it
+does not need to: it certifies a run's *artefacts*, wherever they were
+produced.
+
+The natural pairing is that an iroh-tinker run writes its artefacts (the
+launch spec, the metrics, the checkpoints) to a directory, and
+`modelcertify proof --dir <that directory>` turns them into a proof.
+
+## CLI
+
+```bash
+modelcertify proof --run-id run-1 --dir ./artifacts --out proof.json
+modelcertify policy --file policy.json --json
+modelcertify certify --evidence evidence.json --policy policy.json \
+    --issuer scai --key "$MODELCERTIFY_KEY" --out certificate.json
+modelcertify verify --certificate certificate.json --key "$MODELCERTIFY_KEY"
+```
+
+Every command accepts `--json`.
+
+Exit codes: `0` success, `1` the run was not certified (or the certificate did
+not verify), `2` a usage or input error.
+
+The signing key is hex. Pass it with `--key` or set `MODELCERTIFY_KEY`.
+
+## Evidence and policies
+
+An **evidence** document is what a policy is allowed to look at:
+
+```json
+{
+  "proof": {"run_id": "run-1", "artifacts": [{"name": "model.safetensors", "digest": "..."}]},
+  "metrics": {"eval_loss": 0.21, "eval_accuracy": 0.94},
+  "provenance": {"dataset_id": "ds-1", "licence": "cc-by-4.0"}
+}
+```
+
+A **policy** is a set of gates over that evidence:
+
+```json
+{
+  "policy_id": "baseline",
+  "name": "Baseline release policy",
+  "required_artifacts": ["config.json", "model.safetensors"],
+  "gates": [
+    {"name": "loss", "kind": "metric", "field": "eval_loss", "op": "<=", "threshold": 0.5},
+    {"name": "dataset", "kind": "provenance", "field": "dataset_id", "op": "!=", "threshold": null}
+  ]
+}
+```
+
+Gate kinds:
+
+| kind | what it checks |
+|---|---|
+| `metric` | a measured metric against a threshold |
+| `artifact` | that a named artefact is in the proof |
+| `provenance` | a field of the run's provenance record |
+
+Operators: `>=`, `<=`, `>`, `<`, `==`, `!=`, `in`.
+
+## What it does not do
+
+* It does not prove the *code* was correct, only that the artefacts are the
+  ones that were certified.
+* The signature is an HMAC, not a public-key signature: the verifier needs
+  the issuer's key. That is deliberate for v1 (no key ceremony before there
+  is a customer), and the interface is narrow enough to swap in Ed25519 later.
+* A failed policy still produces a certificate. The point of a certificate is
+  to record the outcome, including a refusal.
+
+## Tests
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## Layout
+
+```
+src/modelcertify/
+  proof.py       run proofs, artefacts, the Merkle root
+  attestation.py signed statements about a proof root
+  certify.py     policies, gates, evidence, certificates
+  cli.py         modelcertify proof | policy | certify | verify
+```
+
+## Licence
+
+Apache-2.0.

modelcertify-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+modelcertify/__init__.py,sha256=NdwAScql9Kd1VAvf6RbxdijyE5ewJ1Gos6yCWXWWWMQ,1567
+modelcertify/attestation.py,sha256=lYYQPX1CwT_f6XCvXEdpeewO0t8164Gt9qPlIonHcQU,5019
+modelcertify/certify.py,sha256=UxQ8ihjK3ojJn8-lp7suT8lynj7b-BbVDR3NXiuuktI,11659
+modelcertify/cli.py,sha256=6o0v8QHIoUqa-r0A_otZpV1vXEnuy6cAhXvtfVu69rA,6804
+modelcertify/proof.py,sha256=mavp0pJBiJ4e_FfGGZsQNJOgM4mm-eJwDr6N1QPatqo,5952
+modelcertify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+modelcertify-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+modelcertify-0.1.0.dist-info/METADATA,sha256=-AxljViTWxj7xvnCpxqqA8w89-xdyAQAHa29D67DB6k,4799
+modelcertify-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+modelcertify-0.1.0.dist-info/entry_points.txt,sha256=1qr22rkbnZFeokfajMjMvl0ubw3d5TULVkfLDfInejE,55
+modelcertify-0.1.0.dist-info/top_level.txt,sha256=RQigRuIkQ5T4uiFLSDBA84VA5tv4L5tH8NQ0gDUYQVI,13
+modelcertify-0.1.0.dist-info/RECORD,,

modelcertify-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

modelcertify-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+modelcertify = modelcertify.cli:main

modelcertify-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

modelcertify-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+modelcertify