agenteval-toolkit 0.1.0__py3-none-any.whl

agenteval/certification/models.py

@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 The AgentEval Authors
+"""Certification: AI system registry, issuance policy, gate decisions and certificate data structures."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class AISystem(BaseModel):
+    """An AI system to certify: binds a scanned asset to an evaluated agent."""
+
+    id: str
+    name: str
+    name_zh: str = ""
+    organization: str
+    industry: str = ""
+    description: str = ""
+    description_zh: str = ""
+    asset_id: str                 # scanned asset ID, e.g. deployment/ml-prod/llama-chat
+    model: str = ""
+    agent: str = ""               # config file name in the agents directory (used by the API)
+    agent_name: str               # agent name in evaluation records
+    exam: str                     # exam paper ID
+    rule_packs: list[str]         # rule pack IDs that must be covered
+    contact: str = ""
+
+
+class CertPolicy(BaseModel):
+    min_security_score: float = 90.0
+    min_quality_score: float = 90.0
+    block_on_severities: list[str] = Field(default_factory=lambda: ["critical", "high"])
+    require_no_pending_attestations: bool = True
+    require_zero_critical_eval_failures: bool = True
+    require_no_eval_errors: bool = True
+    require_llm_judge: bool = True
+    allow_demo_certificates: bool = True
+    max_evidence_age_days: int = 30
+    validity_months: int = 6
+    auto_revoke: bool = True
+    issuer_name: str = "AgentEval Assurance Service"
+    issuer_name_zh: str = "AgentEval 认证服务"
+
+
+class Gate(BaseModel):
+    code: str
+    ok: bool
+    kind: Literal["blocking", "freshness", "demo"] = "blocking"
+    params: dict[str, Any] = Field(default_factory=dict)
+
+
+class PackResult(BaseModel):
+    id: str
+    name: str
+    name_zh: str = ""
+    tier: str
+    jurisdiction: str
+    passed: int = 0
+    failed: int = 0
+    manual: int = 0
+    not_applicable: int = 0
+    score: float | None = None
+    status: Literal["passed", "failed", "not_applicable", "not_assessed"] = "not_assessed"
+
+
+class SecurityAssessment(BaseModel):
+    scan_id: str | None = None
+    scanned_at: datetime | None = None
+    rules_version: str | None = None
+    score: float | None = None
+    passed: int = 0
+    failed: int = 0
+    manual: int = 0
+    blocking_findings: list[str] = Field(default_factory=list)   # "RULE@asset"
+    pending: list[str] = Field(default_factory=list)
+    packs: list[PackResult] = Field(default_factory=list)
+
+
+class QualityAssessment(BaseModel):
+    run_id: str | None = None
+    evaluated_at: datetime | None = None
+    exam: str | None = None
+    exam_version: str | None = None
+    agent_name: str | None = None
+    judge: str | None = None
+    score: float | None = None
+    grade: str | None = None
+    critical_failures: int = 0
+    errors: int = 0
+    cases: int = 0
+    cases_passed: int = 0
+    dimensions: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class Decision(BaseModel):
+    system_id: str
+    evaluated_at: datetime
+    eligible: bool
+    demo: bool
+    gates: list[Gate]
+    security: SecurityAssessment
+    quality: QualityAssessment
+
+    @property
+    def revocation_failures(self) -> list[Gate]:
+        """Failed gates that trigger revocation: substantive gates only; stale evidence does not revoke (certificate validity handles that)."""
+        return [g for g in self.gates if not g.ok and g.kind == "blocking"]
+
+
+CertStatus = Literal["active", "revoked", "expired", "superseded"]

agenteval/certification/render.py

@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 The AgentEval Authors
+"""Rendering of certificates and public verification pages (zh/en), plus optional PDF export."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from importlib import resources
+from pathlib import Path
+
+from jinja2 import Environment, select_autoescape
+from markupsafe import Markup
+
+UI = {
+    "zh": {
+        "html_lang": "zh-CN", "title": "AI 系统安全与质量认证证书", "cert_no": "证书编号",
+        "certifies": "兹证明", "statement": "已通过 AgentEval 工具的安全合规扫描与标准化质量评测，满足本认证全部颁证条件。",
+        "security": "安全", "security_sub": "国家、区域及行业安全规范", "quality": "质量", "quality_sub": "标准化评测指标",
+        "score": "得分", "threshold": "阈值", "tier": "层级", "pack": "规则包", "checks": "通过 / 适用", "status": "结论",
+        "national": "国家", "regional": "区域", "industry": "行业", "unknown": "其他",
+        "st_passed": "通过", "st_failed": "未通过", "st_not_applicable": "不适用", "st_not_assessed": "未评估",
+        "exam": "试卷", "grade": "等级", "judge": "裁判", "cases": "样本通过", "dimension": "维度",
+        "issued": "颁发日期", "expires": "有效期至", "evidence": "证据", "scan": "扫描", "rules": "规则集", "run": "评测",
+        "subject": "被认证系统", "asset": "部署资产", "model": "模型", "agent": "评测对象",
+        "verify": "扫码或访问以下地址验证证书状态", "verify_offline": "离线验证：使用签发方公钥运行",  "signature": "Ed25519 数字签名 · 公钥指纹",
+        "demo": "演示证书 · 不作正式认证用途", "demo_reason": "评测使用离线启发式裁判，正式颁证需使用 LLM 裁判。",
+        "state_revoked": "已吊销", "state_expired": "已过期", "state_superseded": "已被新证书替代",
+        "print": "打印 / 另存为 PDF",
+        # verification page
+        "v_title": "证书验证", "v_valid": "有效", "v_invalid_sig": "签名无效", "v_not_found": "未找到该证书",
+        "v_sig_ok": "签名校验通过", "v_sig_bad": "签名校验失败", "v_view": "查看证书", "v_reason": "原因",
+        "v_status": "状态", "v_org": "所属组织",
+    },
+    "en": {
+        "html_lang": "en", "title": "AI System Assurance Certificate", "cert_no": "Certificate No.",
+        "certifies": "This certifies that",
+        "statement": "has passed the AgentEval security compliance scan and standardized quality evaluation, and meets every certification criterion.",
+        "security": "Security", "security_sub": "National, regional and industry security requirements",
+        "quality": "Quality", "quality_sub": "Standardized evaluation metrics",
+        "score": "Score", "threshold": "Threshold", "tier": "Tier", "pack": "Rule pack", "checks": "Passed / applicable",
+        "status": "Result", "national": "National", "regional": "Regional", "industry": "Industry", "unknown": "Other",
+        "st_passed": "Passed", "st_failed": "Failed", "st_not_applicable": "Not applicable", "st_not_assessed": "Not assessed",
+        "exam": "Exam", "grade": "Grade", "judge": "Judge", "cases": "Cases passed", "dimension": "Dimension",
+        "issued": "Issued", "expires": "Valid until", "evidence": "Evidence", "scan": "Scan", "rules": "Rule set", "run": "Eval run",
+        "subject": "Certified system", "asset": "Deployment", "model": "Model", "agent": "Agent evaluated",
+        "verify": "Scan or visit to verify the certificate status", "verify_offline": "Verify offline with the issuer public key:",  "signature": "Ed25519 signature · key fingerprint",
+        "demo": "DEMO certificate · not valid for certification",
+        "demo_reason": "The evaluation used the offline heuristic judge; a formal certificate requires an LLM judge.",
+        "state_revoked": "Revoked", "state_expired": "Expired", "state_superseded": "Superseded by a newer certificate",
+        "print": "Print / Save as PDF",
+        "v_title": "Certificate verification", "v_valid": "Valid", "v_invalid_sig": "Invalid signature",
+        "v_not_found": "Certificate not found", "v_sig_ok": "Signature verified", "v_sig_bad": "Signature check failed",
+        "v_view": "View certificate", "v_reason": "Reason", "v_status": "Status", "v_org": "Organization",
+    },
+}
+
+
+def qr_svg(url: str) -> Markup:
+    import qrcode
+    import qrcode.image.svg
+
+    img = qrcode.make(url, image_factory=qrcode.image.svg.SvgPathImage, box_size=8, border=1)
+    svg = img.to_string(encoding="unicode")
+    return Markup(svg)
+
+
+def _env() -> Environment:
+    env = Environment(autoescape=select_autoescape(default=True, default_for_string=True))
+    env.filters["date"] = lambda s: datetime.fromisoformat(s).strftime("%Y-%m-%d") if s else "—"
+    return env
+
+
+def _tpl(name: str) -> str:
+    return resources.files("agenteval").joinpath(f"reports/templates/{name}").read_text(encoding="utf-8")
+
+
+def render_certificate(envelope: dict, lang: str = "en", state: str = "active", printable_button: bool = True) -> str:
+    if lang not in UI:
+        raise ValueError(f"Unsupported language: {lang}")
+    c = envelope["certificate"]
+    return _env().from_string(_tpl("certificate.html.j2")).render(
+        c=c, sig=envelope["signature"], t=UI[lang], lang=lang, state=state,
+        qr=qr_svg(c["verify_url"]) if c.get("verify_url") else None, printable_button=printable_button)
+
+
+def to_pdf(html: str, out: Path) -> Path:
+    """Print an A4 landscape PDF with Chromium (requires playwright and its browser)."""
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError as e:  # pragma: no cover
+        raise RuntimeError("PDF export requires playwright: pip install playwright && playwright install chromium") from e
+    out = Path(out)
+    with sync_playwright() as p:
+        b = p.chromium.launch()
+        page = b.new_page()
+        page.set_content(html, wait_until="load")
+        page.emulate_media(media="print")
+        page.pdf(path=str(out), format="A4", landscape=True, print_background=True, prefer_css_page_size=True)
+        b.close()
+    return out

agenteval/certification/signing.py

@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 The AgentEval Authors
+"""Ed25519 signing and verification.
+
+In production, keep the private key in a KMS/HSM and point AGENTEVAL_SIGNING_KEY at the mounted PEM file.
+In development, if no key exists, a local key is generated under ./.agenteval/ (mode 0600) for demo use only.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
+
+ALG = "Ed25519"
+
+
+def canonical(payload: dict[str, Any]) -> bytes:
+    return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+
+
+def default_key_path() -> Path:
+    return Path(os.environ.get("AGENTEVAL_SIGNING_KEY", Path.cwd() / ".agenteval" / "signing-key.pem"))
+
+
+def generate_key(path: Path) -> Ed25519PrivateKey:
+    key = Ed25519PrivateKey.generate()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    pem = key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
+                            serialization.NoEncryption())
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "wb") as fh:
+        fh.write(pem)
+    return key
+
+
+def load_or_create_key(path: Path | None = None) -> Ed25519PrivateKey:
+    path = Path(path or default_key_path())
+    if not path.exists():
+        return generate_key(path)
+    key = serialization.load_pem_private_key(path.read_bytes(), password=None)
+    if not isinstance(key, Ed25519PrivateKey):
+        raise ValueError(f"{path} is not an Ed25519 private key")
+    return key
+
+
+def public_pem(key: Ed25519PrivateKey | Ed25519PublicKey) -> str:
+    pub = key.public_key() if isinstance(key, Ed25519PrivateKey) else key
+    return pub.public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo).decode()
+
+
+def key_id(key: Ed25519PrivateKey | Ed25519PublicKey) -> str:
+    pub = key.public_key() if isinstance(key, Ed25519PrivateKey) else key
+    raw = pub.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw)
+    return hashlib.sha256(raw).hexdigest()[:16]
+
+
+def load_public_key(pem: str | bytes) -> Ed25519PublicKey:
+    key = serialization.load_pem_public_key(pem.encode() if isinstance(pem, str) else pem)
+    if not isinstance(key, Ed25519PublicKey):
+        raise ValueError("Not an Ed25519 public key")
+    return key
+
+
+def sign(payload: dict[str, Any], key: Ed25519PrivateKey) -> dict[str, Any]:
+    sig = key.sign(canonical(payload))
+    return {"certificate": payload,
+            "signature": {"alg": ALG, "key_id": key_id(key), "value": base64.b64encode(sig).decode()}}
+
+
+def verify(envelope: dict[str, Any], public_key: Ed25519PublicKey) -> bool:
+    try:
+        s = envelope["signature"]
+        if s.get("alg") != ALG or s.get("key_id") != key_id(public_key):
+            return False
+        public_key.verify(base64.b64decode(s["value"]), canonical(envelope["certificate"]))
+        return True
+    except (KeyError, TypeError, ValueError, InvalidSignature):
+        return False