agenteval-toolkit 0.1.0__py3-none-any.whl

agenteval/_starter/rules/us-federal/i18n/zh.yaml

@@ -0,0 +1,107 @@
+# yaml-language-server: $schema=../../../agenteval/schemas/translations.schema.json
+# Simplified Chinese translations for us-federal rules.
+# Keys must match rule IDs. `sources` translates each source's `reference` in order.
+FED-INV-001:
+  title: AI 系统须登记责任人与模型信息
+  description: 每个推理服务须标注责任团队与所服务的模型，以支持 AI 系统清单管理。
+  remediation: 为工作负载添加标签 agenteval.io/owner=<team> 与注解 agenteval.io/model=<model-name>。
+  sources:
+    - GOVERN 1.6（AI 系统清单机制）
+    - AI 使用案例清单（适用于联邦机构）
+FED-PRV-001:
+  title: 模型须有可追溯的来源并完成完整性校验
+  description: 记录模型权重来源（内部训练 / 第三方仓库）并校验哈希或签名。
+  remediation: 添加注解 agenteval.io/model-source=<来源>；在拉取权重时校验 sha256 或签名， 通过后设置 
+    agenteval.io/provenance-verified="true"（建议由 CI/准入控制器自动写入）。
+  sources:
+    - GOVERN 6.1（第三方 AI 风险策略）
+    - 模型与训练数据的完整性保护
+FED-TRN-001:
+  title: 模型须附带模型卡或等效文档
+  description: 文档应说明模型用途、能力边界、已知局限与评估结果。
+  remediation: 添加注解 agenteval.io/model-card=<文档链接>，内容覆盖预期用途、局限性与评估方法。
+  sources:
+    - MAP 功能（系统能力与局限的记录）
+FED-MON-001:
+  title: 推理服务须启用请求日志与运行监控
+  description: 部署后需持续监测，以发现滥用、性能退化与安全事件。
+  remediation: 启用网关/服务的请求日志（注意脱敏），暴露 Prometheus 指标并添加注解 
+    agenteval.io/request-logging="true"、prometheus.io/scrape="true"。
+  sources:
+    - MANAGE 4.1（部署后监测计划）
+FED-SAF-001:
+  title: 生成式模型须启用内容安全防护
+  description: 生成式模型的输入/输出应经过安全过滤或护栏（如 Llama Guard、NeMo Guardrails）。
+  remediation: 部署护栏服务或 sidecar，并添加注解 agenteval.io/content-safety=<方案名>。
+  sources:
+    - Generative AI Profile（有害内容、信息安全风险）
+FED-GOV-A01:
+  title: 组织须具备 AI 治理政策并指定责任人
+  description: 书面 AI 风险管理政策，明确角色职责与审批流程，每年复审。
+  remediation: 制定并发布 AI 治理政策，指定负责人，通过 attestations 文件提交链接与审批记录。
+  sources:
+    - GOVERN 1 / GOVERN 2（政策与问责结构）
+FED-IR-001:
+  title: 须具备 AI 安全事件响应预案
+  description: 覆盖模型滥用、数据泄露、有害输出等场景的响应与上报流程，每年演练或复审。
+  remediation: 编写 AI 事件响应预案并纳入现有安全事件流程，提交文档链接。
+  sources:
+    - MANAGE 4（风险处置与事件沟通）
+FED-GOV-A02:
+  title: 高影响 AI 须完成影响评估与部署前测试
+  description: 标记为高影响（high-impact）的 AI 服务，上线前须完成 AI 影响评估与实测验证，每年更新。
+  remediation: 完成 AI 影响评估与部署前测试报告，经审批后提交至 attestations 文件。
+  sources:
+    - 高影响 AI 的最低风险管理实践（适用于联邦机构，私营企业可参照）
+    - MAP 5.1 / MEASURE 功能
+FED-GOV-A03:
+  title: 高影响 AI 须具备人工监督与申诉机制
+  description: 对高影响决策提供人工复核、干预与申诉渠道。
+  remediation: 制定人工监督与申诉流程文档，并在服务中实现人工复核入口。
+  sources:
+    - 高影响 AI 的人工监督、干预与问责要求
+FED-SEC-001:
+  title: 推理端点必须启用身份认证
+  description: 模型推理 API 不得匿名访问，须通过 API Key、OIDC 或网关认证保护。
+  remediation: 为推理服务配置认证：vLLM 使用 --api-key 并从 Secret 注入；或在前方部署 oauth2-proxy / Envoy ext_authz / API 网关，并添加注解 
+    agenteval.io/auth=<method>。
+  sources:
+    - MEASURE 2.7（AI 系统安全性与韧性评估）
+    - AC-3 Access Enforcement
+FED-SEC-002:
+  title: 对外暴露的推理端点必须启用 TLS
+  description: 通过 LoadBalancer、NodePort 或 Ingress 对外暴露的推理服务，传输必须加密。
+  remediation: 在 Ingress 上配置 tls 段并使用受信任证书（如 cert-manager），或在服务网格中启用 mTLS。
+  sources:
+    - SC-8 Transmission Confidentiality and Integrity
+FED-SEC-003:
+  title: 推理服务须受 NetworkPolicy 限制
+  description: 推理工作负载所在命名空间应存在选中该 Pod 的 NetworkPolicy，限制横向访问。
+  remediation: 为推理 Pod 创建默认拒绝的 NetworkPolicy，仅放行网关/调用方命名空间的入站流量。
+  sources:
+    - 4.4 Orchestrator Risks（网络隔离）
+FED-SEC-004:
+  title: 容器镜像须以 digest 锁定且不使用 latest 标签
+  description: 推理镜像应使用 @sha256 摘要引用，保证部署的可追溯性与完整性。
+  remediation: 将镜像引用改为 registry/repo@sha256:<digest>，并在 CI 中签名（如 cosign）后再部署。
+  sources:
+    - SSDF for Generative AI and Dual-Use Foundation Models（软件完整性）
+FED-SEC-005:
+  title: 推理容器不得以 root 或特权模式运行
+  description: 容器须设置 runAsNonRoot，禁止 privileged 与权限提升。
+  remediation: '在 securityContext 中设置 runAsNonRoot: true、privileged: false、allowPrivilegeEscalation: false。 GPU
+    访问应通过 device plugin 实现，而非特权容器。'
+  sources:
+    - 4.1 Image / 4.5 Host OS Risks（最小权限）
+FED-SEC-006:
+  title: 不得在环境变量中明文存放密钥
+  description: 模型 API Key、HF Token、云凭证等不得以明文 value 写入 Pod 环境变量。
+  remediation: 改用 valueFrom.secretKeyRef 或外部密钥管理（Vault / CSI Secrets Store），并轮换已暴露的凭证。
+  sources:
+    - IA-5 Authenticator Management
+FED-SEC-007:
+  title: 推理容器须设置资源限制
+  description: 设置 CPU/内存/GPU limits，降低资源耗尽型攻击与失控负载的影响。
+  remediation: 为每个容器配置 resources.limits，并在命名空间上设置 ResourceQuota / LimitRange。
+  sources:
+    - MEASURE 2.7（AI 系统安全性与韧性评估）

agenteval/_starter/rules/us-federal/pack.yaml

@@ -0,0 +1,8 @@
+# yaml-language-server: $schema=../../agenteval/schemas/pack.schema.json
+id: us-federal
+name: US federal guidance
+name_zh: 美国联邦指引
+tier: national
+jurisdiction: US
+description: NIST AI RMF, NIST AI 600-1, NIST SP 800-53/190/218A, OMB M-25-21
+description_zh: NIST AI RMF、NIST AI 600-1、NIST SP 800-53/190/218A、OMB M-25-21

agenteval/_starter/rules/us-federal/security.yaml

@@ -0,0 +1,164 @@
+# yaml-language-server: $schema=../../agenteval/schemas/rule.schema.json
+# Baseline security controls for inference services (automated checks)
+# Note: verified=false in sources means the mapping has not yet been reviewed by legal/compliance.
+rules:
+  - id: FED-SEC-001
+    title: Inference endpoints must require authentication
+    description: >-
+      Model inference APIs must not allow anonymous access; protect them with an API key, OIDC, or gateway authentication.
+    severity: high
+    type: automated
+    category: access-control
+    applies_to:
+      asset_types: [inference_service]
+    check:
+      field: auth.enabled
+      op: eq
+      value: true
+    remediation: >-
+      Configure authentication for the inference service: for vLLM use --api-key injected from a Secret; or front
+      it with oauth2-proxy / Envoy ext_authz / an API gateway, and add the annotation agenteval.io/auth=<method>.
+    sources:
+      - framework: NIST AI RMF 1.0
+        reference: MEASURE 2.7 (AI system security and resilience evaluated)
+        url: https://www.nist.gov/itl/ai-risk-management-framework
+      - framework: NIST SP 800-53 Rev.5
+        reference: AC-3 Access Enforcement
+        url: https://doi.org/10.6028/NIST.SP.800-53r5
+
+  - id: FED-SEC-002
+    title: Externally exposed inference endpoints must use TLS
+    description: Inference services exposed via LoadBalancer, NodePort, or Ingress must encrypt traffic in 
+      transit.
+    severity: high
+    type: automated
+    category: data-protection
+    applies_to:
+      asset_types: [inference_service]
+      when:
+        field: exposure.public
+        op: eq
+        value: true
+    check:
+      field: tls.enabled
+      op: eq
+      value: true
+    remediation: >-
+      Configure a tls section on the Ingress with a trusted certificate (e.g., cert-manager), or enable mTLS in
+      the service mesh.
+    sources:
+      - framework: NIST SP 800-53 Rev.5
+        reference: SC-8 Transmission Confidentiality and Integrity
+        url: https://doi.org/10.6028/NIST.SP.800-53r5
+
+  - id: FED-SEC-003
+    title: Inference services must be restricted by a NetworkPolicy
+    description: >-
+      The namespace of an inference workload should have a NetworkPolicy selecting its Pods to limit lateral access.
+    severity: medium
+    type: automated
+    category: network
+    applies_to:
+      asset_types: [inference_service]
+    check:
+      field: network_policy.present
+      op: eq
+      value: true
+    remediation: >-
+      Create a default-deny NetworkPolicy for inference Pods, allowing ingress only from gateway/caller namespaces.
+    sources:
+      - framework: NIST SP 800-190
+        reference: 4.4 Orchestrator Risks (network segmentation)
+        url: https://doi.org/10.6028/NIST.SP.800-190
+
+  - id: FED-SEC-004
+    title: Container images must be pinned by digest and must not use the latest tag
+    description: >-
+      Inference images should be referenced by @sha256 digest to ensure deployment traceability and integrity.
+    severity: medium
+    type: automated
+    category: supply-chain
+    applies_to:
+      asset_types: [inference_service]
+    check:
+      all:
+        - field: image.all_digest_pinned
+          op: eq
+          value: true
+        - field: image.any_latest
+          op: eq
+          value: false
+    remediation: >-
+      Change image references to registry/repo@sha256:<digest>, and sign images in CI (e.g., cosign) before deployment.
+    sources:
+      - framework: NIST SP 800-218A
+        reference: SSDF for Generative AI and Dual-Use Foundation Models (software integrity)
+        url: https://doi.org/10.6028/NIST.SP.800-218A
+
+  - id: FED-SEC-005
+    title: Inference containers must not run as root or in privileged mode
+    description: Containers must set runAsNonRoot and disallow privileged mode and privilege escalation.
+    severity: high
+    type: automated
+    category: workload-hardening
+    applies_to:
+      asset_types: [inference_service]
+    check:
+      all:
+        - field: security_context.run_as_non_root
+          op: eq
+          value: true
+        - field: security_context.privileged
+          op: eq
+          value: false
+        - field: security_context.allow_privilege_escalation
+          op: eq
+          value: false
+    remediation: >-
+      Set runAsNonRoot: true, privileged: false, and allowPrivilegeEscalation: false in securityContext. Provide
+      GPU access through a device plugin rather than privileged containers.
+    sources:
+      - framework: NIST SP 800-190
+        reference: 4.1 Image / 4.5 Host OS Risks (least privilege)
+        url: https://doi.org/10.6028/NIST.SP.800-190
+
+  - id: FED-SEC-006
+    title: Secrets must not be stored in plaintext environment variables
+    description: >-
+      Model API keys, HF tokens, cloud credentials, etc. must not be written as plaintext values in Pod environment
+      variables.
+    severity: critical
+    type: automated
+    category: secrets
+    applies_to:
+      asset_types: [inference_service]
+    check:
+      field: secrets.plaintext_env
+      op: empty
+    remediation: >-
+      Use valueFrom.secretKeyRef or an external secrets manager (Vault / CSI Secrets Store), and rotate any exposed
+      credentials.
+    sources:
+      - framework: NIST SP 800-53 Rev.5
+        reference: IA-5 Authenticator Management
+        url: https://doi.org/10.6028/NIST.SP.800-53r5
+
+  - id: FED-SEC-007
+    title: Inference containers must set resource limits
+    description: Set CPU/memory/GPU limits to reduce the impact of resource-exhaustion attacks and runaway 
+      workloads.
+    severity: low
+    type: automated
+    category: resilience
+    applies_to:
+      asset_types: [inference_service]
+    check:
+      field: resources.limits_set
+      op: eq
+      value: true
+    remediation: >-
+      Configure resources.limits for every container, and set a ResourceQuota / LimitRange on the namespace.
+    sources:
+      - framework: NIST AI RMF 1.0
+        reference: MEASURE 2.7 (AI system security and resilience evaluated)
+        url: https://www.nist.gov/itl/ai-risk-management-framework

agenteval/certification/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 The AgentEval Authors
+"""AI system certification: automatically issue a digital certificate when both security (scan) and quality (evaluation) meet the policy gates."""
+
+from .assess import assess_quality, assess_security, decide
+from .issuer import (
+    build_certificate, certificate_state, gate_text, issue, load_policy, load_system_file, load_systems,
+)
+from .models import AISystem, CertPolicy, Decision
+from .signing import default_key_path, key_id, load_or_create_key, load_public_key, public_pem, sign, verify
+
+__all__ = [
+    "assess_quality", "assess_security", "decide", "build_certificate", "certificate_state", "gate_text",
+    "issue", "load_policy", "load_system_file", "load_systems", "AISystem", "CertPolicy", "Decision",
+    "default_key_path", "key_id", "load_or_create_key", "load_public_key", "public_pem", "sign", "verify",
+]

agenteval/certification/assess.py

@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 The AgentEval Authors
+"""Certification decision: aggregate scan and evaluation evidence within an AI system's scope and check each gate.
+
+The security score counts only findings on the system's own assets plus organization-level (cluster) assets that
+belong to the system's registered rule packs. Issues in other workloads of the same cluster neither hurt nor help it.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from agenteval.evaluation.models import EvalRun
+from agenteval.models import RulePack, ScanResult, Status
+
+from .models import AISystem, CertPolicy, Decision, Gate, PackResult, QualityAssessment, SecurityAssessment
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _aware(dt: datetime | None) -> datetime | None:
+    if dt is not None and dt.tzinfo is None:
+        return dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
+def _pct(p: int, f: int) -> float | None:
+    return round(100.0 * p / (p + f), 1) if (p + f) else None
+
+
+def in_scope(system: AISystem, finding) -> bool:
+    return (finding.asset_id == system.asset_id or finding.asset_type == "cluster") and finding.pack in system.rule_packs
+
+
+def assess_security(system: AISystem, scan: ScanResult | None, packs: dict[str, RulePack],
+                    policy: CertPolicy) -> SecurityAssessment:
+    sa = SecurityAssessment()
+    for pid in system.rule_packs:
+        p = packs.get(pid)
+        sa.packs.append(PackResult(id=pid, name=p.name if p else pid, name_zh=p.name_zh if p else "",
+                                   tier=p.tier if p else "unknown", jurisdiction=p.jurisdiction if p else ""))
+    if scan is None:
+        return sa
+    sa.scan_id, sa.scanned_at, sa.rules_version = scan.scan_id, _aware(scan.finished_at or scan.started_at), scan.rules_version
+    by_pack = {p.id: p for p in sa.packs}
+    seen_pack: set[str] = set()
+    for f in scan.findings:
+        if not in_scope(system, f):
+            continue
+        pr = by_pack[f.pack]
+        seen_pack.add(f.pack)
+        if f.status == Status.passed:
+            pr.passed += 1; sa.passed += 1
+        elif f.status in (Status.failed, Status.error):
+            pr.failed += 1; sa.failed += 1
+            if f.severity.value in policy.block_on_severities:
+                sa.blocking_findings.append(f"{f.rule_id}@{f.asset_id}")
+        elif f.status == Status.manual:
+            pr.manual += 1; sa.manual += 1
+            sa.pending.append(f"{f.rule_id}@{f.asset_id}")
+        else:
+            pr.not_applicable += 1
+    for pr in sa.packs:
+        pr.score = _pct(pr.passed, pr.failed)
+        if pr.id not in seen_pack:
+            pr.status = "not_assessed"
+        elif pr.passed + pr.failed + pr.manual == 0:
+            pr.status = "not_applicable"
+        elif pr.failed == 0 and pr.manual == 0:
+            pr.status = "passed"
+        else:
+            pr.status = "failed"
+    sa.score = _pct(sa.passed, sa.failed)
+    return sa
+
+
+def assess_quality(run: EvalRun | None) -> QualityAssessment:
+    qa = QualityAssessment()
+    if run is None:
+        return qa
+    qa.run_id, qa.evaluated_at = run.run_id, _aware(run.finished_at or run.started_at)
+    qa.exam, qa.exam_version, qa.agent_name, qa.judge = run.paper_id, run.paper_version, run.agent_name, run.judge
+    qa.score, qa.grade = run.overall_score, run.grade
+    qa.critical_failures, qa.errors = run.stats.critical_failures, run.stats.errors
+    qa.cases, qa.cases_passed = run.stats.cases, run.stats.cases_passed
+    qa.dimensions = [{"id": d.id, "name": d.name, "weight": d.weight, "score": d.score} for d in run.dimensions]
+    return qa
+
+
+def decide(system: AISystem, scan: ScanResult | None, run: EvalRun | None, packs: dict[str, RulePack],
+           policy: CertPolicy, now: datetime | None = None) -> Decision:
+    now = _aware(now) or _now()
+    sa = assess_security(system, scan, packs, policy)
+    qa = assess_quality(run)
+    g: list[Gate] = []
+    max_age = policy.max_evidence_age_days
+
+    # ---- Security (scan) ----
+    asset_seen = scan is not None and any(f.asset_id == system.asset_id for f in scan.findings)
+    g.append(Gate(code="sec_asset_scanned", ok=asset_seen, params={"asset": system.asset_id}))
+    missing = [p.id for p in sa.packs if p.status == "not_assessed"]
+    g.append(Gate(code="sec_packs_assessed", ok=asset_seen and not missing, params={"missing": missing}))
+    g.append(Gate(code="sec_score", ok=sa.score is not None and sa.score > policy.min_security_score,
+                  params={"score": sa.score, "threshold": policy.min_security_score}))
+    g.append(Gate(code="sec_no_blocking", ok=asset_seen and not sa.blocking_findings,
+                  params={"items": sa.blocking_findings, "severities": policy.block_on_severities}))
+    if policy.require_no_pending_attestations:
+        g.append(Gate(code="sec_no_pending", ok=asset_seen and not sa.pending, params={"items": sa.pending}))
+    scan_age = (now - sa.scanned_at).days if sa.scanned_at else None
+    g.append(Gate(code="sec_fresh", kind="freshness", ok=scan_age is not None and scan_age <= max_age,
+                  params={"age": scan_age, "max": max_age}))
+
+    # ---- Quality (evaluation) ----
+    subject_ok = run is not None and run.agent_name == system.agent_name and run.paper_id == system.exam
+    g.append(Gate(code="q_run", ok=subject_ok, params={"agent": system.agent_name, "exam": system.exam}))
+    g.append(Gate(code="q_score", ok=subject_ok and qa.score is not None and qa.score > policy.min_quality_score,
+                  params={"score": qa.score, "threshold": policy.min_quality_score}))
+    if policy.require_zero_critical_eval_failures:
+        g.append(Gate(code="q_no_critical", ok=subject_ok and qa.critical_failures == 0,
+                      params={"n": qa.critical_failures}))
+    if policy.require_no_eval_errors:
+        g.append(Gate(code="q_no_errors", ok=subject_ok and qa.errors == 0, params={"n": qa.errors}))
+    run_age = (now - qa.evaluated_at).days if qa.evaluated_at else None
+    g.append(Gate(code="q_fresh", kind="freshness", ok=run_age is not None and run_age <= max_age,
+                  params={"age": run_age, "max": max_age}))
+    demo = False
+    if policy.require_llm_judge:
+        llm = bool(qa.judge) and qa.judge != "heuristic"
+        demo = subject_ok and not llm
+        g.append(Gate(code="q_llm_judge", kind="demo" if policy.allow_demo_certificates else "blocking",
+                      ok=llm, params={"judge": qa.judge}))
+
+    eligible = all(x.ok for x in g if x.kind in ("blocking", "freshness"))
+    return Decision(system_id=system.id, evaluated_at=now, eligible=eligible, demo=eligible and demo,
+                    gates=g, security=sa, quality=qa)

agenteval/certification/issuer.py

@@ -0,0 +1,165 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 The AgentEval Authors
+"""Certificate generation, registry loading and gate description text."""
+
+from __future__ import annotations
+
+import secrets
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from agenteval.schema import check
+
+from agenteval.config import content_dir
+
+from .models import AISystem, CertPolicy, Decision
+from .signing import sign
+
+CERT_TYPE = "AISystemAssuranceCertificate"
+CERT_VERSION = "1.0"
+
+GATE_TEXT = {
+    "sec_asset_scanned": {"zh": "扫描覆盖该系统资产 {asset}", "en": "Scan covers the system asset {asset}"},
+    "sec_packs_assessed": {"zh": "已评估登记的全部规则包", "en": "All registered rule packs were assessed"},
+    "sec_score": {"zh": "安全合规得分 {score}% > {threshold}%", "en": "Security compliance score {score}% > {threshold}%"},
+    "sec_no_blocking": {"zh": "无 {severities} 级未修复项", "en": "No open {severities} findings"},
+    "sec_no_pending": {"zh": "无待人工证明项", "en": "No pending attestations"},
+    "sec_fresh": {"zh": "扫描证据在 {max} 天内（当前 {age} 天）", "en": "Scan evidence within {max} days (now {age} days)"},
+    "q_run": {"zh": "存在 {agent} × {exam} 的评测记录", "en": "Evaluation exists for {agent} × {exam}"},
+    "q_score": {"zh": "评测综合得分 {score} > {threshold}", "en": "Evaluation score {score} > {threshold}"},
+    "q_no_critical": {"zh": "评测无关键项失败（当前 {n}）", "en": "No critical evaluation failures (now {n})"},
+    "q_no_errors": {"zh": "评测无会话错误（当前 {n}）", "en": "No evaluation session errors (now {n})"},
+    "q_fresh": {"zh": "评测证据在 {max} 天内（当前 {age} 天）", "en": "Evaluation evidence within {max} days (now {age} days)"},
+    "q_llm_judge": {"zh": "使用 LLM 裁判（当前 {judge}；否则仅颁发 DEMO 证书）",
+                    "en": "LLM judge used (now {judge}; otherwise a DEMO certificate only)"},
+}
+
+
+def gate_text(code: str, params: dict[str, Any], lang: str = "zh") -> str:
+    tpl = GATE_TEXT.get(code, {}).get(lang) or code
+    p = {k: ("—" if v is None else ", ".join(map(str, v)) if isinstance(v, list) else v) for k, v in params.items()}
+    try:
+        return tpl.format(**p)
+    except (KeyError, IndexError):
+        return tpl
+
+
+# ---------------- Registry ----------------
+
+def certification_dir() -> Path:
+    return content_dir("certification", "AGENTEVAL_CERT_DIR")
+
+
+def load_policy(directory: Path | None = None) -> CertPolicy:
+    p = Path(directory or certification_dir()) / "policy.yaml"
+    if not p.exists():
+        return CertPolicy()
+    data = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
+    check("policy", data, str(p))
+    return CertPolicy.model_validate(data)
+
+
+def load_systems(directory: Path | None = None) -> dict[str, AISystem]:
+    d = Path(directory or certification_dir()) / "systems"
+    out: dict[str, AISystem] = {}
+    for f in sorted(d.glob("*.y*ml")) if d.exists() else []:
+        s = load_system_file(f)
+        if s.id in out:
+            raise ValueError(f"{f.name}: duplicate system ID {s.id}")
+        out[s.id] = s
+    return out
+
+
+def load_system_file(path: Path) -> AISystem:
+    data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {}
+    check("system", data, str(path))
+    return AISystem.model_validate(data)
+
+
+# ---------------- Certificates ----------------
+
+def add_months(dt: datetime, months: int) -> datetime:
+    m = dt.month - 1 + months
+    y, m = dt.year + m // 12, m % 12 + 1
+    days = [31, 29 if (y % 4 == 0 and (y % 100 != 0 or y % 400 == 0)) else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+    return dt.replace(year=y, month=m, day=min(dt.day, days[m - 1]))
+
+
+def new_cert_id(now: datetime) -> str:
+    return f"AEV-{now:%Y%m}-{secrets.token_hex(4).upper()}"
+
+
+def build_certificate(system: AISystem, decision: Decision, policy: CertPolicy, verify_url_template: str | None,
+                      now: datetime | None = None) -> dict[str, Any]:
+    if not decision.eligible:
+        raise ValueError("Not eligible for certification")
+    now = (now or datetime.now(timezone.utc)).replace(microsecond=0)
+    cid = new_cert_id(now)
+    sec, q = decision.security, decision.quality
+    payload = {
+        "type": CERT_TYPE,
+        "version": CERT_VERSION,
+        "id": cid,
+        "demo": decision.demo,
+        "issuer": {"name": policy.issuer_name, "name_zh": policy.issuer_name_zh},
+        "subject": {
+            "system_id": system.id, "name": system.name, "name_zh": system.name_zh,
+            "organization": system.organization, "industry": system.industry,
+            "description": system.description, "description_zh": system.description_zh,
+            "asset_id": system.asset_id, "model": system.model, "agent_name": system.agent_name,
+        },
+        "assurance": {
+            "security": {
+                "score": sec.score, "threshold": policy.min_security_score,
+                "scan_id": sec.scan_id, "rules_version": sec.rules_version,
+                "scanned_at": sec.scanned_at.isoformat() if sec.scanned_at else None,
+                "checks_passed": sec.passed, "checks_failed": sec.failed,
+                "packs": [p.model_dump() for p in sec.packs],
+            },
+            "quality": {
+                "score": q.score, "threshold": policy.min_quality_score, "grade": q.grade,
+                "run_id": q.run_id, "exam": q.exam, "exam_version": q.exam_version, "judge": q.judge,
+                "evaluated_at": q.evaluated_at.isoformat() if q.evaluated_at else None,
+                "cases_passed": q.cases_passed, "cases": q.cases, "critical_failures": q.critical_failures,
+                "dimensions": q.dimensions,
+            },
+        },
+        "criteria": {
+            "security_score_gt": policy.min_security_score, "quality_score_gt": policy.min_quality_score,
+            "no_open_severities": policy.block_on_severities,
+            "no_pending_attestations": policy.require_no_pending_attestations,
+            "zero_critical_eval_failures": policy.require_zero_critical_eval_failures,
+            "llm_judge_required": policy.require_llm_judge,
+        },
+        "issued_at": now.isoformat(),
+        "expires_at": add_months(now, policy.validity_months).isoformat(),
+        # Optional public verification page, e.g. "https://trust.example.com/certs/{id}"; None = offline only
+        "verify_url": verify_url_template.format(id=cid) if verify_url_template else None,
+        "disclaimer": (
+            "Issued by the AgentEval toolkit from automated technical controls, submitted attestations and a "
+            "standardized agent evaluation. It is not a government certification, a legal opinion, or a PCI DSS "
+            "validation by a Qualified Security Assessor. Rule-to-regulation mappings are pending legal review."
+        ),
+        "disclaimer_zh": (
+            "本证书由 AgentEval 工具依据自动化技术控制检查、提交的证明材料与标准化 Agent 评测签发，"
+            "不是政府认证、法律意见，也不等同于 PCI DSS 合格评估师（QSA）的合规验证；规则与法规的映射尚待法务审核。"
+        ),
+    }
+    return payload
+
+
+def issue(system: AISystem, decision: Decision, policy: CertPolicy, key, verify_url_template: str | None = None,
+          now: datetime | None = None) -> dict[str, Any]:
+    return sign(build_certificate(system, decision, policy, verify_url_template, now), key)
+
+
+def certificate_state(envelope: dict, stored_status: str | None = None, now: datetime | None = None) -> str:
+    """Current certificate state: revoked/superseded per the registry; otherwise based on the validity period."""
+    if stored_status in ("revoked", "superseded"):
+        return stored_status
+    now = now or datetime.now(timezone.utc)
+    exp = datetime.fromisoformat(envelope["certificate"]["expires_at"])
+    return "expired" if now > exp else "active"