agenteval-toolkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. agenteval/__init__.py +5 -0
  2. agenteval/_starter/certification/policy.yaml +15 -0
  3. agenteval/_starter/certification/systems/northwind-cs-staging.yaml +16 -0
  4. agenteval/_starter/certification/systems/northwind-cs.yaml +17 -0
  5. agenteval/_starter/examples/agents/demo-good.yaml +41 -0
  6. agenteval/_starter/examples/agents/demo-weak.yaml +34 -0
  7. agenteval/_starter/examples/agents/openai-compatible.example.yaml +15 -0
  8. agenteval/_starter/examples/agents/webhook.example.yaml +21 -0
  9. agenteval/_starter/examples/attestations-certification.yaml +16 -0
  10. agenteval/_starter/examples/attestations.yaml +19 -0
  11. agenteval/_starter/examples/snapshots/sample-cluster.json +323 -0
  12. agenteval/_starter/exams/customer-service/cs-ecommerce-v1.yaml +222 -0
  13. agenteval/_starter/rules/industry-pci-dss/i18n/zh.yaml +27 -0
  14. agenteval/_starter/rules/industry-pci-dss/pack.yaml +9 -0
  15. agenteval/_starter/rules/industry-pci-dss/rules.yaml +83 -0
  16. agenteval/_starter/rules/us-ca/i18n/zh.yaml +21 -0
  17. agenteval/_starter/rules/us-ca/pack.yaml +9 -0
  18. agenteval/_starter/rules/us-ca/rules.yaml +72 -0
  19. agenteval/_starter/rules/us-federal/governance.yaml +216 -0
  20. agenteval/_starter/rules/us-federal/i18n/zh.yaml +107 -0
  21. agenteval/_starter/rules/us-federal/pack.yaml +8 -0
  22. agenteval/_starter/rules/us-federal/security.yaml +164 -0
  23. agenteval/certification/__init__.py +16 -0
  24. agenteval/certification/assess.py +137 -0
  25. agenteval/certification/issuer.py +165 -0
  26. agenteval/certification/models.py +113 -0
  27. agenteval/certification/render.py +100 -0
  28. agenteval/certification/signing.py +86 -0
  29. agenteval/cli.py +533 -0
  30. agenteval/collectors/__init__.py +30 -0
  31. agenteval/collectors/kubernetes.py +70 -0
  32. agenteval/collectors/normalize.py +247 -0
  33. agenteval/config.py +49 -0
  34. agenteval/engine/__init__.py +12 -0
  35. agenteval/engine/conditions.py +109 -0
  36. agenteval/engine/evaluator.py +133 -0
  37. agenteval/engine/loader.py +131 -0
  38. agenteval/evaluation/__init__.py +17 -0
  39. agenteval/evaluation/adapters.py +276 -0
  40. agenteval/evaluation/assertions.py +73 -0
  41. agenteval/evaluation/judges.py +214 -0
  42. agenteval/evaluation/messages.py +56 -0
  43. agenteval/evaluation/models.py +174 -0
  44. agenteval/evaluation/papers.py +103 -0
  45. agenteval/evaluation/report.py +80 -0
  46. agenteval/evaluation/runner.py +154 -0
  47. agenteval/i18n.py +118 -0
  48. agenteval/models.py +136 -0
  49. agenteval/reports/__init__.py +86 -0
  50. agenteval/reports/templates/certificate.html.j2 +178 -0
  51. agenteval/reports/templates/eval_report.html.j2 +191 -0
  52. agenteval/reports/templates/report.html.j2 +75 -0
  53. agenteval/schema.py +66 -0
  54. agenteval/schemas/attestations.schema.json +76 -0
  55. agenteval/schemas/exam.schema.json +85 -0
  56. agenteval/schemas/pack.schema.json +48 -0
  57. agenteval/schemas/policy.schema.json +67 -0
  58. agenteval/schemas/rule.schema.json +298 -0
  59. agenteval/schemas/system.schema.json +76 -0
  60. agenteval/schemas/translations.schema.json +32 -0
  61. agenteval_toolkit-0.1.0.dist-info/METADATA +307 -0
  62. agenteval_toolkit-0.1.0.dist-info/RECORD +67 -0
  63. agenteval_toolkit-0.1.0.dist-info/WHEEL +5 -0
  64. agenteval_toolkit-0.1.0.dist-info/entry_points.txt +2 -0
  65. agenteval_toolkit-0.1.0.dist-info/licenses/LICENSE +202 -0
  66. agenteval_toolkit-0.1.0.dist-info/licenses/NOTICE +19 -0
  67. agenteval_toolkit-0.1.0.dist-info/top_level.txt +1 -0
agenteval/models.py ADDED
@@ -0,0 +1,136 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # Copyright 2026 The AgentEval Authors
3
+ """Core data models: rules, assets, findings and scan results."""
4
+
5
+ from __future__ import annotations
6
+
7
+ from datetime import date, datetime, timezone
8
+ from enum import Enum
9
+ from typing import Any
10
+
11
+ from pydantic import BaseModel, Field
12
+
13
+
14
+ class Severity(str, Enum):
15
+ critical = "critical"
16
+ high = "high"
17
+ medium = "medium"
18
+ low = "low"
19
+ info = "info"
20
+
21
+ @property
22
+ def rank(self) -> int:
23
+ return ["info", "low", "medium", "high", "critical"].index(self.value)
24
+
25
+
26
+ class RuleType(str, Enum):
27
+ automated = "automated" # Decided automatically by the scanner from collected facts
28
+ attestation = "attestation" # Requires a human-submitted attestation (e.g. an impact assessment)
29
+
30
+
31
+ class Status(str, Enum):
32
+ passed = "pass"
33
+ failed = "fail"
34
+ not_applicable = "not_applicable"
35
+ manual = "manual" # Awaiting human attestation
36
+ error = "error"
37
+
38
+
39
+ class Source(BaseModel):
40
+ """Regulation/framework source a rule maps to."""
41
+
42
+ framework: str
43
+ reference: str
44
+ url: str | None = None
45
+ # Set to True only after legal/compliance has reviewed the mapping
46
+ verified: bool = False
47
+
48
+
49
+ class Applicability(BaseModel):
50
+ asset_types: list[str]
51
+ when: dict[str, Any] | None = None # Optional precondition; same syntax as check
52
+
53
+
54
+ class RulePack(BaseModel):
55
+ """Rule pack, organized by tier (national / regional / industry)."""
56
+
57
+ id: str
58
+ name: str
59
+ name_zh: str = ""
60
+ tier: str # national | regional | industry
61
+ jurisdiction: str
62
+ description: str = ""
63
+ description_zh: str = ""
64
+
65
+
66
+ class Rule(BaseModel):
67
+ id: str
68
+ title: str
69
+ description: str
70
+ severity: Severity
71
+ type: RuleType
72
+ jurisdiction: str = "us-federal"
73
+ category: str
74
+ applies_to: Applicability
75
+ check: dict[str, Any] | None = None
76
+ attestation: dict[str, Any] | None = None
77
+ remediation: str
78
+ sources: list[Source]
79
+ effective_date: date | None = None
80
+ version: str = "1.0.0"
81
+ enabled: bool = True
82
+ pack: str = "" # Set by the loader from the pack.yaml in the rule's directory
83
+ # Attached by the loader from i18n/<lang>.yaml: {lang: {title, description, remediation, sources: [reference...]}}
84
+ translations: dict[str, dict[str, Any]] = Field(default_factory=dict)
85
+
86
+
87
+ class Asset(BaseModel):
88
+ """Normalized asset produced by a collector. facts holds the flat/nested facts used for rule evaluation."""
89
+
90
+ id: str
91
+ type: str
92
+ name: str
93
+ namespace: str | None = None
94
+ facts: dict[str, Any] = Field(default_factory=dict)
95
+
96
+
97
+ class Finding(BaseModel):
98
+ rule_id: str
99
+ pack: str = ""
100
+ rule_title: str
101
+ severity: Severity
102
+ category: str
103
+ asset_id: str
104
+ asset_name: str
105
+ asset_type: str
106
+ status: Status
107
+ message: str # Default Chinese text (backward compatibility)
108
+ message_code: str = "" # for multilingual rendering; see agenteval/i18n.py
109
+ message_params: dict[str, Any] = Field(default_factory=dict)
110
+ evidence: dict[str, Any] = Field(default_factory=dict)
111
+ remediation: str
112
+ sources: list[Source]
113
+ translations: dict[str, dict[str, Any]] = Field(default_factory=dict)
114
+
115
+
116
+ class ScanSummary(BaseModel):
117
+ total: int = 0
118
+ by_status: dict[str, int] = Field(default_factory=dict)
119
+ failed_by_severity: dict[str, int] = Field(default_factory=dict)
120
+ score: float = 100.0 # passed / (passed + failed); manual items are excluded
121
+
122
+
123
+ class ScanResult(BaseModel):
124
+ scan_id: str
125
+ source: str
126
+ started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
127
+ finished_at: datetime | None = None
128
+ rules_version: str
129
+ asset_count: int
130
+ findings: list[Finding]
131
+ summary: ScanSummary
132
+ disclaimer: str = (
133
+ "This report is an evidence-based assessment of technical controls and does not constitute legal "
134
+ "advice or a determination of regulatory compliance. Rule-to-regulation mappings must be reviewed "
135
+ "by your legal/compliance team."
136
+ )
@@ -0,0 +1,86 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # Copyright 2026 The AgentEval Authors
3
+ """Report output: JSON / SARIF 2.1.0 / HTML."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ from importlib import resources
9
+
10
+ from jinja2 import Environment, select_autoescape
11
+
12
+ from agenteval import __version__
13
+ from agenteval.i18n import DEFAULT_LANG, UI, localize_result
14
+ from agenteval.models import ScanResult, Severity, Status
15
+
16
+ FORMATS = ("json", "sarif", "html")
17
+
18
+ _SARIF_LEVEL = {
19
+ Severity.critical: "error", Severity.high: "error",
20
+ Severity.medium: "warning", Severity.low: "note", Severity.info: "note",
21
+ }
22
+
23
+
24
+ def to_json(result: ScanResult) -> str:
25
+ return result.model_dump_json(indent=2)
26
+
27
+
28
+ def to_sarif(result: ScanResult) -> str:
29
+ rules_seen: dict[str, dict] = {}
30
+ results = []
31
+ for f in result.findings:
32
+ if f.status not in (Status.failed, Status.manual, Status.error):
33
+ continue
34
+ if f.rule_id not in rules_seen:
35
+ rules_seen[f.rule_id] = {
36
+ "id": f.rule_id,
37
+ "name": f.rule_id.replace("-", ""),
38
+ "shortDescription": {"text": f.rule_title},
39
+ "help": {"text": f.remediation},
40
+ "properties": {
41
+ "severity": f.severity.value,
42
+ "category": f.category,
43
+ "sources": [s.model_dump() for s in f.sources],
44
+ },
45
+ }
46
+ results.append({
47
+ "ruleId": f.rule_id,
48
+ "level": _SARIF_LEVEL[f.severity] if f.status == Status.failed else "note",
49
+ "kind": "fail" if f.status == Status.failed else "review",
50
+ "message": {"text": f"[{f.asset_id}] {f.message}"},
51
+ "locations": [{"logicalLocations": [{"fullyQualifiedName": f.asset_id, "kind": f.asset_type}]}],
52
+ "properties": {"status": f.status.value, "evidence": f.evidence},
53
+ })
54
+ doc = {
55
+ "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
56
+ "version": "2.1.0",
57
+ "runs": [{
58
+ "tool": {"driver": {
59
+ "name": "agenteval", "version": __version__,
60
+ "informationUri": "https://github.com/your-org/agenteval",
61
+ "rules": list(rules_seen.values()),
62
+ }},
63
+ "automationDetails": {"id": f"agenteval/{result.scan_id}"},
64
+ "properties": {"rulesVersion": result.rules_version, "summary": result.summary.model_dump()},
65
+ "results": results,
66
+ }],
67
+ }
68
+ return json.dumps(doc, indent=2, ensure_ascii=False, default=str)
69
+
70
+
71
+ def to_html(result: ScanResult, lang: str = DEFAULT_LANG) -> str:
72
+ env = Environment(autoescape=select_autoescape(default=True, default_for_string=True))
73
+ env.filters["tojson_pretty"] = lambda v: json.dumps(v, ensure_ascii=False, indent=1, default=str)
74
+ tpl_text = resources.files("agenteval").joinpath("reports/templates/report.html.j2").read_text(encoding="utf-8")
75
+ return env.from_string(tpl_text).render(r=result, t=UI[lang])
76
+
77
+
78
+ def render(result: ScanResult, fmt: str, lang: str = DEFAULT_LANG) -> str:
79
+ result = localize_result(result, lang)
80
+ if fmt == "json":
81
+ return to_json(result)
82
+ if fmt == "sarif":
83
+ return to_sarif(result)
84
+ if fmt == "html":
85
+ return to_html(result, lang)
86
+ raise ValueError(f"Unsupported format: {fmt} (choose from: {', '.join(FORMATS)})")
@@ -0,0 +1,178 @@
1
+ <!doctype html>
2
+ <html lang="{{ t.html_lang }}">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>{{ t.title }} · {{ c.id }}</title>
7
+ <link rel="preconnect" href="https://fonts.googleapis.com">
8
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500&family=Source+Serif+4:opsz,wght@8..60,600&display=swap">
9
+ <style>
10
+ /* 证书为单一视觉:浅色纸面,固定配色以保证打印一致 */
11
+ :root {
12
+ --paper: #fbfcfd; --ink: #142036; --muted: #5a6779; --line: #cfd7e1; --soft: #eef2f6;
13
+ --accent: #145c4f; --accent-soft: #e3f0ec; --bad: #a32020; --bad-soft: #fbe9e9; --demo: #b42318;
14
+ --serif: "Source Serif 4", "Noto Serif CJK SC", "Songti SC", Georgia, serif;
15
+ --sans: "IBM Plex Sans", "Inter", "Noto Sans CJK SC", "PingFang SC", "Microsoft YaHei", system-ui, sans-serif;
16
+ --mono: "IBM Plex Mono", "DejaVu Sans Mono", ui-monospace, monospace;
17
+ color-scheme: light;
18
+ }
19
+ * { box-sizing: border-box; }
20
+ html, body { margin: 0; background: #e6eaef; color: var(--ink); font: 13px/1.45 var(--sans); }
21
+ .toolbar { max-width: 1062px; margin: 16px auto 0; padding-inline: 16px; display: flex; justify-content: flex-end; }
22
+ .toolbar button { font: 500 13px var(--sans); padding: 6px 12px; border-radius: 6px; border: 1px solid var(--line);
23
+ background: #fff; color: var(--ink); cursor: pointer; }
24
+ .wrap { overflow-x: auto; padding: 16px; }
25
+ .sheet { position: relative; width: 1062px; margin: 0 auto; background: var(--paper); padding: 26px 34px 22px;
26
+ border: 1px solid var(--line); box-shadow: 0 1px 2px rgba(20, 32, 54, .08); overflow: hidden; }
27
+ .sheet::before { content: ""; position: absolute; inset: 10px; border: 1.5px solid var(--accent); pointer-events: none; }
28
+ .sheet::after { content: ""; position: absolute; inset: 14px; border: .5px solid var(--accent); opacity: .5; pointer-events: none; }
29
+ .top { display: flex; justify-content: space-between; align-items: baseline; gap: 16px; }
30
+ .issuer { font: 600 13px var(--sans); letter-spacing: .06em; text-transform: uppercase; color: var(--accent); }
31
+ .no { font: 500 12px var(--mono); color: var(--muted); }
32
+ h1 { font: 600 30px/1.15 var(--serif); text-align: center; margin: 14px 0 4px; letter-spacing: .01em; }
33
+ .certifies { text-align: center; color: var(--muted); margin: 6px 0 2px; }
34
+ .name { font: 600 26px/1.2 var(--serif); text-align: center; margin: 0; text-wrap: balance; }
35
+ .org { text-align: center; color: var(--muted); margin: 2px 0 0; }
36
+ .statement { text-align: center; max-width: 760px; margin: 8px auto 0; }
37
+ .facts { display: flex; justify-content: center; flex-wrap: wrap; gap: 6px 22px; margin-top: 8px; font-size: 12px; color: var(--muted); }
38
+ .facts b { color: var(--ink); font-weight: 500; font-family: var(--mono); font-size: 11.5px; }
39
+ .panels { display: grid; grid-template-columns: 1.25fr 1fr; gap: 16px; margin-top: 14px; }
40
+ .panel { border: 1px solid var(--line); border-radius: 8px; padding: 12px 14px; background: #fff; min-width: 0; }
41
+ .panel-head { display: flex; justify-content: space-between; align-items: flex-start; gap: 10px; margin-bottom: 8px; }
42
+ .panel-head h2 { font: 600 15px var(--sans); margin: 0; }
43
+ .panel-head p { margin: 1px 0 0; color: var(--muted); font-size: 11.5px; }
44
+ .big { font: 600 28px/1 var(--sans); color: var(--accent); white-space: nowrap; text-align: right; }
45
+ .big small { display: block; font: 400 11px var(--sans); color: var(--muted); margin-top: 3px; }
46
+ table { width: 100%; border-collapse: collapse; font-size: 12px; }
47
+ th, td { text-align: left; padding: 4px 6px; border-bottom: 1px solid var(--soft); vertical-align: top; }
48
+ th { color: var(--muted); font-weight: 500; font-size: 11px; }
49
+ td.num { font-variant-numeric: tabular-nums; white-space: nowrap; }
50
+ .pill { display: inline-block; padding: 0 7px; border-radius: 999px; font-size: 11px; font-weight: 600; white-space: nowrap; }
51
+ .pill.passed { background: var(--accent-soft); color: var(--accent); }
52
+ .pill.failed { background: var(--bad-soft); color: var(--bad); }
53
+ .pill.not_applicable, .pill.not_assessed { background: var(--soft); color: var(--muted); }
54
+ .meta-q { display: flex; flex-wrap: wrap; gap: 4px 14px; font-size: 11.5px; color: var(--muted); margin-bottom: 6px; }
55
+ .meta-q b { color: var(--ink); font-weight: 500; }
56
+ .bottom { display: grid; grid-template-columns: 1fr auto auto; gap: 18px; align-items: end; margin-top: 14px; }
57
+ .dates { display: flex; gap: 26px; }
58
+ .dates div span { display: block; color: var(--muted); font-size: 11px; }
59
+ .dates div b { font: 600 15px var(--sans); }
60
+ .evidence { font: 11px/1.5 var(--mono); color: var(--muted); margin-top: 6px; overflow-wrap: anywhere; }
61
+ .seal { width: 92px; height: 92px; border-radius: 50%; border: 2px solid var(--accent); display: grid; place-items: center;
62
+ text-align: center; color: var(--accent); font: 600 9.5px/1.25 var(--sans); letter-spacing: .04em; position: relative; }
63
+ .seal::before { content: ""; position: absolute; inset: 5px; border-radius: 50%; border: .75px solid var(--accent); }
64
+ .seal b { display: block; font-size: 22px; letter-spacing: 0; }
65
+ .qr { display: grid; justify-items: center; gap: 4px; width: 150px; }
66
+ .qr svg { width: 92px; height: 92px; background: #fff; }
67
+ .qr span { font-size: 10px; color: var(--muted); text-align: center; }
68
+ .verify-url { font: 9.5px var(--mono); color: var(--muted); overflow-wrap: anywhere; text-align: center; }
69
+ .sigline { margin-top: 10px; padding-top: 8px; border-top: 1px solid var(--soft); display: flex; justify-content: space-between;
70
+ gap: 16px; font-size: 10.5px; color: var(--muted); }
71
+ .sigline code { font: 10.5px var(--mono); color: var(--ink); }
72
+ .disclaimer { font-size: 10px; color: var(--muted); margin: 6px 0 0; }
73
+ .demo-band { margin-top: 10px; background: var(--bad-soft); color: var(--demo); border: 1px solid #f1c4c0; border-radius: 6px;
74
+ padding: 5px 10px; font-size: 11.5px; text-align: center; }
75
+ .demo-band b { letter-spacing: .04em; }
76
+ .watermark { position: absolute; inset: 0; display: grid; place-items: center; pointer-events: none; }
77
+ .watermark span { transform: rotate(-24deg); font: 700 150px/1 var(--sans); color: rgba(180, 35, 24, .07); letter-spacing: .12em; }
78
+ .stamp { position: absolute; top: 120px; right: 60px; transform: rotate(-12deg); border: 4px solid var(--bad); color: var(--bad);
79
+ font: 700 34px/1 var(--sans); padding: 8px 18px; border-radius: 8px; letter-spacing: .08em; background: rgba(255, 255, 255, .7); }
80
+ @page { size: A4 landscape; margin: 7mm; }
81
+ @media print {
82
+ html, body { background: #fff; }
83
+ .toolbar { display: none; }
84
+ .wrap { padding: 0; overflow: visible; }
85
+ .sheet { box-shadow: none; border: none; width: 1062px; zoom: .84; }
86
+ }
87
+ </style>
88
+ </head>
89
+ <body>
90
+ {% set s = c.subject %}{% set sec = c.assurance.security %}{% set q = c.assurance.quality %}
91
+ {% if printable_button %}<div class="toolbar"><button type="button" onclick="window.print()">{{ t.print }}</button></div>{% endif %}
92
+ <div class="wrap">
93
+ <article class="sheet" aria-label="{{ t.title }}">
94
+ {% if c.demo %}<div class="watermark" aria-hidden="true"><span>DEMO</span></div>{% endif %}
95
+ {% if state != 'active' %}<div class="stamp">{{ t['state_' ~ state] }}</div>{% endif %}
96
+
97
+ <div class="top">
98
+ <div class="issuer">{{ c.issuer.name_zh if lang == 'zh' and c.issuer.name_zh else c.issuer.name }}</div>
99
+ <div class="no">{{ t.cert_no }} {{ c.id }}</div>
100
+ </div>
101
+
102
+ <h1>{{ t.title }}</h1>
103
+ <p class="certifies">{{ t.certifies }}</p>
104
+ <p class="name">{{ s.name_zh if lang == 'zh' and s.name_zh else s.name }}</p>
105
+ <p class="org">{{ s.organization }}{% if s.industry %} · {{ s.industry }}{% endif %}</p>
106
+ <p class="statement">{{ t.statement }}</p>
107
+ <div class="facts">
108
+ <span>{{ t.asset }} <b>{{ s.asset_id }}</b></span>
109
+ {% if s.model %}<span>{{ t.model }} <b>{{ s.model }}</b></span>{% endif %}
110
+ <span>{{ t.agent }} <b>{{ s.agent_name }}</b></span>
111
+ </div>
112
+
113
+ <div class="panels">
114
+ <section class="panel">
115
+ <div class="panel-head">
116
+ <div><h2>{{ t.security }}</h2><p>{{ t.security_sub }}</p></div>
117
+ <div class="big">{{ '%.1f' % sec.score }}%<small>{{ t.threshold }} &gt; {{ sec.threshold | int }}%</small></div>
118
+ </div>
119
+ <table>
120
+ <thead><tr><th>{{ t.tier }}</th><th>{{ t.pack }}</th><th>{{ t.checks }}</th><th>{{ t.status }}</th></tr></thead>
121
+ <tbody>
122
+ {% for p in sec.packs %}
123
+ <tr>
124
+ <td>{{ t[p.tier] if p.tier in ('national', 'regional', 'industry') else t.unknown }}</td>
125
+ <td>{{ p.name_zh if lang == 'zh' and p.name_zh else p.name }}</td>
126
+ <td class="num">{{ p.passed }} / {{ p.passed + p.failed + p.manual }}</td>
127
+ <td><span class="pill {{ p.status }}">{{ t['st_' ~ p.status] }}</span></td>
128
+ </tr>
129
+ {% endfor %}
130
+ </tbody>
131
+ </table>
132
+ </section>
133
+ <section class="panel">
134
+ <div class="panel-head">
135
+ <div><h2>{{ t.quality }}</h2><p>{{ t.quality_sub }}</p></div>
136
+ <div class="big">{{ '%.1f' % q.score }}<small>{{ t.grade }} {{ q.grade }} · {{ t.threshold }} &gt; {{ q.threshold | int }}</small></div>
137
+ </div>
138
+ <div class="meta-q">
139
+ <span>{{ t.exam }} <b>{{ q.exam }}@{{ q.exam_version }}</b></span>
140
+ <span>{{ t.cases }} <b>{{ q.cases_passed }}/{{ q.cases }}</b></span>
141
+ <span>{{ t.judge }} <b>{{ q.judge }}</b></span>
142
+ </div>
143
+ <table>
144
+ <thead><tr><th>{{ t.dimension }}</th><th>{{ t.score }}</th></tr></thead>
145
+ <tbody>
146
+ {% for d in q.dimensions %}
147
+ <tr><td>{{ d.name }} <span style="color:var(--muted)">· {{ (d.weight * 100) | round | int }}%</span></td>
148
+ <td class="num">{{ d.score if d.score is not none else '—' }}</td></tr>
149
+ {% endfor %}
150
+ </tbody>
151
+ </table>
152
+ </section>
153
+ </div>
154
+
155
+ {% if c.demo %}<div class="demo-band"><b>{{ t.demo }}</b> — {{ t.demo_reason }}</div>{% endif %}
156
+
157
+ <div class="bottom">
158
+ <div>
159
+ <div class="dates">
160
+ <div><span>{{ t.issued }}</span><b>{{ c.issued_at | date }}</b></div>
161
+ <div><span>{{ t.expires }}</span><b>{{ c.expires_at | date }}</b></div>
162
+ </div>
163
+ <div class="evidence">{{ t.evidence }}: {{ t.scan }} {{ sec.scan_id }} · {{ t.rules }} {{ sec.rules_version }} · {{ t.run }} {{ q.run_id }}</div>
164
+ </div>
165
+ <div class="seal" aria-hidden="true"><div>AGENTEVAL<b>✓</b>{{ 'ASSURED' if lang == 'en' else '认证' }}</div></div>
166
+ {% if qr %}<div class="qr">{{ qr }}<span>{{ t.verify }}</span><div class="verify-url">{{ c.verify_url }}</div></div>
167
+ {% else %}<div class="qr"><span>{{ t.verify_offline }}</span><div class="verify-url">agenteval cert verify {{ c.id }}.json --public-key issuer.pem</div></div>{% endif %}
168
+ </div>
169
+
170
+ <div class="sigline">
171
+ <span>{{ t.signature }} <code>{{ sig.key_id }}</code></span>
172
+ <span><code>{{ sig.value[:44] }}…</code></span>
173
+ </div>
174
+ <p class="disclaimer">{{ c.disclaimer_zh if lang == 'zh' else c.disclaimer }}</p>
175
+ </article>
176
+ </div>
177
+ </body>
178
+ </html>
@@ -0,0 +1,191 @@
1
+ <!doctype html>
2
+ <html lang="{{ t.html_lang }}">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>{{ t.title }} · {{ r.agent_name }}</title>
7
+ <style>
8
+ :root {
9
+ color-scheme: light;
10
+ --surface: #fcfcfb; --surface-2: #f1f1ef; --line: #e2e1dd;
11
+ --text-primary: #0b0b0b; --text-secondary: #52514e; --text-muted: #77756f;
12
+ --series-1: #2a78d6; --track: #e4ecf7;
13
+ --good: #1f7a3f; --good-bg: #e7f4ec; --critical: #b42318; --critical-bg: #fcebea; --warning: #8a5a00;
14
+ }
15
+ @media (prefers-color-scheme: dark) {
16
+ :root:where(:not([data-theme="light"])) {
17
+ color-scheme: dark;
18
+ --surface: #1a1a19; --surface-2: #232321; --line: #34332f;
19
+ --text-primary: #ffffff; --text-secondary: #c3c2b7; --text-muted: #9a988f;
20
+ --series-1: #3987e5; --track: #22324a;
21
+ --good: #6fd39b; --good-bg: #183324; --critical: #ff8a80; --critical-bg: #3a1d1b; --warning: #f0c060;
22
+ }
23
+ }
24
+ :root[data-theme="dark"] {
25
+ color-scheme: dark;
26
+ --surface: #1a1a19; --surface-2: #232321; --line: #34332f;
27
+ --text-primary: #ffffff; --text-secondary: #c3c2b7; --text-muted: #9a988f;
28
+ --series-1: #3987e5; --track: #22324a;
29
+ --good: #6fd39b; --good-bg: #183324; --critical: #ff8a80; --critical-bg: #3a1d1b; --warning: #f0c060;
30
+ }
31
+ * { box-sizing: border-box; }
32
+ body { margin: 0; background: var(--surface); color: var(--text-primary);
33
+ font: 14px/1.55 system-ui, -apple-system, "Segoe UI", "PingFang SC", sans-serif; }
34
+ main { max-width: 1080px; margin: 0 auto; padding: 28px 16px 56px; }
35
+ h1 { font-size: 22px; margin: 0 0 4px; }
36
+ h2 { font-size: 16px; margin: 32px 0 12px; }
37
+ .meta { color: var(--text-secondary); font-size: 13px; }
38
+ .meta span + span::before { content: " · "; color: var(--text-muted); }
39
+ .hero { display: grid; grid-template-columns: minmax(200px, 260px) 1fr; gap: 16px; margin-top: 20px; }
40
+ @media (max-width: 640px) { .hero { grid-template-columns: 1fr; } }
41
+ .card { background: var(--surface-2); border: 1px solid var(--line); border-radius: 10px; padding: 16px; }
42
+ .score { font-size: 56px; font-weight: 650; line-height: 1; }
43
+ .score small { font-size: 18px; color: var(--text-secondary); font-weight: 500; }
44
+ .badges { display: flex; gap: 8px; flex-wrap: wrap; margin-top: 12px; }
45
+ .badge { display: inline-flex; align-items: center; gap: 4px; padding: 2px 10px; border-radius: 999px;
46
+ font-weight: 600; font-size: 13px; border: 1px solid var(--line); color: var(--text-primary); }
47
+ .badge.good { background: var(--good-bg); color: var(--good); border-color: transparent; }
48
+ .badge.bad { background: var(--critical-bg); color: var(--critical); border-color: transparent; }
49
+ .kpis { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 12px; }
50
+ .kpi b { display: block; font-size: 22px; }
51
+ .kpi { color: var(--text-secondary); font-size: 13px; }
52
+ .kpi b { color: var(--text-primary); }
53
+ .bars { display: grid; gap: 10px; }
54
+ .bar-row { display: grid; grid-template-columns: minmax(120px, 200px) 1fr 52px; align-items: center; gap: 12px; }
55
+ .bar-label { font-size: 13px; }
56
+ .bar-label small { display: block; color: var(--text-muted); font-size: 12px; }
57
+ .track { height: 14px; background: var(--track); border-radius: 0 4px 4px 0; position: relative; }
58
+ .fill { height: 14px; background: var(--series-1); border-radius: 0 4px 4px 0; }
59
+ .pass-mark { position: absolute; top: -4px; bottom: -4px; width: 1px; background: var(--text-muted); }
60
+ .bar-val { font-variant-numeric: tabular-nums; text-align: right; font-weight: 600; }
61
+ .note { border-left: 3px solid var(--warning); padding: 8px 12px; background: var(--surface-2); font-size: 13px;
62
+ color: var(--text-secondary); margin-top: 16px; }
63
+ .wrap { overflow-x: auto; }
64
+ table { width: 100%; border-collapse: collapse; }
65
+ th, td { text-align: left; padding: 8px; border-bottom: 1px solid var(--line); vertical-align: top; }
66
+ th { font-size: 12px; color: var(--text-secondary); font-weight: 600; }
67
+ td.num { font-variant-numeric: tabular-nums; }
68
+ .ok { color: var(--good); font-weight: 600; white-space: nowrap; }
69
+ .ko { color: var(--critical); font-weight: 600; white-space: nowrap; }
70
+ .ko-msg { color: var(--critical); font-size: 12px; margin-top: 2px; }
71
+ details { margin-top: 6px; }
72
+ details summary { cursor: pointer; color: var(--text-secondary); }
73
+ .msg { margin: 6px 0; padding: 8px 10px; border-radius: 8px; background: var(--surface); border: 1px solid var(--line);
74
+ white-space: pre-wrap; word-break: break-word; }
75
+ .msg .who { font-size: 12px; color: var(--text-muted); display: block; }
76
+ .tool { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12px; color: var(--text-secondary); }
77
+ .small { font-size: 12px; color: var(--text-secondary); }
78
+ table.cases { table-layout: fixed; min-width: 720px; }
79
+ table.cases td { overflow-wrap: anywhere; }
80
+ tr.case-row td { border-bottom: none; padding-bottom: 2px; }
81
+ tr.case-detail td { padding-top: 0; }
82
+ table.inner { table-layout: fixed; background: var(--surface); border: 1px solid var(--line); border-radius: 8px; margin: 6px 0; }
83
+ table.inner td { border-bottom: 1px solid var(--line); word-break: break-word; }
84
+ td.code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }
85
+ footer { margin-top: 40px; color: var(--text-muted); font-size: 12px; border-top: 1px solid var(--line); padding-top: 12px; }
86
+ </style>
87
+ </head>
88
+ <body>
89
+ <main>
90
+ <h1>{{ t.title }}</h1>
91
+ <div class="meta">
92
+ <span>{{ t.paper }}: {{ r.paper_title }} ({{ r.paper_id }}@{{ r.paper_version }})</span>
93
+ <span>{{ t.agent }}: {{ r.agent_name }}</span>
94
+ <span>{{ t.judge }}: {{ r.judge }}</span>
95
+ <span>{{ t.run }}: {{ r.run_id }}</span>
96
+ <span>{{ r.started_at.strftime('%Y-%m-%d %H:%M UTC') }}</span>
97
+ </div>
98
+
99
+ <div class="hero">
100
+ <div class="card">
101
+ <div class="small">{{ t.overall }}</div>
102
+ <div class="score">{{ '%.1f' % r.overall_score }}<small> / 100</small></div>
103
+ <div class="badges">
104
+ <span class="badge">{{ t.grade }} {{ r.grade }}</span>
105
+ {% if r.passed %}<span class="badge good">✓ {{ t.passed }}</span>
106
+ {% else %}<span class="badge bad">✕ {{ t.not_passed }}</span>{% endif %}
107
+ </div>
108
+ <div class="small" style="margin-top:8px">{{ t.passing }}: {{ r.passing_score }}</div>
109
+ </div>
110
+ <div class="card kpis">
111
+ <div class="kpi"><b>{{ r.stats.cases_passed }} / {{ r.stats.cases }}</b>{{ t.cases }}</div>
112
+ <div class="kpi"><b>{{ r.stats.pass_rate }}%</b>{{ t.case_pass_rate }}</div>
113
+ <div class="kpi"><b>{{ r.stats.critical_failures }}</b>{{ t.critical }}</div>
114
+ <div class="kpi"><b>{{ r.stats.latency_p50_ms | int if r.stats.latency_p50_ms is not none else '—' }} / {{ r.stats.latency_p95_ms | int if r.stats.latency_p95_ms is not none else '—' }} ms</b>{{ t.latency }}</div>
115
+ </div>
116
+ </div>
117
+
118
+ {% if r.judge == 'heuristic' %}<p class="note">{{ t.heuristic_note }}</p>{% endif %}
119
+
120
+ <h2>{{ t.dimensions }}</h2>
121
+ <div class="card bars" role="table" aria-label="{{ t.dimensions }}">
122
+ {% for d in r.dimensions %}
123
+ <div class="bar-row" role="row" title="{{ d.name }}: {{ d.score if d.score is not none else t.na }} ({{ t.weight }} {{ (d.weight * 100) | round | int }}%, {{ d.items }} {{ t.items_label }})">
124
+ <div class="bar-label" role="cell">{{ d.name }}<small>{{ t.weight }} {{ (d.weight * 100) | round | int }}% · {{ d.items }} {{ t.items_label }}</small></div>
125
+ <div class="track" role="cell">
126
+ {% if d.score is not none %}<div class="fill" style="width: {{ d.score }}%"></div>{% endif %}
127
+ <div class="pass-mark" style="left: {{ r.passing_score }}%" aria-hidden="true"></div>
128
+ </div>
129
+ <div class="bar-val" role="cell">{{ d.score if d.score is not none else '—' }}</div>
130
+ </div>
131
+ {% endfor %}
132
+ </div>
133
+
134
+ <h2>{{ t.case_results }}</h2>
135
+ <div class="wrap">
136
+ <table class="cases">
137
+ <colgroup><col style="width:76px"><col><col style="width:140px"><col style="width:84px"><col style="width:64px"><col style="width:96px"></colgroup>
138
+ <thead><tr><th>ID</th><th>{{ t.cases }}</th><th>{{ t.category }}</th><th>{{ t.difficulty }}</th><th>{{ t.score }}</th><th>{{ t.status }}</th></tr></thead>
139
+ <tbody>
140
+ {% for c in r.cases %}
141
+ <tr class="case-row">
142
+ <td><b>{{ c.id }}</b></td>
143
+ <td>{{ c.title }}
144
+ {% if c.error %}<div class="ko-msg">{{ t.error }}: {{ c.error }}</div>{% endif %}
145
+ {% if c.critical_failures %}<div class="ko-msg">✕ {{ t.critical }}: {{ c.critical_failures | join(', ') }}</div>{% endif %}
146
+ </td>
147
+ <td class="small">{{ c.category }}</td>
148
+ <td class="small">{{ c.difficulty }}</td>
149
+ <td class="num"><b>{{ c.score }}</b></td>
150
+ <td class="{{ 'ok' if c.passed else 'ko' }}">{{ ('✓ ' ~ t.case_pass) if c.passed else ('✕ ' ~ t.case_fail) }}</td>
151
+ </tr>
152
+ <tr class="case-detail"><td></td><td colspan="5">
153
+ <details><summary>{{ t.transcript }}</summary>
154
+ {% for m in c.transcript %}
155
+ <div class="msg"><span class="who">{{ t.user if m.role == 'user' else t.agent_role }}{% if m.latency_ms is not none %} · {{ m.latency_ms | int }} ms{% endif %}</span>{{ m.content }}{% if m.tool_calls %}
156
+ <span class="tool">{{ t.tools }}: {% for tc in m.tool_calls %}{{ tc.name }}({{ tc.arguments | tojson }}){% if not loop.last %}, {% endif %}{% endfor %}</span>{% endif %}</div>
157
+ {% endfor %}
158
+ </details>
159
+ {% if c.assertions %}
160
+ <details><summary>{{ t.assertions }} ({{ c.assertions | selectattr('passed') | list | length }}/{{ c.assertions | length }})</summary>
161
+ <table class="inner">
162
+ {% for a in c.assertions %}
163
+ <tr><td class="{{ 'ok' if a.passed else 'ko' }}" style="width:28px">{{ '✓' if a.passed else '✕' }}</td>
164
+ <td class="small" style="width:170px">T{{ a.turn }} · {{ a.type }}{% if a.critical %} · critical{% endif %}<br>{{ a.dimension }}</td>
165
+ <td class="small code">{{ a.value | tojson }}</td>
166
+ <td class="small" style="width:30%">{{ a.detail }}</td></tr>
167
+ {% endfor %}
168
+ </table>
169
+ </details>
170
+ {% endif %}
171
+ {% if c.rubric %}
172
+ <details><summary>{{ t.rubric }}</summary>
173
+ <table class="inner">
174
+ {% for g in c.rubric %}
175
+ <tr><td class="num" style="width:48px"><b>{{ g.score if g.score is not none else '—' }}</b>/5</td>
176
+ <td class="small" style="width:140px">{{ g.dimension }}</td>
177
+ <td class="small">{{ g.criterion }}<br><i>{{ g.rationale }}</i></td></tr>
178
+ {% endfor %}
179
+ </table>
180
+ </details>
181
+ {% endif %}
182
+ </td></tr>
183
+ {% endfor %}
184
+ </tbody>
185
+ </table>
186
+ </div>
187
+
188
+ <footer>{{ t.disclaimer }}</footer>
189
+ </main>
190
+ </body>
191
+ </html>