agenteval-toolkit 0.1.0__py3-none-any.whl

agenteval/reports/templates/report.html.j2

@@ -0,0 +1,75 @@
+<!doctype html>
+<html lang="{{ t.html_lang }}">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{{ t.title }} {{ r.scan_id }}</title>
+<style>
+  :root { --bg:#fff; --fg:#1b1d22; --muted:#5d6370; --line:#e3e5ea; --card:#f7f8fa;
+          --fail:#c0352b; --pass:#2f7d4f; --manual:#9a6a00; --na:#6b7280; }
+  @media (prefers-color-scheme: dark) {
+    :root { --bg:#14161a; --fg:#e8eaee; --muted:#9aa1ad; --line:#2b2f37; --card:#1c1f25;
+            --fail:#ff7a6e; --pass:#6fd39b; --manual:#f0c060; --na:#9aa1ad; }
+  }
+  body { margin:0; background:var(--bg); color:var(--fg); font:14px/1.5 system-ui, -apple-system, "PingFang SC", sans-serif; }
+  main { max-width:1100px; margin:0 auto; padding:24px 16px 48px; }
+  h1 { font-size:22px; margin:0 0 4px; }
+  .meta { color:var(--muted); font-size:13px; }
+  .tiles { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:12px; margin:20px 0; }
+  .tile { background:var(--card); border:1px solid var(--line); border-radius:8px; padding:12px; }
+  .tile b { display:block; font-size:24px; }
+  .disclaimer { border-left:3px solid var(--manual); padding:8px 12px; background:var(--card); font-size:13px; }
+  table { width:100%; border-collapse:collapse; margin-top:16px; }
+  th, td { text-align:left; padding:8px; border-bottom:1px solid var(--line); vertical-align:top; }
+  th { font-size:12px; color:var(--muted); font-weight:600; }
+  .st { font-weight:600; white-space:nowrap; }
+  .st-fail { color:var(--fail); } .st-pass { color:var(--pass); }
+  .st-manual { color:var(--manual); } .st-not_applicable, .st-error { color:var(--na); }
+  details summary { cursor:pointer; color:var(--muted); }
+  pre { white-space:pre-wrap; word-break:break-all; font-size:12px; margin:4px 0 0; }
+  .wrap { overflow-x:auto; }
+</style>
+</head>
+<body>
+<main>
+  <h1>{{ t.title }}</h1>
+  <div class="meta">{{ t.scan_id }} {{ r.scan_id }} · {{ t.source }} {{ r.source }} · {{ t.rules_version }} {{ r.rules_version }} · {{ r.started_at.strftime('%Y-%m-%d %H:%M UTC') }}</div>
+
+  <div class="tiles">
+    <div class="tile"><b>{{ r.summary.score }}%</b>{{ t.score }}</div>
+    <div class="tile"><b>{{ r.asset_count }}</b>{{ t.assets }}</div>
+    <div class="tile"><b>{{ r.summary.by_status.get('fail', 0) }}</b>{{ t.failed }}</div>
+    <div class="tile"><b>{{ r.summary.by_status.get('manual', 0) }}</b>{{ t.manual }}</div>
+    <div class="tile"><b>{{ r.summary.by_status.get('pass', 0) }}</b>{{ t.passed }}</div>
+  </div>
+
+  <p class="disclaimer">{{ r.disclaimer }}</p>
+
+  <div class="wrap">
+  <table>
+    <thead><tr><th>{{ t.col_status }}</th><th>{{ t.col_severity }}</th><th>{{ t.col_rule }}</th><th>{{ t.col_asset }}</th><th>{{ t.col_detail }}</th></tr></thead>
+    <tbody>
+    {% for f in r.findings %}
+      <tr>
+        <td class="st st-{{ f.status.value }}">{{ t['status_' ~ f.status.value] }}</td>
+        <td>{{ t['sev_' ~ f.severity.value] }}</td>
+        <td><b>{{ f.rule_id }}</b><br>{{ f.rule_title }}</td>
+        <td>{{ f.asset_id }}</td>
+        <td>
+          {{ f.message }}
+          {% if f.status.value in ('fail', 'manual') %}
+          <details><summary>{{ t.remediation }}</summary>
+            <p>{{ f.remediation }}</p>
+            <ul>{% for s in f.sources %}<li>{{ s.framework }} — {{ s.reference }}{% if not s.verified %}{{ t.unverified }}{% endif %}</li>{% endfor %}</ul>
+          </details>
+          {% endif %}
+          {% if f.evidence %}<details><summary>{{ t.evidence }}</summary><pre>{{ f.evidence | tojson_pretty }}</pre></details>{% endif %}
+        </td>
+      </tr>
+    {% endfor %}
+    </tbody>
+  </table>
+  </div>
+</main>
+</body>
+</html>

agenteval/schema.py

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 The AgentEval Authors
+"""JSON Schema validation for AgentEval's YAML inputs.
+
+Schemas live in ``agenteval/schemas/*.schema.json`` (Draft 2020-12) and are
+exported for editors with ``agenteval schemas --out schemas/``.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import date, datetime
+from functools import lru_cache
+from importlib import resources
+from typing import Any
+
+from jsonschema import Draft202012Validator
+
+SCHEMA_NAMES = ("rule", "pack", "translations", "attestations", "exam", "system", "policy")
+
+
+@lru_cache(maxsize=None)
+def schema(name: str) -> dict:
+    text = resources.files("agenteval").joinpath(f"schemas/{name}.schema.json").read_text(encoding="utf-8")
+    return json.loads(text)
+
+
+@lru_cache(maxsize=None)
+def validator(name: str) -> Draft202012Validator:
+    """``rule`` validates one rule; ``rule-file`` validates a whole rules file."""
+    if name == "rule":
+        s = schema("rule")
+        return Draft202012Validator({"$schema": s["$schema"], "$defs": s["$defs"], "$ref": "#/$defs/rule"})
+    return Draft202012Validator(schema("rule" if name == "rule-file" else name))
+
+
+def _plain(v: Any) -> Any:
+    """YAML turns unquoted dates into date objects; validate them as ISO strings."""
+    if isinstance(v, (date, datetime)):
+        return v.isoformat()
+    if isinstance(v, dict):
+        return {k: _plain(x) for k, x in v.items()}
+    if isinstance(v, list):
+        return [_plain(x) for x in v]
+    return v
+
+
+def errors(name: str, data: Any, where: str = "") -> list[str]:
+    """Return human-readable validation errors (empty list when valid)."""
+    out = []
+    for e in sorted(validator(name).iter_errors(_plain(data)), key=lambda e: list(e.path)):
+        path = "/".join(str(p) for p in e.path) or "<root>"
+        out.append(f"{where} {path}: {e.message}".strip())
+    return out
+
+
+class SchemaError(ValueError):
+    def __init__(self, errs: list[str]):
+        self.errors = errs
+        super().__init__("Schema validation failed:\n  " + "\n  ".join(errs))
+
+
+def check(name: str, data: Any, where: str = "") -> None:
+    errs = errors(name, data, where)
+    if errs:
+        raise SchemaError(errs)

agenteval/schemas/attestations.schema.json

@@ -0,0 +1,76 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agenteval.dev/schemas/v1/attestations.schema.json",
+  "title": "AgentEval attestations file",
+  "oneOf": [
+    {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "attestations"
+      ],
+      "properties": {
+        "attestations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/attestation"
+          }
+        }
+      }
+    },
+    {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/attestation"
+      }
+    }
+  ],
+  "$defs": {
+    "attestation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "rule_id",
+        "attested_by",
+        "attested_at",
+        "evidence"
+      ],
+      "properties": {
+        "rule_id": {
+          "type": "string",
+          "pattern": "^[A-Z]{2,6}(-[A-Z0-9]{2,8})+$"
+        },
+        "asset_id": {
+          "type": "string",
+          "minLength": 1,
+          "default": "*",
+          "description": "Asset ID, or \"*\" for every asset the rule applies to."
+        },
+        "attested_by": {
+          "type": "string",
+          "minLength": 1
+        },
+        "attested_at": {
+          "type": "string",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}([T ].*)?$",
+          "description": "ISO date (YYYY-MM-DD). Used for max_age_days checks."
+        },
+        "evidence": {
+          "type": "object",
+          "additionalProperties": {
+            "type": [
+              "string",
+              "number",
+              "boolean",
+              "null"
+            ]
+          },
+          "description": "Keys must match the rule's attestation.evidence_required. Prefer links or ticket IDs over pasted content."
+        },
+        "notes": {
+          "type": "string"
+        }
+      }
+    }
+  }
+}

agenteval/schemas/exam.schema.json

@@ -0,0 +1,85 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agenteval.dev/schemas/v1/exam.schema.json",
+  "title": "agenteval exam paper",
+  "type": "object",
+  "required": ["id", "title", "domain", "version", "dimensions", "cases"],
+  "additionalProperties": false,
+  "properties": {
+    "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,63}$"},
+    "title": {"type": "string", "minLength": 3},
+    "domain": {"type": "string"},
+    "version": {"type": "string"},
+    "language": {"type": "string"},
+    "description": {"type": "string"},
+    "passing_score": {"type": "number", "minimum": 0, "maximum": 100},
+    "case_passing_score": {"type": "number", "minimum": 0, "maximum": 100},
+    "context": {"type": "string"},
+    "tools": {"type": "array", "items": {"type": "object", "required": ["name"]}},
+    "dimensions": {
+      "type": "array", "minItems": 1,
+      "items": {
+        "type": "object", "required": ["id", "name", "weight"], "additionalProperties": false,
+        "properties": {
+          "id": {"type": "string", "pattern": "^[a-z_]+$"},
+          "name": {"type": "string"},
+          "weight": {"type": "number", "exclusiveMinimum": 0},
+          "description": {"type": "string"}
+        }
+      }
+    },
+    "cases": {
+      "type": "array", "minItems": 1,
+      "items": {
+        "type": "object", "required": ["id", "title", "category", "turns"], "additionalProperties": false,
+        "properties": {
+          "id": {"type": "string", "pattern": "^[A-Z]{2,6}-[0-9]{3}$"},
+          "title": {"type": "string"},
+          "category": {"type": "string"},
+          "difficulty": {"enum": ["easy", "medium", "hard"]},
+          "weight": {"type": "number", "exclusiveMinimum": 0},
+          "tags": {"type": "array", "items": {"type": "string"}},
+          "tool_mocks": {"type": "object"},
+          "turns": {
+            "type": "array", "minItems": 1,
+            "items": {
+              "type": "object", "required": ["user"], "additionalProperties": false,
+              "properties": {
+                "user": {"type": "string", "minLength": 1},
+                "expect": {"type": "array", "items": {"$ref": "#/$defs/assertion"}}
+              }
+            }
+          },
+          "rubric": {
+            "type": "array",
+            "items": {
+              "type": "object", "required": ["id", "dimension", "criterion"], "additionalProperties": false,
+              "properties": {
+                "id": {"type": "string"},
+                "dimension": {"type": "string"},
+                "criterion": {"type": "string", "minLength": 5},
+                "weight": {"type": "number", "exclusiveMinimum": 0},
+                "signals": {"type": "array", "items": {"type": "string"}}
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "assertion": {
+      "type": "object", "required": ["type", "dimension"], "additionalProperties": false,
+      "properties": {
+        "type": {"enum": ["contains", "contains_any", "contains_all", "not_contains", "regex", "not_regex",
+                          "tool_called", "tool_not_called", "max_latency_ms", "min_chars", "max_chars"]},
+        "value": {},
+        "dimension": {"type": "string"},
+        "weight": {"type": "number", "exclusiveMinimum": 0},
+        "critical": {"type": "boolean"},
+        "args_contains": {"type": "object"},
+        "description": {"type": "string"}
+      }
+    }
+  }
+}

agenteval/schemas/pack.schema.json

@@ -0,0 +1,48 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agenteval.dev/schemas/v1/pack.schema.json",
+  "title": "AgentEval rule pack metadata (pack.yaml)",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "id",
+    "name",
+    "tier",
+    "jurisdiction"
+  ],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9-]{1,62}$",
+      "description": "Stable pack ID referenced by systems' rule_packs."
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Display name (English, the source language)."
+    },
+    "name_zh": {
+      "type": "string",
+      "description": "Optional Simplified Chinese display name."
+    },
+    "tier": {
+      "enum": [
+        "national",
+        "regional",
+        "industry"
+      ],
+      "description": "Where the obligations come from."
+    },
+    "jurisdiction": {
+      "type": "string",
+      "minLength": 1,
+      "description": "e.g. US, US-CA, or an industry code such as PCI-DSS."
+    },
+    "description": {
+      "type": "string"
+    },
+    "description_zh": {
+      "type": "string"
+    }
+  }
+}

agenteval/schemas/policy.schema.json

@@ -0,0 +1,67 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agenteval.dev/schemas/v1/policy.schema.json",
+  "title": "AgentEval certification policy (certification/policy.yaml)",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "min_security_score": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 100,
+      "description": "Security score must be strictly greater than this."
+    },
+    "min_quality_score": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 100,
+      "description": "Evaluation score must be strictly greater than this."
+    },
+    "block_on_severities": {
+      "type": "array",
+      "uniqueItems": true,
+      "items": {
+        "enum": [
+          "critical",
+          "high",
+          "medium",
+          "low",
+          "info"
+        ]
+      }
+    },
+    "require_no_pending_attestations": {
+      "type": "boolean"
+    },
+    "require_zero_critical_eval_failures": {
+      "type": "boolean"
+    },
+    "require_no_eval_errors": {
+      "type": "boolean"
+    },
+    "require_llm_judge": {
+      "type": "boolean"
+    },
+    "allow_demo_certificates": {
+      "type": "boolean"
+    },
+    "max_evidence_age_days": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "validity_months": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 36
+    },
+    "auto_revoke": {
+      "type": "boolean"
+    },
+    "issuer_name": {
+      "type": "string"
+    },
+    "issuer_name_zh": {
+      "type": "string"
+    }
+  }
+}