armor-agent-plugin 6.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of armor-agent-plugin might be problematic. Click here for more details.
- armor/__init__.py +9 -0
- armor/__main__.py +3 -0
- armor/cli.py +209 -0
- armor/data/ai_standards/_catalog/latest.json +7 -0
- armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_catalog.json +1517 -0
- armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_catalog.md +345 -0
- armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.csv +54 -0
- armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.json +1446 -0
- armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.md +57 -0
- armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/manifest.json +29 -0
- armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/source_coverage.json +360 -0
- armor/data/ai_standards/eu-ai-act/history.jsonl +1 -0
- armor/data/ai_standards/eu-ai-act/latest.json +7 -0
- armor/data/ai_standards/eu-ai-act/versions/2026-06-04__e1a1c700/manifest.json +5 -0
- armor/data/ai_standards/mitre-atlas/history.jsonl +1 -0
- armor/data/ai_standards/mitre-atlas/latest.json +7 -0
- armor/data/ai_standards/mitre-atlas/versions/2026-06-04__32346fb3/manifest.json +5 -0
- armor/data/ai_standards/nist-ai-rmf/history.jsonl +1 -0
- armor/data/ai_standards/nist-ai-rmf/latest.json +7 -0
- armor/data/ai_standards/nist-ai-rmf/versions/2026-06-04__b157a100/manifest.json +5 -0
- armor/data/ai_standards/owasp-llm-top10/history.jsonl +1 -0
- armor/data/ai_standards/owasp-llm-top10/latest.json +7 -0
- armor/data/ai_standards/owasp-llm-top10/versions/2026-06-04__0a592025/manifest.json +5 -0
- armor/file_lock.py +114 -0
- armor/pipeline/__init__.py +17 -0
- armor/pipeline/adversary.py +140 -0
- armor/pipeline/conform.py +259 -0
- armor/pipeline/conform_audit.py +192 -0
- armor/pipeline/load.py +94 -0
- armor/pipeline/report.py +357 -0
- armor/pipeline/schema.py +113 -0
- armor/pipeline/schemas/posture_report.schema.json +400 -0
- armor/pipeline/state.py +182 -0
- armor/pipeline/triage.py +225 -0
- armor/pipeline/triage_audit.py +153 -0
- armor_agent_plugin-6.0.0.dist-info/METADATA +574 -0
- armor_agent_plugin-6.0.0.dist-info/RECORD +50 -0
- armor_agent_plugin-6.0.0.dist-info/WHEEL +5 -0
- armor_agent_plugin-6.0.0.dist-info/entry_points.txt +2 -0
- armor_agent_plugin-6.0.0.dist-info/licenses/LICENSE +21 -0
- armor_agent_plugin-6.0.0.dist-info/top_level.txt +2 -0
- catalog/__init__.py +1 -0
- catalog/ai_standards_versioning.py +114 -0
- catalog/download_aiid_slim.py +90 -0
- catalog/download_eu_ai_act_cellar.py +86 -0
- catalog/download_mitre_atlas.py +78 -0
- catalog/download_nist_ai_rmf.py +77 -0
- catalog/download_owasp_llm_top10.py +71 -0
- catalog/generate_guardrail_catalog.py +990 -0
- catalog/refresh_catalog.py +77 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Shared red-team adversary request/delta helpers for Phases 3 and 5."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .state import MAX_AUDIT_REVISIONS, RunDir
|
|
10
|
+
|
|
11
|
+
VERDICT_CONFIRM = "CONFIRM"
|
|
12
|
+
VERDICT_REVISE = "REVISE"
|
|
13
|
+
VERDICT_INVESTIGATE = "INVESTIGATE"
|
|
14
|
+
|
|
15
|
+
ALLOWED_VERDICTS = {VERDICT_CONFIRM, VERDICT_REVISE, VERDICT_INVESTIGATE}
|
|
16
|
+
|
|
17
|
+
AUDIT_RESPONSE_SCHEMA: dict[str, Any] = {
|
|
18
|
+
"type": "object",
|
|
19
|
+
"required": ["verdicts"],
|
|
20
|
+
"properties": {
|
|
21
|
+
"verdicts": {
|
|
22
|
+
"type": "array",
|
|
23
|
+
"items": {
|
|
24
|
+
"type": "object",
|
|
25
|
+
"required": ["sid", "verdict"],
|
|
26
|
+
"properties": {
|
|
27
|
+
"sid": {"type": "string"},
|
|
28
|
+
"verdict": {"type": "string", "enum": sorted(ALLOWED_VERDICTS)},
|
|
29
|
+
"critique": {"type": "string"},
|
|
30
|
+
"revised_row": {"type": "object"},
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
"overall_note": {"type": "string"},
|
|
35
|
+
},
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
_ADVERSARY_INSTRUCTIONS = (
|
|
39
|
+
"You are an independent red-team adversary auditing the target_artifact rows. "
|
|
40
|
+
"For each row, assign verdict in {'CONFIRM', 'REVISE', 'INVESTIGATE'}. "
|
|
41
|
+
"If 'REVISE', provide a critique and a revised_row of the exact same shape as the target row. "
|
|
42
|
+
"You may NOT invent new sids or change scope. "
|
|
43
|
+
"You may NOT read source files beyond the file:line citations already present in the target rows. "
|
|
44
|
+
"Output ONLY strict JSON conforming to response_schema (no Markdown fences, no prose)."
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def build_request(
|
|
49
|
+
audit_kind: str,
|
|
50
|
+
target_artifact: dict[str, Any],
|
|
51
|
+
rows_key: str,
|
|
52
|
+
revision: int,
|
|
53
|
+
extra_context: dict[str, Any] | None = None,
|
|
54
|
+
) -> dict[str, Any]:
|
|
55
|
+
return {
|
|
56
|
+
"phase": audit_kind,
|
|
57
|
+
"revision": revision,
|
|
58
|
+
"max_revisions": MAX_AUDIT_REVISIONS,
|
|
59
|
+
"instructions": _ADVERSARY_INSTRUCTIONS,
|
|
60
|
+
"target_rows_key": rows_key,
|
|
61
|
+
"target_artifact": target_artifact,
|
|
62
|
+
"extra_context": extra_context or {},
|
|
63
|
+
"response_schema": AUDIT_RESPONSE_SCHEMA,
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def write_request(run: RunDir, phase: str, payload: dict[str, Any]) -> Path:
|
|
68
|
+
req_path = run.request(phase)
|
|
69
|
+
req_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
|
70
|
+
return req_path
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def read_response(run: RunDir, phase: str) -> dict[str, Any]:
|
|
74
|
+
resp_path = run.response(phase)
|
|
75
|
+
if not resp_path.is_file():
|
|
76
|
+
raise SystemExit(f"{phase} response not found at {resp_path}")
|
|
77
|
+
try:
|
|
78
|
+
return json.loads(resp_path.read_text(encoding="utf-8"))
|
|
79
|
+
except json.JSONDecodeError as exc:
|
|
80
|
+
raise SystemExit(f"{phase} response invalid JSON: {exc}") from exc
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def apply_deltas(
|
|
84
|
+
target_artifact: dict[str, Any],
|
|
85
|
+
rows_key: str,
|
|
86
|
+
audit_response: dict[str, Any],
|
|
87
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
88
|
+
original_rows: list[dict[str, Any]] = list(target_artifact.get(rows_key, []))
|
|
89
|
+
by_sid: dict[str, dict[str, Any]] = {str(r["sid"]): dict(r) for r in original_rows}
|
|
90
|
+
order: list[str] = [str(r["sid"]) for r in original_rows]
|
|
91
|
+
|
|
92
|
+
counts: dict[str, int] = {v: 0 for v in sorted(ALLOWED_VERDICTS)}
|
|
93
|
+
revisions_applied: list[dict[str, Any]] = []
|
|
94
|
+
open_questions: list[dict[str, Any]] = []
|
|
95
|
+
unknown_sids: list[str] = []
|
|
96
|
+
|
|
97
|
+
for item in audit_response.get("verdicts", []):
|
|
98
|
+
if not isinstance(item, dict):
|
|
99
|
+
continue
|
|
100
|
+
verdict = item.get("verdict")
|
|
101
|
+
if verdict not in ALLOWED_VERDICTS:
|
|
102
|
+
continue
|
|
103
|
+
counts[verdict] += 1
|
|
104
|
+
sid = str(item.get("sid") or "")
|
|
105
|
+
if sid not in by_sid:
|
|
106
|
+
unknown_sids.append(sid)
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
critique = str(item.get("critique") or "")
|
|
110
|
+
if verdict == VERDICT_REVISE:
|
|
111
|
+
revised = item.get("revised_row")
|
|
112
|
+
if isinstance(revised, dict):
|
|
113
|
+
old_row = dict(by_sid[sid])
|
|
114
|
+
new_row = {**old_row, **revised, "sid": sid}
|
|
115
|
+
by_sid[sid] = new_row
|
|
116
|
+
revisions_applied.append(
|
|
117
|
+
{
|
|
118
|
+
"sid": sid,
|
|
119
|
+
"critique": critique,
|
|
120
|
+
"before": old_row,
|
|
121
|
+
"after": new_row,
|
|
122
|
+
}
|
|
123
|
+
)
|
|
124
|
+
elif verdict == VERDICT_INVESTIGATE:
|
|
125
|
+
open_questions.append({"sid": sid, "critique": critique})
|
|
126
|
+
|
|
127
|
+
new_rows = [by_sid[sid] for sid in order]
|
|
128
|
+
new_artifact = {**target_artifact, rows_key: new_rows}
|
|
129
|
+
summary = {
|
|
130
|
+
"verdict_counts": counts,
|
|
131
|
+
"revisions_applied": revisions_applied,
|
|
132
|
+
"open_questions": open_questions,
|
|
133
|
+
"unknown_sids": unknown_sids,
|
|
134
|
+
"overall_note": str(audit_response.get("overall_note") or ""),
|
|
135
|
+
}
|
|
136
|
+
return new_artifact, summary
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def converged(summary: dict[str, Any], revision: int) -> bool:
|
|
140
|
+
return revision >= MAX_AUDIT_REVISIONS or len(summary.get("revisions_applied", [])) == 0
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Phase 4: Guardrail conformance assessment and citation cross-verification (LLM exchange)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from .state import RunDir, hallucination_log, require_phase_done, verify_citation
|
|
11
|
+
|
|
12
|
+
ALLOWED_VERDICTS = {"implemented", "partial", "weak", "gap", "inconclusive", "n/a"}
|
|
13
|
+
NON_GAP = {"implemented", "partial", "weak"}
|
|
14
|
+
|
|
15
|
+
RESPONSE_SCHEMA: dict[str, Any] = {
|
|
16
|
+
"type": "object",
|
|
17
|
+
"required": ["rows"],
|
|
18
|
+
"properties": {
|
|
19
|
+
"rows": {
|
|
20
|
+
"type": "array",
|
|
21
|
+
"items": {
|
|
22
|
+
"type": "object",
|
|
23
|
+
"required": ["sid", "verdict", "reasoning"],
|
|
24
|
+
"properties": {
|
|
25
|
+
"sid": {"type": "string"},
|
|
26
|
+
"verdict": {"type": "string", "enum": sorted(ALLOWED_VERDICTS)},
|
|
27
|
+
"reasoning": {"type": "string", "minLength": 10},
|
|
28
|
+
"citations": {"type": "array", "items": {"type": "string"}},
|
|
29
|
+
"gap_summary": {"type": "string"},
|
|
30
|
+
"remediation": {"type": "string"},
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
_INSTRUCTIONS = """Assess the codebase under project_root against each in-scope guardrail in in_scope.
|
|
38
|
+
|
|
39
|
+
Requirements:
|
|
40
|
+
1. Assign a verdict from {'implemented', 'partial', 'weak', 'gap', 'inconclusive', 'n/a'}.
|
|
41
|
+
2. Verdicts in {'implemented', 'partial', 'weak'} MUST carry >= 1 citation in the format
|
|
42
|
+
relative/path.ext:line or relative/path.ext:start-end. The Python runner cross-verifies every
|
|
43
|
+
citation against project_root; invalid or out-of-bounds citations auto-demote the row to
|
|
44
|
+
'inconclusive' and are logged to hallucination_log.jsonl.
|
|
45
|
+
3. 'remediation' is MANDATORY for verdicts in {'gap', 'weak', 'partial', 'inconclusive'} and must be:
|
|
46
|
+
(1) codebase-specific (name real files/dirs; give exact target paths for new artefacts),
|
|
47
|
+
(2) action-leading (start with an imperative verb; not "Implement the guardrail"),
|
|
48
|
+
(3) concrete (reference real entry points + exact data/config),
|
|
49
|
+
(4) anchored to the guardrail's verification_test,
|
|
50
|
+
(5) free of banned phrases ("Implement the guardrail", "Add CI enforcement + persistent audit trail",
|
|
51
|
+
"as appropriate", "follow best practices").
|
|
52
|
+
4. For verdict == 'gap', also populate 'gap_summary'.
|
|
53
|
+
5. Cover every input sid in in_scope exactly once.
|
|
54
|
+
6. Output ONLY strict JSON conforming to response_schema (no Markdown fences, no prose)."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _verify(project: Path, citations: Any) -> tuple[list[str], list[dict[str, str]]]:
|
|
58
|
+
verified: list[str] = []
|
|
59
|
+
rejected: list[dict[str, str]] = []
|
|
60
|
+
if not isinstance(citations, list):
|
|
61
|
+
return verified, rejected
|
|
62
|
+
for raw_cit in citations:
|
|
63
|
+
if not isinstance(raw_cit, str) or not raw_cit.strip():
|
|
64
|
+
rejected.append({"citation": str(raw_cit), "reason": "non-string or empty citation"})
|
|
65
|
+
continue
|
|
66
|
+
cit = raw_cit.strip()
|
|
67
|
+
ok, reason = verify_citation(project, cit)
|
|
68
|
+
if ok:
|
|
69
|
+
verified.append(cit)
|
|
70
|
+
else:
|
|
71
|
+
rejected.append({"citation": cit, "reason": reason})
|
|
72
|
+
return verified, rejected
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _append_hallucinations(project: Path, phase: str, sid: str, rejected: list[dict[str, str]]) -> None:
|
|
76
|
+
if not rejected:
|
|
77
|
+
return
|
|
78
|
+
log_path = hallucination_log(project)
|
|
79
|
+
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
|
|
80
|
+
with open(log_path, "a", encoding="utf-8") as f:
|
|
81
|
+
for item in rejected:
|
|
82
|
+
rec = {
|
|
83
|
+
"ts": ts,
|
|
84
|
+
"phase": phase,
|
|
85
|
+
"sid": sid,
|
|
86
|
+
"citation": item["citation"],
|
|
87
|
+
"reason": item["reason"],
|
|
88
|
+
}
|
|
89
|
+
f.write(json.dumps(rec) + "\n")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def prepare(project: Path, run: RunDir) -> Path:
|
|
93
|
+
loaded = require_phase_done(run, "load")
|
|
94
|
+
triage_audit = require_phase_done(run, "triage-audit")
|
|
95
|
+
|
|
96
|
+
catalog_by_sid = {row["sid"]: row for row in loaded["guardrails"]}
|
|
97
|
+
in_scope: list[dict[str, Any]] = []
|
|
98
|
+
for row in triage_audit.get("applicability", []):
|
|
99
|
+
if row.get("status") in ("applies", "partial_scope"):
|
|
100
|
+
sid = row["sid"]
|
|
101
|
+
cat = catalog_by_sid.get(sid, {})
|
|
102
|
+
in_scope.append(
|
|
103
|
+
{
|
|
104
|
+
"sid": sid,
|
|
105
|
+
"status": row["status"],
|
|
106
|
+
"reason": row.get("reason", ""),
|
|
107
|
+
"scope_notes": row.get("scope_notes", ""),
|
|
108
|
+
"family": cat.get("family", ""),
|
|
109
|
+
"family_name": cat.get("family_name", ""),
|
|
110
|
+
"description": cat.get("description", ""),
|
|
111
|
+
"verification_test": cat.get("verification_test", ""),
|
|
112
|
+
"implementation_hints": cat.get("implementation_hints", ""),
|
|
113
|
+
}
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
payload = {
|
|
117
|
+
"phase": "conform",
|
|
118
|
+
"project_root": str(project.resolve()),
|
|
119
|
+
"instructions": _INSTRUCTIONS,
|
|
120
|
+
"system_profile": triage_audit.get("system_profile", {}),
|
|
121
|
+
"in_scope": in_scope,
|
|
122
|
+
"response_schema": RESPONSE_SCHEMA,
|
|
123
|
+
}
|
|
124
|
+
req_path = run.request("conform")
|
|
125
|
+
req_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
|
126
|
+
return req_path
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _render_md(artifact: dict[str, Any]) -> str:
|
|
130
|
+
lines: list[str] = [
|
|
131
|
+
"# Phase 4 — Conformance",
|
|
132
|
+
"",
|
|
133
|
+
f"- **Rejected citations (hallucinations)**: {artifact.get('hallucination_count', 0)}",
|
|
134
|
+
"",
|
|
135
|
+
"## Verdict Summary",
|
|
136
|
+
"",
|
|
137
|
+
]
|
|
138
|
+
by_verdict = artifact.get("by_verdict", {})
|
|
139
|
+
for v in sorted(by_verdict):
|
|
140
|
+
lines.append(f"- **{v}**: {by_verdict[v]}")
|
|
141
|
+
lines.extend(
|
|
142
|
+
[
|
|
143
|
+
"",
|
|
144
|
+
"## Guardrails",
|
|
145
|
+
"",
|
|
146
|
+
"| SID | Verdict | Citations | Reasoning |",
|
|
147
|
+
"| --- | --- | --- | --- |",
|
|
148
|
+
]
|
|
149
|
+
)
|
|
150
|
+
for row in artifact.get("rows", []):
|
|
151
|
+
v = str(row.get("verdict", ""))
|
|
152
|
+
if row.get("demoted_due_to_hallucination"):
|
|
153
|
+
v = f"{v} *(demoted)*"
|
|
154
|
+
cits = ", ".join(row.get("citations", []))
|
|
155
|
+
reasoning = str(row.get("reasoning", "")).replace("|", "\\|").replace("\n", " ")
|
|
156
|
+
lines.append(f"| {row.get('sid', '')} | {v} | {cits} | {reasoning} |")
|
|
157
|
+
lines.append("")
|
|
158
|
+
return "\n".join(lines)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def finalize(project: Path, run: RunDir) -> dict[str, Any]:
|
|
162
|
+
require_phase_done(run, "load")
|
|
163
|
+
triage_audit = require_phase_done(run, "triage-audit")
|
|
164
|
+
|
|
165
|
+
resp_path = run.response("conform")
|
|
166
|
+
if not resp_path.is_file():
|
|
167
|
+
raise SystemExit(f"conform response not found at {resp_path}")
|
|
168
|
+
try:
|
|
169
|
+
resp = json.loads(resp_path.read_text(encoding="utf-8"))
|
|
170
|
+
except json.JSONDecodeError as exc:
|
|
171
|
+
raise SystemExit(f"conform response invalid JSON: {exc}") from exc
|
|
172
|
+
|
|
173
|
+
expected_sids = {
|
|
174
|
+
row["sid"]
|
|
175
|
+
for row in triage_audit.get("applicability", [])
|
|
176
|
+
if row.get("status") in ("applies", "partial_scope")
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
raw_rows = resp.get("rows")
|
|
180
|
+
if not isinstance(raw_rows, list):
|
|
181
|
+
raise SystemExit("conform response invalid:\n - 'rows' must be a list")
|
|
182
|
+
|
|
183
|
+
seen: set[str] = set()
|
|
184
|
+
errors: list[str] = []
|
|
185
|
+
cleaned_rows: list[dict[str, Any]] = []
|
|
186
|
+
hallucination_count = 0
|
|
187
|
+
|
|
188
|
+
for idx, row in enumerate(raw_rows):
|
|
189
|
+
if not isinstance(row, dict):
|
|
190
|
+
errors.append(f"row {idx}: not an object")
|
|
191
|
+
continue
|
|
192
|
+
sid = row.get("sid")
|
|
193
|
+
verdict = row.get("verdict")
|
|
194
|
+
reasoning = str(row.get("reasoning") or "").strip()
|
|
195
|
+
|
|
196
|
+
if sid not in expected_sids:
|
|
197
|
+
errors.append(f"{sid}: out-of-scope or unknown sid")
|
|
198
|
+
continue
|
|
199
|
+
if sid in seen:
|
|
200
|
+
errors.append(f"{sid}: duplicate row")
|
|
201
|
+
continue
|
|
202
|
+
seen.add(sid)
|
|
203
|
+
|
|
204
|
+
if verdict not in ALLOWED_VERDICTS:
|
|
205
|
+
errors.append(f"{sid}: bad verdict {verdict!r} (allowed: {sorted(ALLOWED_VERDICTS)})")
|
|
206
|
+
continue
|
|
207
|
+
if not reasoning:
|
|
208
|
+
errors.append(f"{sid}: empty reasoning")
|
|
209
|
+
continue
|
|
210
|
+
|
|
211
|
+
verified, rejected = _verify(project, row.get("citations") or [])
|
|
212
|
+
if rejected:
|
|
213
|
+
hallucination_count += len(rejected)
|
|
214
|
+
_append_hallucinations(project, "conform", sid, rejected)
|
|
215
|
+
|
|
216
|
+
original_verdict = verdict
|
|
217
|
+
demoted = False
|
|
218
|
+
if verdict in NON_GAP and not verified:
|
|
219
|
+
verdict = "inconclusive"
|
|
220
|
+
demoted = True
|
|
221
|
+
|
|
222
|
+
cleaned_rows.append(
|
|
223
|
+
{
|
|
224
|
+
"sid": sid,
|
|
225
|
+
"verdict": verdict,
|
|
226
|
+
"original_verdict": original_verdict,
|
|
227
|
+
"demoted_due_to_hallucination": demoted,
|
|
228
|
+
"reasoning": reasoning,
|
|
229
|
+
"citations": verified,
|
|
230
|
+
"rejected_citations": rejected,
|
|
231
|
+
"gap_summary": str(row.get("gap_summary") or ""),
|
|
232
|
+
"remediation": str(row.get("remediation") or ""),
|
|
233
|
+
}
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
missing = sorted(expected_sids - seen)
|
|
237
|
+
for sid in missing:
|
|
238
|
+
errors.append(f"{sid}: missing from response")
|
|
239
|
+
|
|
240
|
+
if errors:
|
|
241
|
+
msg = "\n - ".join(errors[:20])
|
|
242
|
+
raise SystemExit(f"conform response invalid:\n - {msg}")
|
|
243
|
+
|
|
244
|
+
by_verdict: dict[str, int] = {}
|
|
245
|
+
for r in cleaned_rows:
|
|
246
|
+
v = r["verdict"]
|
|
247
|
+
by_verdict[v] = by_verdict.get(v, 0) + 1
|
|
248
|
+
|
|
249
|
+
artifact: dict[str, Any] = {
|
|
250
|
+
"phase": "conform",
|
|
251
|
+
"by_verdict": by_verdict,
|
|
252
|
+
"hallucination_count": hallucination_count,
|
|
253
|
+
"rows": cleaned_rows,
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
art_path = run.artifact("conform")
|
|
257
|
+
art_path.write_text(json.dumps(artifact, indent=2) + "\n", encoding="utf-8")
|
|
258
|
+
(run.root / "04_conformance.md").write_text(_render_md(artifact), encoding="utf-8")
|
|
259
|
+
return artifact
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Phase 5: Adversarial red-team audit of conformance rows + citation re-verification (LLM loop <= 3 rounds)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from . import adversary
|
|
10
|
+
from .conform import ALLOWED_VERDICTS, NON_GAP, _append_hallucinations, _verify
|
|
11
|
+
from .state import MAX_AUDIT_REVISIONS, RunDir, require_phase_done
|
|
12
|
+
|
|
13
|
+
PHASE = "conform-audit"
|
|
14
|
+
ROWS_KEY = "rows"
|
|
15
|
+
|
|
16
|
+
_AUDIT_DIRECTIONS = (
|
|
17
|
+
"(a) DOWNGRADE rows whose evidence is weak or fabricated; "
|
|
18
|
+
"(b) UPGRADE rows you can defend with additional citations."
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _trail_path(run: RunDir) -> Path:
|
|
23
|
+
return run.root / ".conform_audit_trail.json"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _load_trail(run: RunDir) -> list[dict[str, Any]]:
|
|
27
|
+
p = _trail_path(run)
|
|
28
|
+
if p.is_file():
|
|
29
|
+
try:
|
|
30
|
+
return json.loads(p.read_text(encoding="utf-8"))
|
|
31
|
+
except json.JSONDecodeError:
|
|
32
|
+
return []
|
|
33
|
+
return []
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _save_trail(run: RunDir, trail: list[dict[str, Any]]) -> None:
|
|
37
|
+
_trail_path(run).write_text(json.dumps(trail, indent=2) + "\n", encoding="utf-8")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _extra_context(previous_revisions: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
|
41
|
+
ctx: dict[str, Any] = {
|
|
42
|
+
"allowed_verdicts": sorted(ALLOWED_VERDICTS),
|
|
43
|
+
"non_gap_verdicts_require_citation": sorted(NON_GAP),
|
|
44
|
+
"audit_directions": _AUDIT_DIRECTIONS,
|
|
45
|
+
}
|
|
46
|
+
if previous_revisions is not None:
|
|
47
|
+
ctx["previous_revisions"] = previous_revisions
|
|
48
|
+
return ctx
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def prepare(project: Path, run: RunDir) -> Path:
|
|
52
|
+
req_path = run.request(PHASE)
|
|
53
|
+
if req_path.is_file():
|
|
54
|
+
return req_path
|
|
55
|
+
conform_art = require_phase_done(run, "conform")
|
|
56
|
+
payload = adversary.build_request(
|
|
57
|
+
audit_kind=PHASE,
|
|
58
|
+
target_artifact=conform_art,
|
|
59
|
+
rows_key=ROWS_KEY,
|
|
60
|
+
revision=1,
|
|
61
|
+
extra_context=_extra_context(),
|
|
62
|
+
)
|
|
63
|
+
return adversary.write_request(run, PHASE, payload)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _render_md(artifact: dict[str, Any]) -> str:
|
|
67
|
+
lines: list[str] = [
|
|
68
|
+
"# Phase 5 — Conformance Audit",
|
|
69
|
+
"",
|
|
70
|
+
f"- **Final revision**: {artifact.get('final_revision')}/{MAX_AUDIT_REVISIONS}",
|
|
71
|
+
f"- **Converged**: {artifact.get('converged')}",
|
|
72
|
+
"",
|
|
73
|
+
"## Verdict Counts",
|
|
74
|
+
"",
|
|
75
|
+
]
|
|
76
|
+
by_verdict = artifact.get("by_verdict", {})
|
|
77
|
+
for v in sorted(by_verdict):
|
|
78
|
+
lines.append(f"- **{v}**: {by_verdict[v]}")
|
|
79
|
+
|
|
80
|
+
open_q = artifact.get("open_questions", [])
|
|
81
|
+
lines.extend(["", f"## Open Questions ({len(open_q)})", ""])
|
|
82
|
+
if open_q:
|
|
83
|
+
for q in open_q:
|
|
84
|
+
lines.append(f"- **{q.get('sid', '')}**: {q.get('critique', '')}")
|
|
85
|
+
else:
|
|
86
|
+
lines.append("_None._")
|
|
87
|
+
|
|
88
|
+
lines.extend(["", "## Audit Trail", ""])
|
|
89
|
+
for rec in artifact.get("audit_trail", []):
|
|
90
|
+
rev = rec.get("revision")
|
|
91
|
+
vc = rec.get("verdict_counts", {})
|
|
92
|
+
revs = len(rec.get("revisions_applied", []))
|
|
93
|
+
hal = rec.get("post_audit_hallucinations", 0)
|
|
94
|
+
lines.append(
|
|
95
|
+
f"- **Round {rev}**: verdicts={vc}, revisions_applied={revs}, post_audit_hallucinations={hal}"
|
|
96
|
+
)
|
|
97
|
+
lines.append("")
|
|
98
|
+
return "\n".join(lines)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def finalize(project: Path, run: RunDir) -> dict[str, Any]:
|
|
102
|
+
req_path = run.request(PHASE)
|
|
103
|
+
if not req_path.is_file():
|
|
104
|
+
raise SystemExit(f"{PHASE} request not found at {req_path}")
|
|
105
|
+
req = json.loads(req_path.read_text(encoding="utf-8"))
|
|
106
|
+
resp = adversary.read_response(run, PHASE)
|
|
107
|
+
|
|
108
|
+
revision = int(req.get("revision", 1))
|
|
109
|
+
target = req["target_artifact"]
|
|
110
|
+
new_target, summary = adversary.apply_deltas(target, ROWS_KEY, resp)
|
|
111
|
+
|
|
112
|
+
post_audit_hallucinations = 0
|
|
113
|
+
reverified_rows: list[dict[str, Any]] = []
|
|
114
|
+
for row in new_target.get(ROWS_KEY, []):
|
|
115
|
+
r = dict(row)
|
|
116
|
+
sid = str(r.get("sid", ""))
|
|
117
|
+
verdict = str(r.get("verdict", "inconclusive"))
|
|
118
|
+
raw_cits = r.get("citations") or []
|
|
119
|
+
|
|
120
|
+
if not raw_cits and verdict in NON_GAP:
|
|
121
|
+
r["verdict"] = "inconclusive"
|
|
122
|
+
r["demoted_due_to_hallucination"] = True
|
|
123
|
+
r["citations"] = []
|
|
124
|
+
else:
|
|
125
|
+
verified, rejected = _verify(project, raw_cits)
|
|
126
|
+
r["citations"] = verified
|
|
127
|
+
existing_rej = list(r.get("rejected_citations") or [])
|
|
128
|
+
if rejected:
|
|
129
|
+
post_audit_hallucinations += len(rejected)
|
|
130
|
+
_append_hallucinations(project, PHASE, sid, rejected)
|
|
131
|
+
r["rejected_citations"] = existing_rej + rejected
|
|
132
|
+
if verdict in NON_GAP and not verified:
|
|
133
|
+
r["verdict"] = "inconclusive"
|
|
134
|
+
r["demoted_due_to_hallucination"] = True
|
|
135
|
+
|
|
136
|
+
reverified_rows.append(r)
|
|
137
|
+
|
|
138
|
+
new_target = {**new_target, ROWS_KEY: reverified_rows}
|
|
139
|
+
|
|
140
|
+
audit_record: dict[str, Any] = {
|
|
141
|
+
"revision": revision,
|
|
142
|
+
"verdict_counts": summary["verdict_counts"],
|
|
143
|
+
"revisions_applied": summary["revisions_applied"],
|
|
144
|
+
"open_questions": summary["open_questions"],
|
|
145
|
+
"unknown_sids": summary["unknown_sids"],
|
|
146
|
+
"overall_note": summary["overall_note"],
|
|
147
|
+
"post_audit_hallucinations": post_audit_hallucinations,
|
|
148
|
+
}
|
|
149
|
+
trail = _load_trail(run)
|
|
150
|
+
trail.append(audit_record)
|
|
151
|
+
|
|
152
|
+
if adversary.converged(summary, revision):
|
|
153
|
+
by_verdict: dict[str, int] = {}
|
|
154
|
+
for r in reverified_rows:
|
|
155
|
+
v = str(r.get("verdict", "inconclusive"))
|
|
156
|
+
by_verdict[v] = by_verdict.get(v, 0) + 1
|
|
157
|
+
|
|
158
|
+
all_open_questions: list[dict[str, Any]] = []
|
|
159
|
+
for r_rec in trail:
|
|
160
|
+
all_open_questions.extend(r_rec.get("open_questions", []))
|
|
161
|
+
|
|
162
|
+
is_converged = len(summary["revisions_applied"]) == 0
|
|
163
|
+
artifact: dict[str, Any] = {
|
|
164
|
+
"phase": PHASE,
|
|
165
|
+
"final_revision": revision,
|
|
166
|
+
"converged": is_converged,
|
|
167
|
+
"by_verdict": by_verdict,
|
|
168
|
+
"rows": reverified_rows,
|
|
169
|
+
"audit_trail": trail,
|
|
170
|
+
"open_questions": all_open_questions,
|
|
171
|
+
}
|
|
172
|
+
art_path = run.artifact(PHASE)
|
|
173
|
+
art_path.write_text(json.dumps(artifact, indent=2) + "\n", encoding="utf-8")
|
|
174
|
+
(run.root / "05_conformance_audit.md").write_text(_render_md(artifact), encoding="utf-8")
|
|
175
|
+
|
|
176
|
+
req_path.unlink(missing_ok=True)
|
|
177
|
+
run.response(PHASE).unlink(missing_ok=True)
|
|
178
|
+
_trail_path(run).unlink(missing_ok=True)
|
|
179
|
+
return artifact
|
|
180
|
+
|
|
181
|
+
next_rev = revision + 1
|
|
182
|
+
_save_trail(run, trail)
|
|
183
|
+
next_payload = adversary.build_request(
|
|
184
|
+
audit_kind=PHASE,
|
|
185
|
+
target_artifact=new_target,
|
|
186
|
+
rows_key=ROWS_KEY,
|
|
187
|
+
revision=next_rev,
|
|
188
|
+
extra_context=_extra_context(previous_revisions=trail),
|
|
189
|
+
)
|
|
190
|
+
adversary.write_request(run, PHASE, next_payload)
|
|
191
|
+
run.response(PHASE).unlink(missing_ok=True)
|
|
192
|
+
return {"status": "another-round-required", "next_revision": next_rev}
|