armor-agent-plugin 6.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of armor-agent-plugin might be problematic. Click here for more details.
- armor_agent_plugin-6.0.0/.claude-plugin/plugin.json +9 -0
- armor_agent_plugin-6.0.0/LICENSE +21 -0
- armor_agent_plugin-6.0.0/MANIFEST.in +6 -0
- armor_agent_plugin-6.0.0/PKG-INFO +574 -0
- armor_agent_plugin-6.0.0/README.md +537 -0
- armor_agent_plugin-6.0.0/agents/armor-agent.md +99 -0
- armor_agent_plugin-6.0.0/pyproject.toml +80 -0
- armor_agent_plugin-6.0.0/setup.cfg +4 -0
- armor_agent_plugin-6.0.0/setup.py +4 -0
- armor_agent_plugin-6.0.0/skills/armor/SKILL.md +175 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/__init__.py +0 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/conftest.py +8 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/__init__.py +0 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/_helpers.py +68 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/test_conform.py +156 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/test_conform_audit_and_report.py +273 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/test_load.py +42 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/test_schema.py +177 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/test_state.py +96 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/test_triage.py +103 -0
- armor_agent_plugin-6.0.0/skills/armor/assets/tests/pipeline/test_triage_audit.py +125 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/_catalog/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_catalog.json +1517 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_catalog.md +345 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.csv +54 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.json +1446 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.md +57 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/manifest.json +29 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/source_coverage.json +360 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/eu-ai-act/history.jsonl +1 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/eu-ai-act/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/eu-ai-act/versions/2026-06-04__e1a1c700/manifest.json +5 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/mitre-atlas/history.jsonl +1 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/mitre-atlas/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/mitre-atlas/versions/2026-06-04__32346fb3/manifest.json +5 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/nist-ai-rmf/history.jsonl +1 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/nist-ai-rmf/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/nist-ai-rmf/versions/2026-06-04__b157a100/manifest.json +5 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/owasp-llm-top10/history.jsonl +1 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/owasp-llm-top10/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/references/ai_standards/owasp-llm-top10/versions/2026-06-04__0a592025/manifest.json +5 -0
- armor_agent_plugin-6.0.0/skills/armor/references/architecture.md +32 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/__init__.py +9 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/__main__.py +3 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/cli.py +209 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/_catalog/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_catalog.json +1517 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_catalog.md +345 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.csv +54 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.json +1446 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/guardrail_matrix.md +57 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/manifest.json +29 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/_catalog/versions/2026-05-14__b1ed6db3/source_coverage.json +360 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/eu-ai-act/history.jsonl +1 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/eu-ai-act/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/eu-ai-act/versions/2026-06-04__e1a1c700/manifest.json +5 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/mitre-atlas/history.jsonl +1 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/mitre-atlas/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/mitre-atlas/versions/2026-06-04__32346fb3/manifest.json +5 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/nist-ai-rmf/history.jsonl +1 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/nist-ai-rmf/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/nist-ai-rmf/versions/2026-06-04__b157a100/manifest.json +5 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/owasp-llm-top10/history.jsonl +1 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/owasp-llm-top10/latest.json +7 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/data/ai_standards/owasp-llm-top10/versions/2026-06-04__0a592025/manifest.json +5 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/file_lock.py +114 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/__init__.py +17 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/adversary.py +140 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/conform.py +259 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/conform_audit.py +192 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/load.py +94 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/report.py +357 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/schema.py +113 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/schemas/posture_report.schema.json +400 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/state.py +182 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/triage.py +225 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor/pipeline/triage_audit.py +153 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor_agent_plugin.egg-info/PKG-INFO +574 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor_agent_plugin.egg-info/SOURCES.txt +91 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor_agent_plugin.egg-info/dependency_links.txt +1 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor_agent_plugin.egg-info/entry_points.txt +2 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor_agent_plugin.egg-info/requires.txt +10 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor_agent_plugin.egg-info/top_level.txt +2 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/armor_run.py +4 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/__init__.py +1 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/ai_standards_versioning.py +114 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/download_aiid_slim.py +90 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/download_eu_ai_act_cellar.py +86 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/download_mitre_atlas.py +78 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/download_nist_ai_rmf.py +77 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/download_owasp_llm_top10.py +71 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/generate_guardrail_catalog.py +990 -0
- armor_agent_plugin-6.0.0/skills/armor/scripts/catalog/refresh_catalog.py +77 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "armor-agent-plugin",
|
|
3
|
+
"description": "ARMOR v6 \u2014 LLM-driven AI guardrail posture pipeline. 6-phase pipeline (load \u2192 triage \u2192 triage-audit \u2192 conform \u2192 conform-audit \u2192 report) against a 53-row AI/agentic standards catalog with citation cross-verification and adversarial red-team loops.",
|
|
4
|
+
"version": "6.0.0",
|
|
5
|
+
"author": {
|
|
6
|
+
"name": "Aveek Sengupta, CIB Markets RegTech",
|
|
7
|
+
"email": "ibt_cp_rrpp_team_reflection@restricted.chase.com"
|
|
8
|
+
}
|
|
9
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Aveek Sengupta
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,574 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: armor-agent-plugin
|
|
3
|
+
Version: 6.0.0
|
|
4
|
+
Summary: ARMOR v6 — LLM-driven AI-guardrail posture pipeline with citation cross-verification and adversarial red-team loops.
|
|
5
|
+
Author-email: "Aveek Sengupta, CIB Markets RegTech" <ibt_cp_rrpp_team_reflection@restricted.chase.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/aveeksg/ARMOR
|
|
8
|
+
Project-URL: Repository, https://github.com/aveeksg/ARMOR
|
|
9
|
+
Project-URL: Issues, https://github.com/aveeksg/ARMOR/issues
|
|
10
|
+
Keywords: ai-safety,ai-governance,guardrails,eu-ai-act,nist-ai-rmf,mitre-atlas,owasp-llm-top10,llm-audit,agentic-security
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Information Technology
|
|
14
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Security
|
|
24
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: pydantic>=2.0
|
|
29
|
+
Provides-Extra: catalog
|
|
30
|
+
Requires-Dist: requests>=2.28; extra == "catalog"
|
|
31
|
+
Requires-Dist: pyyaml>=6.0; extra == "catalog"
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
34
|
+
Requires-Dist: requests>=2.28; extra == "dev"
|
|
35
|
+
Requires-Dist: pyyaml>=6.0; extra == "dev"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# ARMOR v6 (`armor-agent-plugin`)
|
|
39
|
+
|
|
40
|
+
**ARMOR** (**A**I **R**isk **M**anagement & **O**perational **R**esilience) is an LLM-driven AI-guardrail posture pipeline packaged as a Claude Code / Copilot / CIB-marketplace plugin (`v6.0.0`).
|
|
41
|
+
|
|
42
|
+
It audits any codebase on disk against a **53-row catalog of AI & agentic standards**—normalised across the **EU AI Act**, **NIST AI RMF**, **MITRE ATLAS**, **OWASP Top 10 for LLM Applications (2025)**, and the **AI Incident Database (AIID / MIT AI Risk Repository)**—and produces a schema-validated posture report describing, per guardrail, whether the project implements it, backed by deterministically cross-verified `file:line` citations.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Table of Contents
|
|
47
|
+
|
|
48
|
+
1. [Why ARMOR v6? Core Design Principles](#1-why-armor-v6-core-design-principles)
|
|
49
|
+
2. [6-Phase Pipeline Architecture](#2-6-phase-pipeline-architecture)
|
|
50
|
+
3. [Repository Layout](#3-repository-layout)
|
|
51
|
+
4. [Installation & Runtime Requirements](#4-installation--runtime-requirements)
|
|
52
|
+
5. [CLI Reference & Usage](#5-cli-reference--usage)
|
|
53
|
+
6. [Step-by-Step Disk-Queue Protocol Walkthrough](#6-step-by-step-disk-queue-protocol-walkthrough)
|
|
54
|
+
7. [Phase-by-Phase Deep Dive](#7-phase-by-phase-deep-dive)
|
|
55
|
+
- [Phase 1: `load` (Deterministic Python)](#phase-1-load-deterministic-python)
|
|
56
|
+
- [Phase 2: `triage` (LLM Applicability Profiling)](#phase-2-triage-llm-applicability-profiling)
|
|
57
|
+
- [Phase 3: `triage-audit` (Adversarial Red-Team Loop)](#phase-3-triage-audit-adversarial-red-team-loop)
|
|
58
|
+
- [Phase 4: `conform` (LLM Conformance & Citation Verification)](#phase-4-conform-llm-conformance--citation-verification)
|
|
59
|
+
- [Phase 5: `conform-audit` (Adversarial Red-Team + Re-Verification)](#phase-5-conform-audit-adversarial-red-team--re-verification)
|
|
60
|
+
- [Phase 6: `report` (Deterministic Rollup & Schema Validation)](#phase-6-report-deterministic-rollup--schema-validation)
|
|
61
|
+
8. [Citation Cross-Verification & Anti-Hallucination Engine](#8-citation-cross-verification--anti-hallucination-engine)
|
|
62
|
+
9. [Scoring, Weighting & Remediation Ranking](#9-scoring-weighting--remediation-ranking)
|
|
63
|
+
10. [The 12 Guardrail Families & 53 Sub-Guardrails](#10-the-12-guardrail-families--53-sub-guardrails)
|
|
64
|
+
11. [Catalog Subsystem & Content-Addressed Versioning](#11-catalog-subsystem--content-addressed-versioning)
|
|
65
|
+
12. [Agent Wrapper (`armor-agent`) & Verdict Contract](#12-agent-wrapper-armor-agent--verdict-contract)
|
|
66
|
+
13. [Output Directory Structure (`.armor/`)](#13-output-directory-structure-armor)
|
|
67
|
+
14. [Hard Invariants & Security Guarantees](#14-hard-invariants--security-guarantees)
|
|
68
|
+
15. [Testing & Verification](#15-testing--verification)
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## 1. Why ARMOR v6? Core Design Principles
|
|
73
|
+
|
|
74
|
+
Traditional static regex scanners produce high false-positive noise and cannot reason about architectural intent, while unconstrained LLM code reviews hallucinate file paths, invent line numbers, and suffer from sycophantic confirmation bias.
|
|
75
|
+
|
|
76
|
+
ARMOR v6 solves both problems through two non-negotiable architectural properties:
|
|
77
|
+
|
|
78
|
+
1. **Zero Internal LLM API Calls (Disk-Queue Exchange)**:
|
|
79
|
+
The calling IDE agent (Claude Code / GitHub Copilot) **is** the LLM. The Python runner never contacts Bedrock, OpenAI, Anthropic, or any external network service during an audit. Every LLM phase is a deterministic **disk-queue exchange**:
|
|
80
|
+
- The Python runner writes `<run_dir>/queue/<phase>.request.json` containing the exact prompt instructions, input data, and required `response_schema`.
|
|
81
|
+
- The agent inspects the repository, performs the analysis, and writes strict JSON to `<run_dir>/queue/<phase>.response.json`.
|
|
82
|
+
- The Python runner validates the response, enforces all invariants, and writes the phase artifact.
|
|
83
|
+
2. **Deterministic Citation Cross-Verification**:
|
|
84
|
+
Every non-gap conformance verdict (`implemented`, `partial`, `weak`) **must** carry at least one `relative/path.ext:line` or `relative/path.ext:start-end` citation. The Python runner resolves each citation beneath `project_root`, blocks `..` path traversal, checks file existence, and bounds line numbers against the actual line count of the file on disk. Any fabricated or out-of-bounds citation automatically **demotes** the guardrail verdict to `inconclusive`, marks `demoted_due_to_hallucination: true`, and logs an immutable incident record to `.armor/hallucination_log.jsonl`.
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## 2. 6-Phase Pipeline Architecture
|
|
89
|
+
|
|
90
|
+
The pipeline consists of **6 sequential phases**: two pure-Python deterministic phases (`load`, `report`) bookending four LLM disk-queue phases (`triage`, `triage-audit`, `conform`, `conform-audit`), two of which are multi-round adversarial red-team loops bounded at `MAX_AUDIT_REVISIONS = 3`.
|
|
91
|
+
|
|
92
|
+
```mermaid
|
|
93
|
+
flowchart LR
|
|
94
|
+
P1["1. load<br/>(Python)"] --> P2["2. triage<br/>(LLM)"]
|
|
95
|
+
P2 --> P3["3. triage-audit<br/>(LLM Adversary <=3x)"]
|
|
96
|
+
P3 --> P4["4. conform<br/>(LLM + Citation Verify)"]
|
|
97
|
+
P4 --> P5["5. conform-audit<br/>(LLM Adversary <=3x)"]
|
|
98
|
+
P5 --> P6["6. report<br/>(Python + Pydantic)"]
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
| # | Phase | Driver | Mode | Run-Dir Artifact | Purpose |
|
|
102
|
+
| :-: | :--- | :--- | :--- | :--- | :--- |
|
|
103
|
+
| **1** | `load` | Python | `one_shot` | `01_guardrails.json` | Resolve & normalise the 53-row AI standards catalog. |
|
|
104
|
+
| **2** | `triage` | LLM | `two_step` | `02_triage.{json,md}` | Profile the codebase and tag each of the 53 guardrails `applies`, `partial_scope`, or `n_a`. |
|
|
105
|
+
| **3** | `triage-audit` | LLM (Adversary) | `two_step` (loop `<= 3`) | `03_triage_audit.{json,md}` | Red-team every applicability decision (`CONFIRM`, `REVISE`, `INVESTIGATE`). |
|
|
106
|
+
| **4** | `conform` | LLM | `two_step` | `04_conformance.{json,md}` | Score in-scope guardrails, enforce concrete remediations, and cross-verify `file:line` citations. |
|
|
107
|
+
| **5** | `conform-audit` | LLM (Adversary) | `two_step` (loop `<= 3`) | `05_conformance_audit.{json,md}` | Red-team conformance verdicts & citations; re-verify all post-audit citations. |
|
|
108
|
+
| **6** | `report` | Python | `one_shot` | `06_report.json` + `reports/posture_*` | Join all phases, compute weighted coverage & ranked suggestions, and validate against `PostureReport`. |
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## 3. Repository Layout
|
|
113
|
+
|
|
114
|
+
```text
|
|
115
|
+
armor-agent-plugin/
|
|
116
|
+
├── README.md # Comprehensive project & technical documentation
|
|
117
|
+
├── .gitignore # Ignores .armor/, __pycache__/, *.lock, etc.
|
|
118
|
+
├── .claude-plugin/
|
|
119
|
+
│ └── plugin.json # Plugin manifest (v6.0.0)
|
|
120
|
+
├── agents/
|
|
121
|
+
│ └── armor-agent.md # Thin delegation wrapper -> emits fenced YAML verdict
|
|
122
|
+
└── skills/
|
|
123
|
+
└── armor/
|
|
124
|
+
├── SKILL.md # Agent-facing skill contract + mandatory telemetry gate
|
|
125
|
+
├── scripts/
|
|
126
|
+
│ ├── armor_run.py # 4-line runpy launcher shim
|
|
127
|
+
│ ├── armor/ # Core runtime Python package
|
|
128
|
+
│ │ ├── __init__.py # __version__ = "6.0.0"
|
|
129
|
+
│ │ ├── __main__.py # `python -m armor` entry point
|
|
130
|
+
│ │ ├── cli.py # `pipeline` subcommand dispatcher
|
|
131
|
+
│ │ ├── file_lock.py # Cross-platform exclusive file lock (stdlib only)
|
|
132
|
+
│ │ └── pipeline/
|
|
133
|
+
│ │ ├── __init__.py # Public pipeline surface re-exports
|
|
134
|
+
│ │ ├── state.py # RunDir, phase order, citation verifier, fingerprint
|
|
135
|
+
│ │ ├── load.py # Phase 1: Catalog resolution & normalisation
|
|
136
|
+
│ │ ├── triage.py # Phase 2: System profile & applicability triage
|
|
137
|
+
│ │ ├── adversary.py # Shared red-team request & delta engine
|
|
138
|
+
│ │ ├── triage_audit.py # Phase 3: Applicability red-team loop
|
|
139
|
+
│ │ ├── conform.py # Phase 4: Conformance evaluation & citation check
|
|
140
|
+
│ │ ├── conform_audit.py # Phase 5: Conformance red-team loop + re-verify
|
|
141
|
+
│ │ ├── report.py # Phase 6: Rollup, coverage, suggestions, MD/JSON output
|
|
142
|
+
│ │ ├── schema.py # Pydantic v2 PostureReport models (extra="forbid")
|
|
143
|
+
│ │ └── schemas/
|
|
144
|
+
│ │ └── posture_report.schema.json # Generated JSON Schema (byte-synced)
|
|
145
|
+
│ └── catalog/ # Offline catalog build subsystem (--update-catalog)
|
|
146
|
+
│ ├── ai_standards_versioning.py # Content-addressed SHA256 version commit helper
|
|
147
|
+
│ ├── refresh_catalog.py # Umbrella orchestrator (5 downloaders + generator)
|
|
148
|
+
│ ├── download_aiid_slim.py # AI Incident Database GraphQL downloader
|
|
149
|
+
│ ├── download_eu_ai_act_cellar.py # EU AI Act (CELEX 32024R1689) CELLAR downloader
|
|
150
|
+
│ ├── download_mitre_atlas.py # MITRE ATLAS YAML + JSON splits downloader
|
|
151
|
+
│ ├── download_nist_ai_rmf.py # NIST AI 100-1 & 600-1 PDF downloader
|
|
152
|
+
│ ├── download_owasp_llm_top10.py # OWASP LLM Top 10 (2025) PDF downloader
|
|
153
|
+
│ └── generate_guardrail_catalog.py # Normaliser -> 12 families / 53 sub-guardrails
|
|
154
|
+
├── references/
|
|
155
|
+
│ ├── architecture.md # Module-by-module architectural reference
|
|
156
|
+
│ └── ai_standards/ # Versioned 53-row catalog + standards metadata
|
|
157
|
+
│ ├── _catalog/
|
|
158
|
+
│ │ ├── latest.json # Pinned pointer -> 2026-05-14__b1ed6db3
|
|
159
|
+
│ │ └── versions/2026-05-14__b1ed6db3/
|
|
160
|
+
│ │ ├── guardrail_matrix.json # Consumed by Phase 1 load
|
|
161
|
+
│ │ ├── guardrail_matrix.{md,csv}
|
|
162
|
+
│ │ ├── guardrail_catalog.{json,md}
|
|
163
|
+
│ │ ├── source_coverage.json
|
|
164
|
+
│ │ └── manifest.json
|
|
165
|
+
│ ├── eu-ai-act/ # {latest.json, history.jsonl, versions/<v>/...}
|
|
166
|
+
│ ├── nist-ai-rmf/ # {latest.json, history.jsonl, versions/<v>/...}
|
|
167
|
+
│ ├── owasp-llm-top10/ # {latest.json, history.jsonl, versions/<v>/...}
|
|
168
|
+
│ └── mitre-atlas/ # {latest.json, history.jsonl, versions/<v>/...}
|
|
169
|
+
└── assets/
|
|
170
|
+
└── tests/ # 40-test pytest suite
|
|
171
|
+
├── conftest.py
|
|
172
|
+
└── pipeline/
|
|
173
|
+
├── _helpers.py
|
|
174
|
+
├── test_state.py
|
|
175
|
+
├── test_load.py
|
|
176
|
+
├── test_triage.py
|
|
177
|
+
├── test_triage_audit.py
|
|
178
|
+
├── test_conform.py
|
|
179
|
+
├── test_conform_audit_and_report.py
|
|
180
|
+
└── test_schema.py
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## 4. Installation & Runtime Requirements
|
|
186
|
+
|
|
187
|
+
ARMOR requires **Python 3.9+** (tested up to Python 3.14 with Pydantic v2).
|
|
188
|
+
|
|
189
|
+
| Package | Used By | When Required |
|
|
190
|
+
| :--- | :--- | :--- |
|
|
191
|
+
| `pydantic` (`>=2.0`) | `armor/pipeline/schema.py` | **Always** (every pipeline run) |
|
|
192
|
+
| `requests` | `catalog/download_*.py` | Only when running `--update-catalog` |
|
|
193
|
+
| `pyyaml` | `catalog/download_mitre_atlas.py` | Only when running `--update-catalog` |
|
|
194
|
+
| `pytest` | `skills/armor/assets/tests/` | Running the test suite |
|
|
195
|
+
|
|
196
|
+
Everything else in the runtime uses the **Python standard library** (`argparse`, `json`, `pathlib`, `runpy`, `dataclasses`, `hashlib`, `re`, `datetime`, `shutil`, `subprocess`, `os`, `sys`, `tempfile`, `threading`, `contextlib`, and platform-selected `msvcrt` / `fcntl`).
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
pip install pydantic requests pyyaml pytest
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
---
|
|
203
|
+
|
|
204
|
+
## 5. CLI Reference & Usage
|
|
205
|
+
|
|
206
|
+
The package can be invoked either via the marketplace file-path launcher `skills/armor/scripts/armor_run.py` or as `python -m armor` when `skills/armor/scripts` is on `PYTHONPATH`.
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
# Check version
|
|
210
|
+
python skills/armor/scripts/armor_run.py --version
|
|
211
|
+
# -> armor 6.0.0
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### `pipeline` Subcommand Flags
|
|
215
|
+
|
|
216
|
+
```text
|
|
217
|
+
python skills/armor/scripts/armor_run.py pipeline <project> [OPTIONS]
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
| Flag | Action | Description |
|
|
221
|
+
| :--- | :--- | :--- |
|
|
222
|
+
| `--phase {load,triage,triage-audit,conform,conform-audit,report}` | `store` | Run a specific pipeline phase. |
|
|
223
|
+
| `--finalize` | `store_true` | For two-step LLM phases: consume `queue/<phase>.response.json`, validate, and write the phase artifact (or advance the audit loop). |
|
|
224
|
+
| `--status` | `store_true` | Print the active run directory path and the current state of all 6 phases. |
|
|
225
|
+
| `--all` | `store_true` | Advance every phase runnable without an LLM, automatically finalizing any phase whose `.response.json` is already present, and stop at the first phase awaiting an LLM response. |
|
|
226
|
+
| `--catalog PATH` | `store` | Override the default `guardrail_matrix.json` (or `latest.json`) path in Phase 1 (`load`). |
|
|
227
|
+
| `--audit-model NAME` | `store` | Reserved model hint for adversary phases. |
|
|
228
|
+
| `--validate-report` | `store_true` | Validate `<project>/.armor/reports/posture_latest.json` against the Pydantic `PostureReport` contract and exit (`0` = valid, `1` = schema violation, `2` = missing file). |
|
|
229
|
+
| `--write-schema` | `store_true` | Regenerate `armor/pipeline/schemas/posture_report.schema.json` from `PostureReport.model_json_schema()` and exit. |
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
233
|
+
## 6. Step-by-Step Disk-Queue Protocol Walkthrough
|
|
234
|
+
|
|
235
|
+
Below is a complete end-to-end execution trace showing how an IDE agent (or developer) drives ARMOR against a target project at `/path/to/target-repo`:
|
|
236
|
+
|
|
237
|
+
### Step 1: Start a Fresh Run & Advance to Triage (`--all` or `--phase load`)
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --all
|
|
241
|
+
```
|
|
242
|
+
Output:
|
|
243
|
+
```text
|
|
244
|
+
[load] wrote /path/to/target-repo/.armor/runs/2026-11-16T18-00-00Z/01_guardrails.json (53 guardrails)
|
|
245
|
+
[triage] request written: /path/to/target-repo/.armor/runs/2026-11-16T18-00-00Z/queue/triage.request.json
|
|
246
|
+
LLM must write response to: /path/to/target-repo/.armor/runs/2026-11-16T18-00-00Z/queue/triage.response.json
|
|
247
|
+
then run: python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase triage --finalize
|
|
248
|
+
[--all] stopped at triage: awaiting LLM response at /path/to/target-repo/.armor/runs/2026-11-16T18-00-00Z/queue/triage.response.json
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
### Step 2: Fulfill Phase 2 (`triage`)
|
|
252
|
+
1. The agent reads `queue/triage.request.json`.
|
|
253
|
+
2. The agent writes strict JSON to `queue/triage.response.json`.
|
|
254
|
+
3. Finalize Phase 2:
|
|
255
|
+
```bash
|
|
256
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase triage --finalize
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
### Step 3: Prepare & Fulfill Phase 3 (`triage-audit`)
|
|
260
|
+
1. Prepare the adversary audit request:
|
|
261
|
+
```bash
|
|
262
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase triage-audit
|
|
263
|
+
```
|
|
264
|
+
2. The agent reads `queue/triage-audit.request.json`, critiques each row (`CONFIRM`, `REVISE`, or `INVESTIGATE`), and writes `queue/triage-audit.response.json`.
|
|
265
|
+
3. Finalize Phase 3:
|
|
266
|
+
```bash
|
|
267
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase triage-audit --finalize
|
|
268
|
+
```
|
|
269
|
+
- If the adversary issued `>= 1` `REVISE` and `revision < 3`, the runner updates the target rows, writes a bumped `triage-audit.request.json`, removes the old response, and prints:
|
|
270
|
+
```text
|
|
271
|
+
[triage-audit] auditor requested revisions; new request written. Drop a fresh response at .../queue/triage-audit.response.json and re-run with --finalize.
|
|
272
|
+
```
|
|
273
|
+
- Once converged (or at round 3), it writes `03_triage_audit.json` and `03_triage_audit.md`.
|
|
274
|
+
|
|
275
|
+
### Step 4: Prepare & Fulfill Phase 4 (`conform`)
|
|
276
|
+
1. Prepare the conformance request:
|
|
277
|
+
```bash
|
|
278
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase conform
|
|
279
|
+
```
|
|
280
|
+
2. The agent inspects the codebase for every applicable guardrail, attaches `relative/path.ext:line` citations and codebase-specific remediations, and writes `queue/conform.response.json`.
|
|
281
|
+
3. Finalize Phase 4:
|
|
282
|
+
```bash
|
|
283
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase conform --finalize
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
### Step 5: Prepare & Fulfill Phase 5 (`conform-audit`)
|
|
287
|
+
1. Prepare the conformance adversary request:
|
|
288
|
+
```bash
|
|
289
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase conform-audit
|
|
290
|
+
```
|
|
291
|
+
2. The agent red-teams the conformance rows (using only the cited `file:line` locations), writes `queue/conform-audit.response.json`, and finalizes:
|
|
292
|
+
```bash
|
|
293
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase conform-audit --finalize
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
### Step 6: Generate & Validate Phase 6 (`report`)
|
|
297
|
+
```bash
|
|
298
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --phase report
|
|
299
|
+
python skills/armor/scripts/armor_run.py pipeline /path/to/target-repo --validate-report
|
|
300
|
+
```
|
|
301
|
+
Output:
|
|
302
|
+
```text
|
|
303
|
+
[report] wrote .../06_report.json and /path/to/target-repo/.armor/reports/posture_latest.md
|
|
304
|
+
ok: /path/to/target-repo/.armor/reports/posture_latest.json conforms to PostureReport schema
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
---
|
|
308
|
+
|
|
309
|
+
## 7. Phase-by-Phase Deep Dive
|
|
310
|
+
|
|
311
|
+
### Phase 1: `load` (Deterministic Python)
|
|
312
|
+
- **Module**: `skills/armor/scripts/armor/pipeline/load.py`
|
|
313
|
+
- **Catalog Resolution**:
|
|
314
|
+
1. If `--catalog PATH` is provided, resolves `PATH` directly.
|
|
315
|
+
2. Otherwise checks `<skill_root>/references/ai_standards/_catalog/latest.json` and resolves `versions/<version>/guardrail_matrix.json`.
|
|
316
|
+
3. Falls back to walking parent directories for legacy `ai_standards/_catalog/latest.json`.
|
|
317
|
+
- **Normalisation**: Transforms each entry in `sub_guardrails` into:
|
|
318
|
+
```json
|
|
319
|
+
{
|
|
320
|
+
"sid": "GR-01.1",
|
|
321
|
+
"family": "GR-01",
|
|
322
|
+
"family_name": "Risk & impact assessment",
|
|
323
|
+
"description": "...",
|
|
324
|
+
"verification_test": "...",
|
|
325
|
+
"applicability_hint": "...",
|
|
326
|
+
"implementation_hints": "...",
|
|
327
|
+
"standard_refs": "EU AI Act Art 6, 9, 27 | NIST GOVERN-1, MAP-1, MAP-3, MAP-5 | ATLAS AML.M0017 | MIT ..."
|
|
328
|
+
}
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
### Phase 2: `triage` (LLM Applicability Profiling)
|
|
332
|
+
- **Module**: `skills/armor/scripts/armor/pipeline/triage.py`
|
|
333
|
+
- **Context Collection (`_system_profile_inputs`)**:
|
|
334
|
+
- Reads up to the first `6,000` characters of any present manifest/documentation file: `README.md`, `README.rst`, `README.txt`, `pyproject.toml`, `package.json`, `pom.xml`, `build.gradle`, `Cargo.toml`, `go.mod`, `requirements.txt`, `Dockerfile`, `openspec/AGENTS.md`, `AGENTS.md`, `CLAUDE.md`.
|
|
335
|
+
- Lists sorted top-level directory and file names (skipping dotfiles except `.github/`).
|
|
336
|
+
- **Validation**:
|
|
337
|
+
- Ensures every loaded `sid` is present exactly once (no unknown, duplicate, or missing SIDs).
|
|
338
|
+
- Enforces `status` in `{"applies", "partial_scope", "n_a"}` and non-empty `reason`.
|
|
339
|
+
- Produces `02_triage.json` and `02_triage.md`.
|
|
340
|
+
|
|
341
|
+
### Phase 3: `triage-audit` (Adversarial Red-Team Loop)
|
|
342
|
+
- **Modules**: `adversary.py` + `triage_audit.py`
|
|
343
|
+
- **Allowed Verdicts**: `CONFIRM`, `REVISE`, `INVESTIGATE`.
|
|
344
|
+
- **Loop Semantics**:
|
|
345
|
+
- When the adversary outputs `REVISE` for a row, `apply_deltas()` merges `revised_row` over the previous row (preserving `sid` and row order) and records a `{sid, critique, before, after}` entry in `revisions_applied`.
|
|
346
|
+
- When the adversary outputs `INVESTIGATE`, `{sid, critique}` is appended to `open_questions`.
|
|
347
|
+
- **Convergence**: Converges immediately when `len(revisions_applied) == 0`, or force-terminates at `revision == MAX_AUDIT_REVISIONS (3)` with `converged: false`.
|
|
348
|
+
- Intermediate rounds persist to `.triage_audit_trail.json` and are cleaned up upon completion.
|
|
349
|
+
|
|
350
|
+
### Phase 4: `conform` (LLM Conformance & Citation Verification)
|
|
351
|
+
- **Module**: `skills/armor/scripts/armor/pipeline/conform.py`
|
|
352
|
+
- **Scope**: Includes only guardrails whose post-audit triage status is `applies` or `partial_scope` (`n_a` rows are excluded from LLM conformance evaluation).
|
|
353
|
+
- **Allowed Verdicts**: `implemented`, `partial`, `weak`, `gap`, `inconclusive`, `n/a`.
|
|
354
|
+
- **Non-Gap Verdicts (`NON_GAP`)**: `{"implemented", "partial", "weak"}` — **must** include `>= 1` verified citation.
|
|
355
|
+
- **Mandatory Remediation Rules**:
|
|
356
|
+
For verdicts in `{"gap", "weak", "partial", "inconclusive"}`, `remediation` is mandatory and must be:
|
|
357
|
+
1. **Codebase-specific**: Name real files/directories in the project and exact target paths for new artefacts.
|
|
358
|
+
2. **Action-leading**: Start with an imperative verb (never `"Implement the guardrail"`).
|
|
359
|
+
3. **Concrete**: Reference real entry points, functions, and config keys.
|
|
360
|
+
4. **Anchored**: Directly satisfy the guardrail's `verification_test`.
|
|
361
|
+
5. **Free of banned boilerplate phrases**: `"Implement the guardrail"`, `"Add CI enforcement + persistent audit trail"`, `"as appropriate"`, `"follow best practices"`.
|
|
362
|
+
|
|
363
|
+
### Phase 5: `conform-audit` (Adversarial Red-Team + Re-Verification)
|
|
364
|
+
- **Module**: `skills/armor/scripts/armor/pipeline/conform_audit.py`
|
|
365
|
+
- **Adversary Directions**:
|
|
366
|
+
- *(a) DOWNGRADE rows whose evidence is weak or fabricated;*
|
|
367
|
+
- *(b) UPGRADE rows you can defend with additional citations.*
|
|
368
|
+
- The adversary may **not** invent new SIDs, change scope, or read source files beyond the `file:line` citations already present in the target rows.
|
|
369
|
+
- **Post-Delta Citation Re-Verification**:
|
|
370
|
+
After applying adversary revisions in each round, the runner re-runs `verify_citation` across every row in `new_target`. If a revised `NON_GAP` row has no valid citations (or its new citations fail verification), the runner immediately demotes it to `inconclusive`, sets `demoted_due_to_hallucination = True`, logs the rejected citations to `hallucination_log.jsonl` with `"phase": "conform-audit"`, and records the count in `post_audit_hallucinations`.
|
|
371
|
+
|
|
372
|
+
### Phase 6: `report` (Deterministic Rollup & Schema Validation)
|
|
373
|
+
- **Modules**: `report.py` + `schema.py`
|
|
374
|
+
- Joins `01_guardrails.json`, `03_triage_audit.json`, and `05_conformance_audit.json` by `sid`.
|
|
375
|
+
- Guardrails tagged `n_a` in triage inherit `verdict = "n/a"`.
|
|
376
|
+
- Aggregates `open_questions` from both Phase 3 (`triage-audit`) and Phase 5 (`conform-audit`).
|
|
377
|
+
- Validates the assembled report dictionary against `schema.PostureReport` **before** writing bytes to disk (any schema drift aborts loudly).
|
|
378
|
+
- Writes:
|
|
379
|
+
- `<run_dir>/06_report.json`
|
|
380
|
+
- `<project>/.armor/reports/posture_<iso>.json` & `posture_<iso>.md`
|
|
381
|
+
- `<project>/.armor/reports/posture_latest.json` & `posture_latest.md`
|
|
382
|
+
|
|
383
|
+
---
|
|
384
|
+
|
|
385
|
+
## 8. Citation Cross-Verification & Anti-Hallucination Engine
|
|
386
|
+
|
|
387
|
+
Implemented in `verify_citation(project, citation)` (`armor/pipeline/state.py`):
|
|
388
|
+
|
|
389
|
+
### Accepted Grammar
|
|
390
|
+
```regex
|
|
391
|
+
^([^:]+?)(?::(\d+))?(?:-(\d+))?$
|
|
392
|
+
```
|
|
393
|
+
Examples:
|
|
394
|
+
- `src/main.py` (whole file)
|
|
395
|
+
- `src/main.py:42` (single line)
|
|
396
|
+
- `src/main.py:10-25` (line range)
|
|
397
|
+
|
|
398
|
+
### Verification Algorithm
|
|
399
|
+
1. **Regex parse**: Strip whitespace and match against `_CITATION_RE`. No match -> `(False, "unparseable citation")`.
|
|
400
|
+
2. **Path confinement (`..` traversal protection)**: Resolve `(project / rel).resolve()` and check `p.relative_to(project.resolve())`. If `ValueError` is raised -> `(False, "citation escapes project_root")`.
|
|
401
|
+
3. **File existence**: Check `p.is_file()`. If missing -> `(False, "file not found")`.
|
|
402
|
+
4. **Whole-file citation**: If no `:line` part is given -> `(True, "ok")`.
|
|
403
|
+
5. **Line count check**: Read file (`utf-8`, `errors="replace"`) and count lines (`total`). If `OSError` -> `(False, "unreadable")`.
|
|
404
|
+
6. **Bounds check**: Let `lo = int(start)` and `hi = int(end) if end else lo`. If `lo < 1` or `hi < lo` or `hi > total` -> `(False, "line out of bounds (file has <total> lines)")`. Otherwise -> `(True, "ok")`.
|
|
405
|
+
|
|
406
|
+
### Hallucination Logging (`.armor/hallucination_log.jsonl`)
|
|
407
|
+
Every rejected citation in Phase 4 (`conform`) or Phase 5 (`conform-audit`) is appended to `<project>/.armor/hallucination_log.jsonl`:
|
|
408
|
+
```json
|
|
409
|
+
{"ts": "2026-11-16T18-00-00Z", "phase": "conform", "sid": "GR-01.1", "citation": "src/ghost.py:10", "reason": "file not found"}
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
---
|
|
413
|
+
|
|
414
|
+
## 9. Scoring, Weighting & Remediation Ranking
|
|
415
|
+
|
|
416
|
+
### Verdict Weights (`VERDICT_WEIGHT`)
|
|
417
|
+
|
|
418
|
+
| Verdict | Weight | Included in Coverage Denominator? |
|
|
419
|
+
| :--- | :---: | :---: |
|
|
420
|
+
| `implemented` | **1.0** | Yes |
|
|
421
|
+
| `partial` | **0.6** | Yes |
|
|
422
|
+
| `weak` | **0.3** | Yes |
|
|
423
|
+
| `gap` | **0.0** | Yes |
|
|
424
|
+
| `inconclusive` | **0.0** | Yes |
|
|
425
|
+
| `n/a` | — | **No** (excluded from denominator entirely) |
|
|
426
|
+
|
|
427
|
+
- **Overall Coverage (`overall_pct` & `applicable_only_pct`)**:
|
|
428
|
+
$$\text{overall\_pct} = \text{round}\left(100 \times \frac{\sum_{r \in \text{in\_scope}} \text{VERDICT\_WEIGHT}[r.\text{verdict}]}{|\text{in\_scope}|},\ 1\right)$$
|
|
429
|
+
(Returns `0.0` if `in_scope` is empty.)
|
|
430
|
+
- **Per-Family Rollup (`by_family`)**:
|
|
431
|
+
Sorted ascending by `(score_pct, family)` so the **weakest guardrail families appear first**. Families where all guardrails are `n/a` are omitted from `by_family`.
|
|
432
|
+
|
|
433
|
+
### Remediation Suggestions Ranking (`_suggestions`)
|
|
434
|
+
All actionable rows (`verdict` in `{"gap", "inconclusive", "weak", "partial"}`) are ranked deterministically by:
|
|
435
|
+
1. **`-points_unlocked`** (`round(1.0 - VERDICT_WEIGHT[verdict], 2)` descending — so `gap` & `inconclusive` (`+1.0`) rank ahead of `weak` (`+0.7`) and `partial` (`+0.4`)).
|
|
436
|
+
2. **`family_score_pct`** (ascending — weakest family first).
|
|
437
|
+
3. **`_SUGGESTION_KIND_ORDER[verdict]`** (`gap: 0`, `inconclusive: 1`, `weak: 2`, `partial: 3`).
|
|
438
|
+
4. **`sid`** (lexicographical tie-breaker).
|
|
439
|
+
|
|
440
|
+
---
|
|
441
|
+
|
|
442
|
+
## 10. The 12 Guardrail Families & 53 Sub-Guardrails
|
|
443
|
+
|
|
444
|
+
| Family ID | Family Name | Sub-Guardrail Count | SIDs | Key Standards Mapped |
|
|
445
|
+
| :---: | :--- | :---: | :--- | :--- |
|
|
446
|
+
| **GR-01** | Risk & impact assessment | 5 | `GR-01.1` – `GR-01.5` | EU AI Act Art 6, 9, 13, 27, 43; NIST GOVERN-1/4, MAP-1/2/3/5, MANAGE-1..4 |
|
|
447
|
+
| **GR-02** | Data governance | 5 | `GR-02.1` – `GR-02.5` | EU AI Act Art 10, 15, 53; OWASP LLM02, LLM04, LLM08; ATLAS AML.M0006/7/14/18 |
|
|
448
|
+
| **GR-03** | Transparency & disclosure | 4 | `GR-03.1` – `GR-03.4` | EU AI Act Art 11, 13, 14, 50, 53; OWASP LLM09; NIST GOVERN-1/4, MAP-1, MEASURE-2 |
|
|
449
|
+
| **GR-04** | Human oversight | 4 | `GR-04.1` – `GR-04.4` | EU AI Act Art 14; NIST GOVERN-3, MAP-3, MANAGE-1..4; OWASP LLM06, LLM09, LLM10 |
|
|
450
|
+
| **GR-05** | Robustness & accuracy | 5 | `GR-05.1` – `GR-05.5` | EU AI Act Art 15, 43; NIST MEASURE-1/2/4, MANAGE-2/3; OWASP LLM05, LLM09, LLM10 |
|
|
451
|
+
| **GR-06** | Adversarial security | 5 | `GR-06.1` – `GR-06.5` | EU AI Act Art 15, 55; OWASP LLM01, LLM02, LLM05, LLM06, LLM07, LLM10; ATLAS AML.M0002/3/4/15/18/19 |
|
|
452
|
+
| **GR-07** | Privacy & data protection | 4 | `GR-07.1` – `GR-07.4` | EU AI Act Art 10, 12, 15; OWASP LLM02, LLM06; ATLAS AML.M0012, AML.M0018 |
|
|
453
|
+
| **GR-08** | Fairness & non-discrimination | 4 | `GR-08.1` – `GR-08.4` | EU AI Act Art 9, 10, 13, 14, 15, 27; NIST MAP-1/2, MEASURE-1/2; MIT 1.1 |
|
|
454
|
+
| **GR-09** | Content safety | 4 | `GR-09.1` – `GR-09.4` | EU AI Act Art 13, 15; OWASP LLM02, LLM05, LLM09; NIST GOVERN-4, MEASURE-2, MANAGE-2/3 |
|
|
455
|
+
| **GR-10** | Logging, monitoring & incident response | 5 | `GR-10.1` – `GR-10.5` | EU AI Act Art 12, 14, 19, 73; NIST GOVERN-1/3, MEASURE-2, MANAGE-4; ATLAS AML.M0016 |
|
|
456
|
+
| **GR-11** | Supply-chain & component integrity | 4 | `GR-11.1` – `GR-11.4` | EU AI Act Art 15, 53; OWASP LLM03, LLM05; ATLAS AML.M0007, AML.M0014 |
|
|
457
|
+
| **GR-12** | Agentic autonomy & action safety | 4 | `GR-12.1` – `GR-12.4` | EU AI Act Art 14, 15; OWASP LLM06, LLM10; ATLAS AML.M0018; MIT 5.2, 7.1, 7.4 |
|
|
458
|
+
| **Total** | **12 Families** | **53** | `GR-01.1` – `GR-12.4` | **5 Standards Frameworks** |
|
|
459
|
+
|
|
460
|
+
---
|
|
461
|
+
|
|
462
|
+
## 11. Catalog Subsystem & Content-Addressed Versioning
|
|
463
|
+
|
|
464
|
+
The offline catalog subsystem (`skills/armor/scripts/catalog/`) fetches public regulatory and technical standards and normalises them into `_catalog/versions/<date>__<sha8>/`.
|
|
465
|
+
|
|
466
|
+
- **Content-Addressed Versioning (`ai_standards_versioning.py`)**:
|
|
467
|
+
Computes a deterministic SHA256 hash over all staged files (excluding `manifest.json`, which contains a non-deterministic `fetched_at` timestamp) by hashing `relpath + \0 + file_bytes` in sorted order.
|
|
468
|
+
- If `latest.json["sha"] == sha`: discards the staged directory and appends a `status: "unchanged"` entry to `history.jsonl`.
|
|
469
|
+
- Otherwise: moves the staged directory to `versions/<YYYY-MM-DD>__<sha8>/`, updates `latest.json`, and appends `status: "new"` or `"changed"` to `history.jsonl`.
|
|
470
|
+
- **Refresh Orchestrator (`refresh_catalog.py`)**:
|
|
471
|
+
```bash
|
|
472
|
+
# Check if upstream standards have drifted (exit 0 = unchanged, 1 = drift)
|
|
473
|
+
python skills/armor/scripts/catalog/refresh_catalog.py --check-only
|
|
474
|
+
|
|
475
|
+
# Regenerate the 53-row catalog from local standards without hitting the network
|
|
476
|
+
python skills/armor/scripts/catalog/refresh_catalog.py --skip-downloads
|
|
477
|
+
|
|
478
|
+
# Full network refresh (runs all 5 downloaders + generator)
|
|
479
|
+
python skills/armor/scripts/catalog/refresh_catalog.py
|
|
480
|
+
```
|
|
481
|
+
|
|
482
|
+
---
|
|
483
|
+
|
|
484
|
+
## 12. Agent Wrapper (`armor-agent`) & Verdict Contract
|
|
485
|
+
|
|
486
|
+
`agents/armor-agent.md` defines the `armor-agent` delegation wrapper. It routes arguments into the `armor` skill, enforces a **mandatory self-targeting guard**, and emits a deterministic fenced YAML verdict block.
|
|
487
|
+
|
|
488
|
+
### Self-Targeting Guard
|
|
489
|
+
If `<project_root>` resolves to `armor-agent-plugin`, `armor-agent`, or `plugins/armor-agent-plugin/` after path normalisation, `armor-agent` **refuses to run** and immediately emits `verdict: fail` with zeroed counts and:
|
|
490
|
+
```yaml
|
|
491
|
+
notes: "self-targeting refused: armor-agent cannot audit itself"
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
### Deterministic Verdict Rules (First Match Wins)
|
|
495
|
+
|
|
496
|
+
| Priority | Condition | Emitted Verdict |
|
|
497
|
+
| :---: | :--- | :---: |
|
|
498
|
+
| **1** | Any `gap` row (`posture.gap > 0`) **OR** `hallucinations.demoted_rows > 0` | `fail` |
|
|
499
|
+
| **2** | Pipeline aborted (any phase exited non-zero) | `fail` |
|
|
500
|
+
| **3** | Zero `gap` rows, but `>= 1` `weak` row **OR** `>= 1` `open_question` | `warn` |
|
|
501
|
+
| **4** | Only `implemented` / `partial` / `n/a` / `inconclusive` rows **AND** `0` hallucinations **AND** `0` open questions | `pass` |
|
|
502
|
+
|
|
503
|
+
---
|
|
504
|
+
|
|
505
|
+
## 13. Output Directory Structure (`.armor/`)
|
|
506
|
+
|
|
507
|
+
ARMOR is strictly read-only on the target project outside `<project_root>/.armor/`:
|
|
508
|
+
|
|
509
|
+
```text
|
|
510
|
+
<project_root>/.armor/
|
|
511
|
+
├── runs/<YYYY-MM-DDTHH-MM-SSZ>/
|
|
512
|
+
│ ├── 01_guardrails.json # Phase 1 output (53 normalised rows)
|
|
513
|
+
│ ├── 02_triage.json # Phase 2 output (system_profile + applicability)
|
|
514
|
+
│ ├── 02_triage.md # Human-readable triage summary
|
|
515
|
+
│ ├── 03_triage_audit.json # Phase 3 output (converged applicability + audit_trail)
|
|
516
|
+
│ ├── 03_triage_audit.md # Human-readable triage audit trail
|
|
517
|
+
│ ├── 04_conformance.json # Phase 4 output (verdicts + verified/rejected citations)
|
|
518
|
+
│ ├── 04_conformance.md # Human-readable conformance table (*demoted* markers)
|
|
519
|
+
│ ├── 05_conformance_audit.json # Phase 5 output (post-audit rows + audit_trail)
|
|
520
|
+
│ ├── 05_conformance_audit.md # Human-readable conformance audit trail
|
|
521
|
+
│ ├── 06_report.json # Phase 6 Pydantic-validated PostureReport
|
|
522
|
+
│ ├── .triage_audit_trail.json # Transient state (present only mid-Phase 3 loop)
|
|
523
|
+
│ ├── .conform_audit_trail.json # Transient state (present only mid-Phase 5 loop)
|
|
524
|
+
│ └── queue/
|
|
525
|
+
│ ├── <phase>.request.json # Written by Python runner
|
|
526
|
+
│ └── <phase>.response.json # Written by LLM agent (unlinked after audit convergence)
|
|
527
|
+
├── reports/
|
|
528
|
+
│ ├── posture_<YYYY-MM-DDTHH-MM-SSZ>.json
|
|
529
|
+
│ ├── posture_<YYYY-MM-DDTHH-MM-SSZ>.md
|
|
530
|
+
│ ├── posture_latest.json # Latest validated PostureReport JSON
|
|
531
|
+
│ └── posture_latest.md # Latest full Markdown posture report
|
|
532
|
+
└── hallucination_log.jsonl # Append-only log of all rejected/fabricated citations
|
|
533
|
+
```
|
|
534
|
+
|
|
535
|
+
---
|
|
536
|
+
|
|
537
|
+
## 14. Hard Invariants & Security Guarantees
|
|
538
|
+
|
|
539
|
+
1. **No External LLM Calls**: The Python runner never contacts an external LLM or network endpoint; all LLM interactions occur through local `.request.json` / `.response.json` files.
|
|
540
|
+
2. **Mandatory Citation Cross-Verification**: Every `NON_GAP` (`implemented`, `partial`, `weak`) verdict without at least one verified citation under `project_root` is automatically demoted to `inconclusive` and logged to `hallucination_log.jsonl`.
|
|
541
|
+
3. **Bounded Adversarial Loops**: `triage-audit` and `conform-audit` are strictly bounded at `MAX_AUDIT_REVISIONS = 3`; unresolved disputes surface as `open_questions`.
|
|
542
|
+
4. **Pre-Write Schema Enforcement**: `06_report.json` and `posture_latest.json` are validated against `PostureReport` (`extra="forbid"`) before bytes hit disk.
|
|
543
|
+
5. **Byte-Synchronized JSON Schema**: `posture_report.schema.json` is enforced by tests to be byte-identical to `PostureReport.model_json_schema()`.
|
|
544
|
+
6. **Path Confinement**: Only `<project_root>/.armor/` is written under the target repository, and `..` traversal in citations is rejected via `Path.relative_to()`.
|
|
545
|
+
7. **Self-Targeting Prevention**: `armor-agent` refuses to audit itself.
|
|
546
|
+
|
|
547
|
+
---
|
|
548
|
+
|
|
549
|
+
## 15. Testing & Verification
|
|
550
|
+
|
|
551
|
+
ARMOR ships with a 40-test `pytest` suite under `skills/armor/assets/tests/` covering every phase, invariant, error condition, hallucination demotion path, audit loop convergence/force-termination, and Pydantic schema constraint.
|
|
552
|
+
|
|
553
|
+
```bash
|
|
554
|
+
python -m pytest skills/armor/assets/tests -v
|
|
555
|
+
```
|
|
556
|
+
|
|
557
|
+
### Test Coverage Breakdown (40 Tests)
|
|
558
|
+
|
|
559
|
+
| Test Module | Tests | Behaviours Verified |
|
|
560
|
+
| :--- | :---: | :--- |
|
|
561
|
+
| `test_state.py` | **10** | Canonical `PHASES` tuple; `new_run`/`current_run` round-trip; fresh `pending` status; `require_phase_done` guard; file, line, and range citation verification; out-of-bounds line rejection; missing file rejection; `..` path traversal blocking; `project_fingerprint` change detection on edit. |
|
|
562
|
+
| `test_load.py` | **3** | Normalisation of SIDs, families, and `family_name`; artifact persistence equality; explicit `--catalog` path resolution. |
|
|
563
|
+
| `test_triage.py` | **5** | Request generation with all SIDs + `README.md` manifest preview + `response_schema`; happy-path JSON/MD output and `by_status` counts; rejection of unknown SID, missing SID, and invalid status. |
|
|
564
|
+
| `test_triage_audit.py` | **3** | Immediate round-1 convergence on all `CONFIRM`; 2-round convergence after a `REVISE` delta (`audit_trail` length 2); force-termination at `MAX_AUDIT_REVISIONS = 3` with `converged: False`. |
|
|
565
|
+
| `test_conform.py` | **4** | Exclusion of `n_a` rows from `in_scope`; valid citation retention; bogus citation demotion to `inconclusive` + `hallucination_log.jsonl` append; rejection of out-of-scope SIDs. |
|
|
566
|
+
| `test_conform_audit_and_report.py` | **6** | Adversary `REVISE` downgrade (`implemented` -> `weak`); adversary `REVISE` with fabricated citation auto-demoted to `inconclusive`; Phase 6 report join + `posture_latest.{json,md}` pointers; 50% coverage calculation & `n/a`-only family exclusion; suggestion ranking by `points_unlocked` (`+1.0` before `+0.7`, 15% mixed coverage); `open_questions` aggregation. |
|
|
567
|
+
| `test_schema.py` | **9** | `PostureReport` validation (`overall_pct == 50.0`); rejection of unknown top-level keys, unknown row keys, invalid verdicts, malformed `generated_at` timestamps, and out-of-range percentages; zero-drift validation of persisted report; schema persistence (`title == "PostureReport"`); byte-equality between packaged `posture_report.schema.json` and `PostureReport.model_json_schema()`. |
|
|
568
|
+
|
|
569
|
+
---
|
|
570
|
+
|
|
571
|
+
## Author & Attribution
|
|
572
|
+
|
|
573
|
+
- **Artifact**: `armor-agent-plugin` (`v6.0.0`)
|
|
574
|
+
- **Author**: Aveek Sengupta, CIB Markets RegTech (`ibt_cp_rrpp_team_reflection@restricted.chase.com`)
|