agent-assurance 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_assurance-0.5.1/.github/PULL_REQUEST_TEMPLATE.md +11 -0
- agent_assurance-0.5.1/.github/dependabot.yml +8 -0
- agent_assurance-0.5.1/.github/workflows/assurance.yml +221 -0
- agent_assurance-0.5.1/.github/workflows/ci.yml +65 -0
- agent_assurance-0.5.1/.github/workflows/codeql.yml +25 -0
- agent_assurance-0.5.1/.github/workflows/pr-diff-demo.yml +25 -0
- agent_assurance-0.5.1/.github/workflows/release.yml +53 -0
- agent_assurance-0.5.1/.github/workflows/scorecard.yml +35 -0
- agent_assurance-0.5.1/.gitignore +23 -0
- agent_assurance-0.5.1/.pre-commit-hooks.yaml +12 -0
- agent_assurance-0.5.1/CLAUDE.md +37 -0
- agent_assurance-0.5.1/CONTRIBUTING.md +34 -0
- agent_assurance-0.5.1/LICENSE +17 -0
- agent_assurance-0.5.1/PKG-INFO +155 -0
- agent_assurance-0.5.1/README.md +133 -0
- agent_assurance-0.5.1/SECURITY.md +9 -0
- agent_assurance-0.5.1/action.yml +209 -0
- agent_assurance-0.5.1/contrib/claude-code/README.md +37 -0
- agent_assurance-0.5.1/contrib/claude-code/aa-post-edit.sh +25 -0
- agent_assurance-0.5.1/docs/RELEASING.md +15 -0
- agent_assurance-0.5.1/docs/ROADMAP.md +14 -0
- agent_assurance-0.5.1/docs/capability-card.png +0 -0
- agent_assurance-0.5.1/docs/demo-setup.sh +14 -0
- agent_assurance-0.5.1/docs/demo.gif +0 -0
- agent_assurance-0.5.1/docs/demo.tape +40 -0
- agent_assurance-0.5.1/docs/how-it-works.md +104 -0
- agent_assurance-0.5.1/docs/integrations.md +124 -0
- agent_assurance-0.5.1/docs/landscape.md +42 -0
- agent_assurance-0.5.1/docs/pr-comment.png +0 -0
- agent_assurance-0.5.1/docs/real-world.md +158 -0
- agent_assurance-0.5.1/examples/dangerous-agent.yaml +47 -0
- agent_assurance-0.5.1/examples/demo/README.md +5 -0
- agent_assurance-0.5.1/examples/demo/analytics-helper/.mcp.json +12 -0
- agent_assurance-0.5.1/examples/demo/analytics-helper/agent-assurance.yaml +24 -0
- agent_assurance-0.5.1/examples/high-agent.yaml +39 -0
- agent_assurance-0.5.1/examples/medium-agent.yaml +33 -0
- agent_assurance-0.5.1/examples/repos/claude-code-approval-broken/.claude/settings.json +12 -0
- agent_assurance-0.5.1/examples/repos/claude-code-approval-broken/.mcp.json +9 -0
- agent_assurance-0.5.1/examples/repos/claude-code-approval-broken/agent-assurance.yaml +23 -0
- agent_assurance-0.5.1/examples/repos/claude-code-approval-kept/.claude/settings.json +18 -0
- agent_assurance-0.5.1/examples/repos/claude-code-approval-kept/agent-assurance.yaml +23 -0
- agent_assurance-0.5.1/examples/repos/codex-and-tools/.codex/config.toml +10 -0
- agent_assurance-0.5.1/examples/repos/codex-and-tools/agent-assurance.yaml +23 -0
- agent_assurance-0.5.1/examples/repos/codex-and-tools/agent-tools.json +28 -0
- agent_assurance-0.5.1/examples/repos/mcp-promise-broken/.mcp.json +22 -0
- agent_assurance-0.5.1/examples/repos/mcp-promise-broken/agent-assurance.yaml +24 -0
- agent_assurance-0.5.1/examples/repos/mcp-promise-kept/.mcp.json +12 -0
- agent_assurance-0.5.1/examples/repos/mcp-promise-kept/agent-assurance.yaml +24 -0
- agent_assurance-0.5.1/examples/repos/mcp-unknown-server/.claude/settings.json +1 -0
- agent_assurance-0.5.1/examples/repos/mcp-unknown-server/.mcp.json +13 -0
- agent_assurance-0.5.1/examples/repos/mcp-unknown-server/AGENTS.md +3 -0
- agent_assurance-0.5.1/examples/repos/policy-org/.mcp.json +13 -0
- agent_assurance-0.5.1/examples/repos/policy-org/agent-assurance.policy.yaml +26 -0
- agent_assurance-0.5.1/examples/repos/policy-org/agent-assurance.yaml +20 -0
- agent_assurance-0.5.1/examples/safe-agent.yaml +27 -0
- agent_assurance-0.5.1/pyproject.toml +37 -0
- agent_assurance-0.5.1/schema/agent-assurance.policy.schema.json +322 -0
- agent_assurance-0.5.1/schema/agent-assurance.schema.json +167 -0
- agent_assurance-0.5.1/src/agent_assurance/__init__.py +7 -0
- agent_assurance-0.5.1/src/agent_assurance/attest.py +92 -0
- agent_assurance-0.5.1/src/agent_assurance/checks/__init__.py +36 -0
- agent_assurance-0.5.1/src/agent_assurance/checks/base.py +85 -0
- agent_assurance-0.5.1/src/agent_assurance/checks/blast_radius.py +97 -0
- agent_assurance-0.5.1/src/agent_assurance/checks/declared_vs_observed.py +178 -0
- agent_assurance-0.5.1/src/agent_assurance/cli.py +305 -0
- agent_assurance-0.5.1/src/agent_assurance/diff.py +257 -0
- agent_assurance-0.5.1/src/agent_assurance/engine.py +74 -0
- agent_assurance-0.5.1/src/agent_assurance/manifest.py +128 -0
- agent_assurance-0.5.1/src/agent_assurance/mcp_server.py +145 -0
- agent_assurance-0.5.1/src/agent_assurance/policy.py +184 -0
- agent_assurance-0.5.1/src/agent_assurance/reports/__init__.py +8 -0
- agent_assurance-0.5.1/src/agent_assurance/reports/html.py +152 -0
- agent_assurance-0.5.1/src/agent_assurance/reports/json_report.py +48 -0
- agent_assurance-0.5.1/src/agent_assurance/reports/markdown.py +89 -0
- agent_assurance-0.5.1/src/agent_assurance/reports/sarif.py +120 -0
- agent_assurance-0.5.1/src/agent_assurance/risk.py +137 -0
- agent_assurance-0.5.1/src/agent_assurance/scan/__init__.py +98 -0
- agent_assurance-0.5.1/src/agent_assurance/scan/base.py +50 -0
- agent_assurance-0.5.1/src/agent_assurance/scan/catalog.py +298 -0
- agent_assurance-0.5.1/src/agent_assurance/scan/claude_settings.py +226 -0
- agent_assurance-0.5.1/src/agent_assurance/scan/codex_toml.py +63 -0
- agent_assurance-0.5.1/src/agent_assurance/scan/mcp_json.py +149 -0
- agent_assurance-0.5.1/src/agent_assurance/scan/tool_defs.py +124 -0
- agent_assurance-0.5.1/tests/conftest.py +11 -0
- agent_assurance-0.5.1/tests/test_action_yaml.py +33 -0
- agent_assurance-0.5.1/tests/test_attest.py +54 -0
- agent_assurance-0.5.1/tests/test_blast_radius.py +76 -0
- agent_assurance-0.5.1/tests/test_cli_contract.py +114 -0
- agent_assurance-0.5.1/tests/test_diff.py +99 -0
- agent_assurance-0.5.1/tests/test_html.py +59 -0
- agent_assurance-0.5.1/tests/test_mcp_server.py +51 -0
- agent_assurance-0.5.1/tests/test_policy.py +93 -0
- agent_assurance-0.5.1/tests/test_scan.py +319 -0
- agent_assurance-0.5.1/tests/test_sources.py +66 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
## What this changes
|
|
2
|
+
|
|
3
|
+
<!-- one paragraph: what and why -->
|
|
4
|
+
|
|
5
|
+
## Checklist
|
|
6
|
+
|
|
7
|
+
- [ ] `ruff check src tests && pytest -q` pass locally
|
|
8
|
+
- [ ] New scanner / check / catalogue entry comes with: a fixture in `examples/repos/`, a test, a job in `.github/workflows/assurance.yml`, and (catalogue) its source
|
|
9
|
+
- [ ] No LLM in the verdict; nothing executed; no network; secret values never read
|
|
10
|
+
- [ ] Exit codes `0/1/2` unchanged (or the change is called out as breaking)
|
|
11
|
+
- [ ] Docs updated (`README.md` only for what a first-time reader needs; detail in `docs/`)
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
# Dogfooding: runs this repo's own Action against the bundled example manifests.
|
|
2
|
+
#
|
|
3
|
+
# Every job here is an assertion, not a demo. If the gate stops blocking, CI
|
|
4
|
+
# goes red — a green run means "the Action still passes what it should and
|
|
5
|
+
# still blocks what it should".
|
|
6
|
+
name: Agent Assurance (self-demo)
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
branches: [main]
|
|
11
|
+
pull_request:
|
|
12
|
+
workflow_dispatch:
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
medium-agent-passes:
|
|
19
|
+
name: Medium-risk agent (must PASS)
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
23
|
+
- name: Run Agent Assurance
|
|
24
|
+
uses: ./
|
|
25
|
+
with:
|
|
26
|
+
manifest: examples/medium-agent.yaml
|
|
27
|
+
check: all
|
|
28
|
+
fail-on: fail
|
|
29
|
+
|
|
30
|
+
high-agent-reviews:
|
|
31
|
+
name: High-risk agent (must PASS with fail-on=fail, BLOCK with fail-on=review)
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
35
|
+
- name: fail-on=fail lets REVIEW through
|
|
36
|
+
uses: ./
|
|
37
|
+
with:
|
|
38
|
+
manifest: examples/high-agent.yaml
|
|
39
|
+
check: all
|
|
40
|
+
fail-on: fail
|
|
41
|
+
- name: fail-on=review must block
|
|
42
|
+
id: strict
|
|
43
|
+
continue-on-error: true
|
|
44
|
+
uses: ./
|
|
45
|
+
with:
|
|
46
|
+
manifest: examples/high-agent.yaml
|
|
47
|
+
check: all
|
|
48
|
+
fail-on: review
|
|
49
|
+
- name: Assert the strict gate tripped
|
|
50
|
+
if: steps.strict.outcome != 'failure'
|
|
51
|
+
run: |
|
|
52
|
+
echo "::error::fail-on=review did not block a HIGH-risk agent (outcome=${{ steps.strict.outcome }})"
|
|
53
|
+
exit 1
|
|
54
|
+
|
|
55
|
+
dangerous-agent-blocks:
|
|
56
|
+
name: Dangerous agent (must BLOCK)
|
|
57
|
+
runs-on: ubuntu-latest
|
|
58
|
+
steps:
|
|
59
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
60
|
+
- name: Run Agent Assurance (expected to fail)
|
|
61
|
+
id: gate
|
|
62
|
+
continue-on-error: true
|
|
63
|
+
uses: ./
|
|
64
|
+
with:
|
|
65
|
+
manifest: examples/dangerous-agent.yaml
|
|
66
|
+
check: all
|
|
67
|
+
fail-on: fail
|
|
68
|
+
sarif: aa.sarif
|
|
69
|
+
- name: Assert the gate tripped
|
|
70
|
+
if: steps.gate.outcome != 'failure'
|
|
71
|
+
run: |
|
|
72
|
+
echo "::error::The assurance gate did NOT block a CRITICAL agent (outcome=${{ steps.gate.outcome }})"
|
|
73
|
+
exit 1
|
|
74
|
+
- name: Assert SARIF was produced and points at the manifest
|
|
75
|
+
run: |
|
|
76
|
+
test -s aa.sarif
|
|
77
|
+
python3 - <<'PY'
|
|
78
|
+
import json
|
|
79
|
+
s = json.load(open("aa.sarif"))
|
|
80
|
+
r = s["runs"][0]["results"][0]
|
|
81
|
+
uri = r["locations"][0]["physicalLocation"]["artifactLocation"]["uri"]
|
|
82
|
+
assert uri == "examples/dangerous-agent.yaml", uri
|
|
83
|
+
assert r["kind"] == "fail" and r["level"] == "error", r
|
|
84
|
+
print("SARIF ok:", uri, r["level"])
|
|
85
|
+
PY
|
|
86
|
+
- name: Upload SARIF artifact (inspectable from the run page)
|
|
87
|
+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
88
|
+
with:
|
|
89
|
+
name: aa-sarif
|
|
90
|
+
path: aa.sarif
|
|
91
|
+
|
|
92
|
+
scan-promise-kept:
|
|
93
|
+
name: scan — promise kept (must PASS) + attestation written
|
|
94
|
+
runs-on: ubuntu-latest
|
|
95
|
+
permissions:
|
|
96
|
+
contents: read
|
|
97
|
+
id-token: write # Sigstore signing
|
|
98
|
+
attestations: write
|
|
99
|
+
steps:
|
|
100
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
101
|
+
- name: Run Agent Assurance (scan)
|
|
102
|
+
uses: ./
|
|
103
|
+
with:
|
|
104
|
+
mode: scan
|
|
105
|
+
directory: examples/repos/mcp-promise-kept
|
|
106
|
+
manifest: examples/repos/mcp-promise-kept/agent-assurance.yaml
|
|
107
|
+
fail-on: review # a kept promise has nothing to review either
|
|
108
|
+
attest: sign # in-toto statement, signed with Sigstore (public repo)
|
|
109
|
+
- name: Assert the attestation is a valid in-toto statement about the right files
|
|
110
|
+
run: |
|
|
111
|
+
python3 - <<'PY'
|
|
112
|
+
import json
|
|
113
|
+
s = json.load(open("aa-attestation.json"))
|
|
114
|
+
assert s["_type"] == "https://in-toto.io/Statement/v1"
|
|
115
|
+
assert s["predicateType"].endswith("/attestation/v1")
|
|
116
|
+
names = sorted(x["name"] for x in s["subject"])
|
|
117
|
+
assert names == [".mcp.json", "agent-assurance.yaml"], names
|
|
118
|
+
assert s["predicate"]["report"]["verdict"] == "PASS"
|
|
119
|
+
assert s["predicate"]["git"]["commit"]
|
|
120
|
+
print("attestation ok:", names, s["predicate"]["git"]["commit"][:7])
|
|
121
|
+
PY
|
|
122
|
+
- name: Verify the signed attestation the way a third party would
|
|
123
|
+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
|
124
|
+
env:
|
|
125
|
+
GH_TOKEN: ${{ github.token }}
|
|
126
|
+
run: |
|
|
127
|
+
gh attestation verify examples/repos/mcp-promise-kept/.mcp.json \
|
|
128
|
+
-R "$GITHUB_REPOSITORY" \
|
|
129
|
+
--predicate-type https://github.com/kunko-ai-labs/agent-assurance/attestation/v1
|
|
130
|
+
|
|
131
|
+
scan-promise-broken:
|
|
132
|
+
name: scan — promise broken (must BLOCK)
|
|
133
|
+
runs-on: ubuntu-latest
|
|
134
|
+
steps:
|
|
135
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
136
|
+
- name: Run Agent Assurance (scan, expected to fail)
|
|
137
|
+
id: gate
|
|
138
|
+
continue-on-error: true
|
|
139
|
+
uses: ./
|
|
140
|
+
with:
|
|
141
|
+
mode: scan
|
|
142
|
+
directory: examples/repos/mcp-promise-broken
|
|
143
|
+
manifest: examples/repos/mcp-promise-broken/agent-assurance.yaml
|
|
144
|
+
sarif: aa-scan.sarif
|
|
145
|
+
- name: Assert the gate tripped
|
|
146
|
+
if: steps.gate.outcome != 'failure'
|
|
147
|
+
run: |
|
|
148
|
+
echo "::error::scan did NOT block a broken promise (outcome=${{ steps.gate.outcome }})"
|
|
149
|
+
exit 1
|
|
150
|
+
- name: Assert SARIF anchors AA-002 at .mcp.json lines
|
|
151
|
+
run: |
|
|
152
|
+
python3 - <<'PY'
|
|
153
|
+
import json
|
|
154
|
+
s = json.load(open("aa-scan.sarif"))
|
|
155
|
+
r = next(x for x in s["runs"][0]["results"] if x["ruleId"] == "AA-002")
|
|
156
|
+
locs = [(l["physicalLocation"]["artifactLocation"]["uri"], l["physicalLocation"]["region"]["startLine"]) for l in r["locations"]]
|
|
157
|
+
assert r["kind"] == "fail" and locs == [(".mcp.json", 11), (".mcp.json", 16)], locs
|
|
158
|
+
print("SARIF ok:", locs)
|
|
159
|
+
PY
|
|
160
|
+
|
|
161
|
+
scan-claude-code-approval-kept:
|
|
162
|
+
name: scan — Claude Code permissions within declared autonomy (must PASS)
|
|
163
|
+
runs-on: ubuntu-latest
|
|
164
|
+
steps:
|
|
165
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
166
|
+
- uses: ./
|
|
167
|
+
with:
|
|
168
|
+
mode: scan
|
|
169
|
+
directory: examples/repos/claude-code-approval-kept
|
|
170
|
+
manifest: examples/repos/claude-code-approval-kept/agent-assurance.yaml
|
|
171
|
+
fail-on: review
|
|
172
|
+
|
|
173
|
+
scan-claude-code-approval-broken:
|
|
174
|
+
name: scan — allow Bash(*) with declared L2 (must BLOCK)
|
|
175
|
+
runs-on: ubuntu-latest
|
|
176
|
+
steps:
|
|
177
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
178
|
+
- id: gate
|
|
179
|
+
continue-on-error: true
|
|
180
|
+
uses: ./
|
|
181
|
+
with:
|
|
182
|
+
mode: scan
|
|
183
|
+
directory: examples/repos/claude-code-approval-broken
|
|
184
|
+
manifest: examples/repos/claude-code-approval-broken/agent-assurance.yaml
|
|
185
|
+
- name: Assert the gate tripped
|
|
186
|
+
if: steps.gate.outcome != 'failure'
|
|
187
|
+
run: |
|
|
188
|
+
echo "::error::scan did NOT block an auto-approved shell under a declared L2 (outcome=${{ steps.gate.outcome }})"
|
|
189
|
+
exit 1
|
|
190
|
+
|
|
191
|
+
scan-policy-org:
|
|
192
|
+
name: scan — organisation policy (own catalogue, stricter gate; must PASS with fail-on=fail)
|
|
193
|
+
runs-on: ubuntu-latest
|
|
194
|
+
steps:
|
|
195
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
196
|
+
- uses: ./
|
|
197
|
+
with:
|
|
198
|
+
mode: scan
|
|
199
|
+
directory: examples/repos/policy-org
|
|
200
|
+
manifest: examples/repos/policy-org/agent-assurance.yaml
|
|
201
|
+
card: aa-card.html
|
|
202
|
+
- name: Assert the policy was recorded in the summary
|
|
203
|
+
run: grep -q "acme-corp-2026-09" aa-report.md && test -s aa-card.html
|
|
204
|
+
|
|
205
|
+
scan-codex-and-tools:
|
|
206
|
+
name: scan — Codex config.toml + serialized tool definitions (must BLOCK on the token, review the guess)
|
|
207
|
+
runs-on: ubuntu-latest
|
|
208
|
+
steps:
|
|
209
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
210
|
+
- id: gate
|
|
211
|
+
continue-on-error: true
|
|
212
|
+
uses: ./
|
|
213
|
+
with:
|
|
214
|
+
mode: scan
|
|
215
|
+
directory: examples/repos/codex-and-tools
|
|
216
|
+
manifest: examples/repos/codex-and-tools/agent-assurance.yaml
|
|
217
|
+
- name: Assert the gate tripped and the inferred class stayed a review
|
|
218
|
+
if: steps.gate.outcome != 'failure'
|
|
219
|
+
run: |
|
|
220
|
+
echo "::error::scan did NOT block an undeclared credential from .codex/config.toml"; exit 1
|
|
221
|
+
- run: grep -q "inferred from its name" aa-report.md
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
strategy:
|
|
15
|
+
fail-fast: false
|
|
16
|
+
matrix:
|
|
17
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
20
|
+
|
|
21
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
22
|
+
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python-version }}
|
|
25
|
+
|
|
26
|
+
- name: Install
|
|
27
|
+
run: pip install -e ".[dev]"
|
|
28
|
+
|
|
29
|
+
- name: Lint (ruff)
|
|
30
|
+
run: ruff check src/ tests/
|
|
31
|
+
|
|
32
|
+
- name: Test (pytest)
|
|
33
|
+
run: pytest -q
|
|
34
|
+
|
|
35
|
+
- name: Exit-code contract with the installed binary
|
|
36
|
+
# 0 = pass, 1 = gate tripped, 2 = usage error. The Action depends on this.
|
|
37
|
+
run: |
|
|
38
|
+
agent-assurance --version
|
|
39
|
+
agent-assurance validate examples/safe-agent.yaml
|
|
40
|
+
agent-assurance check blast-radius examples/safe-agent.yaml
|
|
41
|
+
set +e
|
|
42
|
+
agent-assurance check all examples/dangerous-agent.yaml > /dev/null; c1=$?
|
|
43
|
+
agent-assurance check all examples/high-agent.yaml --fail-on review > /dev/null; c2=$?
|
|
44
|
+
agent-assurance check all examples/does-not-exist.yaml > /dev/null 2>&1; c3=$?
|
|
45
|
+
agent-assurance scan examples/repos/mcp-promise-kept > /dev/null; c4=$?
|
|
46
|
+
agent-assurance scan examples/repos/mcp-promise-broken > /dev/null; c5=$?
|
|
47
|
+
agent-assurance scan examples/repos/mcp-unknown-server --fail-on review > /dev/null; c6=$?
|
|
48
|
+
agent-assurance scan examples/repos/claude-code-approval-kept --fail-on review > /dev/null; c7=$?
|
|
49
|
+
agent-assurance scan examples/repos/claude-code-approval-broken > /dev/null; c8=$?
|
|
50
|
+
agent-assurance diff examples/repos/mcp-promise-kept examples/repos/mcp-promise-kept --fail-on-delta > /dev/null; c9=$?
|
|
51
|
+
agent-assurance diff examples/repos/mcp-promise-kept examples/repos/mcp-promise-broken --fail-on-delta > /dev/null; c10=$?
|
|
52
|
+
agent-assurance scan examples/repos/codex-and-tools > /dev/null; c11=$?
|
|
53
|
+
agent-assurance scan examples/repos/policy-org > /dev/null; c12=$?
|
|
54
|
+
set -e
|
|
55
|
+
echo "dangerous=$c1 high/review=$c2 missing=$c3 kept=$c4 broken=$c5 unknown/review=$c6 cc-kept=$c7 cc-broken=$c8 diff-same=$c9 diff-broken=$c10"
|
|
56
|
+
test "$c1" -eq 1 && test "$c2" -eq 1 && test "$c3" -eq 2
|
|
57
|
+
test "$c4" -eq 0 && test "$c5" -eq 1 && test "$c6" -eq 1
|
|
58
|
+
test "$c7" -eq 0 && test "$c8" -eq 1
|
|
59
|
+
test "$c9" -eq 0 && test "$c10" -eq 1
|
|
60
|
+
test "$c11" -eq 1 && test "$c12" -eq 0
|
|
61
|
+
|
|
62
|
+
- name: SARIF from the installed binary is valid JSON with a real location
|
|
63
|
+
run: |
|
|
64
|
+
agent-assurance check all examples/dangerous-agent.yaml --format sarif -o aa.sarif || true
|
|
65
|
+
python3 -c "import json; s=json.load(open('aa.sarif')); r=s['runs'][0]['results'][0]; assert r['locations'][0]['physicalLocation']['artifactLocation']['uri']=='examples/dangerous-agent.yaml'; print('ok')"
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
name: CodeQL
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, develop]
|
|
6
|
+
pull_request:
|
|
7
|
+
schedule:
|
|
8
|
+
- cron: "23 5 * * 2"
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
analyze:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
permissions:
|
|
17
|
+
security-events: write
|
|
18
|
+
contents: read
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
21
|
+
- uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
|
|
22
|
+
with:
|
|
23
|
+
languages: python
|
|
24
|
+
queries: security-and-quality
|
|
25
|
+
- uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Living demo of `mode: diff`: on every pull request, report what the change
|
|
2
|
+
# does to examples/demo/analytics-helper (a copy of the promise-kept fixture kept apart from the tests). A PR that widens that repo's reach
|
|
3
|
+
# turns red and gets a comment; see the open demo PR.
|
|
4
|
+
name: Agent Assurance (PR diff demo)
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
pull_request:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
pull-requests: write
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
diff-promise-kept:
|
|
15
|
+
name: diff — examples/demo/analytics-helper
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
19
|
+
- uses: ./
|
|
20
|
+
with:
|
|
21
|
+
mode: diff
|
|
22
|
+
directory: examples/demo/analytics-helper
|
|
23
|
+
manifest: examples/demo/analytics-helper/agent-assurance.yaml
|
|
24
|
+
comment: "true"
|
|
25
|
+
fail-on-delta: "true"
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Release: on a v* tag, build sdist + wheel, attest their provenance (Sigstore),
|
|
2
|
+
# attach them to the GitHub release and publish to PyPI via Trusted Publishing
|
|
3
|
+
# (no API token stored anywhere; PyPI trusts this workflow's OIDC identity).
|
|
4
|
+
name: Release
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
tags: ["v*.*.*"]
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
build:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
permissions:
|
|
17
|
+
contents: write # attach artifacts to the release
|
|
18
|
+
id-token: write # Sigstore
|
|
19
|
+
attestations: write
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
22
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
23
|
+
with:
|
|
24
|
+
python-version: "3.12"
|
|
25
|
+
- run: pip install build && python -m build
|
|
26
|
+
- name: Tag must match the package version
|
|
27
|
+
run: |
|
|
28
|
+
v=$(python -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
29
|
+
test "v$v" = "${GITHUB_REF_NAME}" || { echo "::error::tag ${GITHUB_REF_NAME} != version $v"; exit 1; }
|
|
30
|
+
- uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
|
|
31
|
+
with:
|
|
32
|
+
subject-path: dist/*
|
|
33
|
+
- name: Attach to the GitHub release
|
|
34
|
+
env:
|
|
35
|
+
GH_TOKEN: ${{ github.token }}
|
|
36
|
+
run: gh release upload "${GITHUB_REF_NAME}" dist/* --clobber
|
|
37
|
+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
38
|
+
with:
|
|
39
|
+
name: dist
|
|
40
|
+
path: dist/
|
|
41
|
+
|
|
42
|
+
publish:
|
|
43
|
+
needs: build
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
environment: pypi # create it in Settings > Environments; PyPI trusted publisher points here
|
|
46
|
+
permissions:
|
|
47
|
+
id-token: write
|
|
48
|
+
steps:
|
|
49
|
+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
50
|
+
with:
|
|
51
|
+
name: dist
|
|
52
|
+
path: dist/
|
|
53
|
+
- uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# OpenSSF Scorecard: supply-chain hygiene of this repo, published to the
|
|
2
|
+
# Scorecard API and to the Security tab. Results: https://scorecard.dev/viewer/?uri=github.com/kunko-ai-labs/agent-assurance
|
|
3
|
+
name: OpenSSF Scorecard
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches: [main]
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: "17 6 * * 1"
|
|
10
|
+
workflow_dispatch:
|
|
11
|
+
|
|
12
|
+
permissions: read-all
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
analysis:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
permissions:
|
|
18
|
+
security-events: write
|
|
19
|
+
id-token: write
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
22
|
+
with:
|
|
23
|
+
persist-credentials: false
|
|
24
|
+
- uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
|
|
25
|
+
with:
|
|
26
|
+
results_file: results.sarif
|
|
27
|
+
results_format: sarif
|
|
28
|
+
publish_results: true
|
|
29
|
+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
30
|
+
with:
|
|
31
|
+
name: scorecard
|
|
32
|
+
path: results.sarif
|
|
33
|
+
- uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
|
|
34
|
+
with:
|
|
35
|
+
sarif_file: results.sarif
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
build/
|
|
6
|
+
dist/
|
|
7
|
+
.venv/
|
|
8
|
+
venv/
|
|
9
|
+
|
|
10
|
+
# Tooling caches
|
|
11
|
+
.pytest_cache/
|
|
12
|
+
.ruff_cache/
|
|
13
|
+
.mypy_cache/
|
|
14
|
+
|
|
15
|
+
# Editor / OS
|
|
16
|
+
.DS_Store
|
|
17
|
+
.idea/
|
|
18
|
+
.vscode/
|
|
19
|
+
|
|
20
|
+
# Local outputs
|
|
21
|
+
*.sarif
|
|
22
|
+
aa-report.md
|
|
23
|
+
docs/.frames/
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Use from another repo's .pre-commit-config.yaml:
|
|
2
|
+
# - repo: https://github.com/kunko-ai-labs/agent-assurance
|
|
3
|
+
# rev: v0.3.0
|
|
4
|
+
# hooks:
|
|
5
|
+
# - id: agent-assurance-scan
|
|
6
|
+
- id: agent-assurance-scan
|
|
7
|
+
name: agent-assurance scan (declared vs observed)
|
|
8
|
+
description: Verify the agent configuration against agent-assurance.yaml before it is committed.
|
|
9
|
+
entry: agent-assurance scan .
|
|
10
|
+
language: python
|
|
11
|
+
pass_filenames: false
|
|
12
|
+
files: ^(\.mcp\.json|\.claude/settings(\.local)?\.json|\.cursor/mcp\.json|\.vscode/mcp\.json|\.gemini/settings\.json|agent-assurance\.yaml)$
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# agent-assurance — reglas del proyecto
|
|
2
|
+
|
|
3
|
+
## Qué es (y qué no)
|
|
4
|
+
|
|
5
|
+
**Declarado vs observado.** El manifiesto (`agent-assurance.yaml`) es la **promesa** de lo que un agente de IA puede hacer; `scan` **observa** lo que la configuración del repo concede de verdad (configs MCP de Claude Code/Cursor/Gemini/VS Code y permisos de Claude Code) y AA-002 falla si la promesa se rompe, señalando fichero:línea. `diff` dice qué hizo *este cambio*. AA-001 mide el radio de impacto. Tres puntos de aplicación con el mismo motor: al editar (hook / servidor MCP `would_break`), en el PR (Action `mode: diff` con comentario) y en cada push/release (SARIF).
|
|
6
|
+
|
|
7
|
+
Modelo de permisos de Claude Code: `allow` no concede capacidad, **quita al humano del bucle** (aprobación); las reglas se colapsan por clase; los comandos de solo lectura son `read`; una regla acotada es `scoped` y no rompe la promesa (solo *review*).
|
|
8
|
+
|
|
9
|
+
Por qué esto y no "diff de permisos": ya existe ×5 sin tracción (`docs/landscape.md`). Nadie verifica promesas ni mapea a OWASP.
|
|
10
|
+
|
|
11
|
+
No es: un motor en runtime, un interceptor de tool calls, un ledger, un sistema de aprobaciones. Eso es otro producto (carril empresa de Kunko AI Labs); aquí no entra código de eso.
|
|
12
|
+
|
|
13
|
+
## Contratos que no se rompen sin avisar
|
|
14
|
+
|
|
15
|
+
- **Exit codes**: `0` pass/review · `1` gate · `2` uso/manifiesto inválido. Están en `tests/test_cli_contract.py` y en `.github/workflows/ci.yml`. La Action depende de ellos.
|
|
16
|
+
- **Riesgo reproducible a mano**: cada punto sale de `risk.py` con su motivo. Nunca un LLM en el veredicto.
|
|
17
|
+
- **SARIF**: apunta al fichero real, con región; PASS sale como `kind: pass` (sin alerta).
|
|
18
|
+
- **Estándares**: mapeo a OWASP ASI; lo que viene de APTS se etiqueta `(adapted)`. No se reclama conformidad que no existe.
|
|
19
|
+
- **Lo no soportado es `UNKNOWN`**, nunca inferido. Un `UNKNOWN` nunca da PASS silencioso (AA-001 ≥ REVIEW; AA-002 "not verifiable").
|
|
20
|
+
- **El escáner no ejecuta nada**: ni servidores MCP ni código del repo; nunca lee el valor de un secreto, solo el nombre de la variable.
|
|
21
|
+
- **Catálogo de servidores MCP** (`scan/catalog.py`) con fuente por entrada; añadir un servidor = una entrada + su fuente.
|
|
22
|
+
|
|
23
|
+
## Cómo se trabaja
|
|
24
|
+
|
|
25
|
+
- Entorno: `.venv` con `pip install -e ".[dev]"`. Antes de commitear: `ruff check src tests && pytest -q`.
|
|
26
|
+
- El CI de dogfood (`assurance.yml`) es una **alarma**: cada job afirma que la Action pasa lo que debe y bloquea lo que debe. No añadir `continue-on-error` sin un paso que afirme el resultado esperado.
|
|
27
|
+
- Una unidad shippable por semana con demo de 30 s. Si una feature no cabe en una demo, está mal cortada.
|
|
28
|
+
- Ejemplos en `examples/` cubren las cuatro bandas (safe=LOW, medium=MEDIUM, high=HIGH, dangerous=CRITICAL); `examples/repos/` cubre promesa cumplida / rota / no verificable. Un check o un scanner nuevo añade su fixture y su job en `assurance.yml`.
|
|
29
|
+
- Antes de una feature nueva: pasada corta de mercado con fuente y fecha (`docs/landscape.md`), y proponer la versión que nadie ocupa.
|
|
30
|
+
- Commits en inglés, imperativo, prefijo `feat:|fix:|ci:|docs:|test:`.
|
|
31
|
+
- Nombres de paso en `action.yml` con `:` van entre comillas (un YAML roto tumba todos los jobs); `tests/test_action_yaml.py` lo comprueba.
|
|
32
|
+
- El GIF del README se regenera con `vhs docs/demo.tape` + el ffmpeg del comentario del tape; VHS necesita ejecutarse fuera del sandbox.
|
|
33
|
+
- Probar en repos ajenos reales antes de cada release (17/09: ocho repos; salieron tres falsos positivos que un senior habría rechazado).
|
|
34
|
+
|
|
35
|
+
## Roadmap vigente
|
|
36
|
+
|
|
37
|
+
Está en el README (§Roadmap). Las decisiones de programa viven en el repo privado `vmbb13/ideas-ia-plan-personal` (doc 12). Si algo aquí contradice a ese doc, manda el doc y se abre issue.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
## What lives where
|
|
4
|
+
|
|
5
|
+
| Path | Audience | What |
|
|
6
|
+
|---|---|---|
|
|
7
|
+
| `src/agent_assurance/` | developers | the product: scanners, checks, risk model, reports, CLI, MCP server |
|
|
8
|
+
| `action.yml`, `.pre-commit-hooks.yaml`, `contrib/` | users | ways to run it (GitHub Action, pre-commit, Claude Code hook/MCP) |
|
|
9
|
+
| `examples/` | users, tests | manifests and mini-repos that double as fixtures and demos |
|
|
10
|
+
| `schema/` | users | JSON schemas for the manifest and the policy file |
|
|
11
|
+
| `docs/` | users, buyers, auditors | how it works, integrations, roadmap, real-world scans, landscape, images |
|
|
12
|
+
| `tests/` | developers | contract tests (exit codes, SARIF, YAML shape, every fixture's verdict) |
|
|
13
|
+
|
|
14
|
+
Launch material (video, composition, share copy) is **not** in this repo; the video is attached to the release.
|
|
15
|
+
|
|
16
|
+
## Branches
|
|
17
|
+
|
|
18
|
+
- `main` — releasable at all times. Only merges from `develop` (or hotfixes). Every release is a tag `vX.Y.Z` plus a moving `vX.Y` for the Action.
|
|
19
|
+
- `develop` — integration branch. Day-to-day work lands here through feature branches.
|
|
20
|
+
- `feat/*`, `fix/*`, `docs/*` — short-lived; open a PR into `develop`. The CI (`ci.yml`) and the dogfood (`assurance.yml`) run on every PR.
|
|
21
|
+
- `demo/*` — living demos that must stay red (see PR #16). Never merged.
|
|
22
|
+
|
|
23
|
+
Release: `develop` → PR → `main` → bump version in `pyproject.toml` and `__init__.py` → tag → GitHub release with notes → move the `vX.Y` tag.
|
|
24
|
+
|
|
25
|
+
## Before you push
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install -e ".[dev]"
|
|
29
|
+
ruff check src tests && pytest -q
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Rules that do not bend (see `CLAUDE.md`): no LLM in the verdict; nothing executed or sent; unknown is `UNKNOWN`; exit codes `0/1/2` are a contract; a step name with `:` in `action.yml` goes in quotes; a new scanner or check comes with a fixture in `examples/repos/`, a test, and a job in `assurance.yml`; a new MCP catalogue entry cites its source.
|
|
33
|
+
|
|
34
|
+
Before a release: scan a few real public repos (`docs/real-world.md`) and re-check `docs/landscape.md`.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
Copyright 2026 Kunko AI Labs
|
|
6
|
+
|
|
7
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
you may not use this file except in compliance with the License.
|
|
9
|
+
You may obtain a copy of the License at
|
|
10
|
+
|
|
11
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
|
|
13
|
+
Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
See the License for the specific language governing permissions and
|
|
17
|
+
limitations under the License.
|