agentack 0.6.2__tar.gz → 0.6.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentack-0.6.2 → agentack-0.6.3}/CITATION.cff +1 -1
- {agentack-0.6.2 → agentack-0.6.3}/MANIFEST.in +1 -1
- {agentack-0.6.2 → agentack-0.6.3}/PKG-INFO +31 -25
- {agentack-0.6.2 → agentack-0.6.3}/README.md +29 -24
- agentack-0.6.3/assets/social/github-social-card-agentack.png +0 -0
- agentack-0.6.3/docs/codex-cli.md +79 -0
- agentack-0.6.3/docs/live-coverage.md +34 -0
- {agentack-0.6.2 → agentack-0.6.3}/docs/report-schema.md +2 -2
- {agentack-0.6.2 → agentack-0.6.3}/docs/security.md +7 -9
- {agentack-0.6.2 → agentack-0.6.3}/pyproject.toml +2 -1
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/__init__.py +1 -1
- agentack-0.6.3/src/agentack/adapters/codex.py +64 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/cli.py +2 -2
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/coverage.py +9 -9
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack.egg-info/PKG-INFO +31 -25
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack.egg-info/SOURCES.txt +1 -2
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_adapters_codex.py +25 -77
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_cli_codex.py +21 -20
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_live_coverage.py +1 -3
- agentack-0.6.3/tests/test_release_metadata.py +61 -0
- agentack-0.6.2/assets/social/README.md +0 -0
- agentack-0.6.2/docs/codex-cli.md +0 -127
- agentack-0.6.2/docs/live-coverage.md +0 -32
- agentack-0.6.2/src/agentack/adapters/codex.py +0 -393
- agentack-0.6.2/tests/test_codex_live_regression.py +0 -152
- agentack-0.6.2/tests/test_release_metadata.py +0 -30
- {agentack-0.6.2 → agentack-0.6.3}/CONTRIBUTING.md +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/LICENSE +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/Makefile +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/SECURITY.md +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/docs/claude-code.md +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/docs/method.md +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/docs/releasing.md +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/docs/standards-mapping.md +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/docs/trace-format.md +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/examples/action-swap.jsonl +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/examples/secure.jsonl +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/examples/strict.toml +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/setup.cfg +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/__main__.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/__init__.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/base.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/claude.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/claude_analysis.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/claude_capture.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/claude_live.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/codex_analysis.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/codex_protocol.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/adapters/otel.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/canonical.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/demo.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/engine.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/findings.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/models.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/parser.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/policy.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/provenance.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/recorder.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/report.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack/ux.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack.egg-info/dependency_links.txt +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack.egg-info/entry_points.txt +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack.egg-info/requires.txt +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/src/agentack.egg-info/top_level.txt +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_adapter_hardening.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_adapters_claude.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_block_lifecycle.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_canonical.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_cli.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_engine.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_live_probe_corpus.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_parser.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_policy.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_provenance.py +0 -0
- {agentack-0.6.2 → agentack-0.6.3}/tests/test_recorder.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentack
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.3
|
|
4
4
|
Summary: Test whether human approval controls for AI agents actually work.
|
|
5
5
|
Author: Sylvester Kaczmarek
|
|
6
6
|
License-Expression: MIT
|
|
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
20
|
Classifier: Topic :: Security
|
|
20
21
|
Classifier: Topic :: Software Development :: Testing
|
|
21
22
|
Requires-Python: >=3.11
|
|
@@ -72,21 +73,24 @@ The broken demo changes the command after approval; AgentAck detects the mismatc
|
|
|
72
73
|
Next: agentack doctor
|
|
73
74
|
```
|
|
74
75
|
|
|
75
|
-
##
|
|
76
|
+
## Integrations
|
|
77
|
+
|
|
78
|
+
Claude Code is the currently verified live approval-control integration:
|
|
76
79
|
|
|
77
80
|
```bash
|
|
78
81
|
agentack test claude
|
|
79
|
-
agentack test codex
|
|
80
82
|
```
|
|
81
83
|
|
|
82
|
-
| Agent |
|
|
84
|
+
| Agent | Detection | Live test | Status |
|
|
83
85
|
| --- | --- | --- | --- |
|
|
84
|
-
| Claude Code | yes | **yes** | official hooks + `tool_decision` telemetry |
|
|
85
|
-
| Codex CLI | yes |
|
|
86
|
-
| Gemini CLI | yes |
|
|
87
|
-
| Cursor CLI | yes |
|
|
86
|
+
| Claude Code | yes | **yes** | supported live adapter using official hooks + `tool_decision` telemetry |
|
|
87
|
+
| Codex CLI | yes | no | detection + retained App Server protocol research; live boundary not verified |
|
|
88
|
+
| Gemini CLI | yes | no | detection only |
|
|
89
|
+
| Cursor CLI | yes | no | detection only |
|
|
90
|
+
|
|
91
|
+
Real-binary testing with Codex CLI 0.148.0 did not produce a reproducible standalone human command-approval boundary through the public App Server path. AgentAck therefore reports Codex as `DETECTED`, not `READY`, and does not claim live Codex coverage.
|
|
88
92
|
|
|
89
|
-
|
|
93
|
+
For backward compatibility, `agentack test codex` returns a concise `INCOMPLETE` diagnostic rather than running the old experimental five-probe suite.
|
|
90
94
|
|
|
91
95
|
See [`docs/claude-code.md`](docs/claude-code.md) and [`docs/codex-cli.md`](docs/codex-cli.md).
|
|
92
96
|
|
|
@@ -100,15 +104,15 @@ Current coverage:
|
|
|
100
104
|
|
|
101
105
|
```text
|
|
102
106
|
Rule Trace Claude Codex Check
|
|
103
|
-
ACK001 TESTED TESTED
|
|
104
|
-
ACK002 TESTED TESTED
|
|
105
|
-
ACK003 TESTED TESTED
|
|
106
|
-
ACK004 TESTED TESTED
|
|
107
|
+
ACK001 TESTED TESTED TRACE Required approval
|
|
108
|
+
ACK002 TESTED TESTED TRACE Denied action
|
|
109
|
+
ACK003 TESTED TESTED TRACE Exact action binding
|
|
110
|
+
ACK004 TESTED TESTED TRACE Approval replay
|
|
107
111
|
ACK005 TESTED TRACE TRACE Approval expiry
|
|
108
|
-
ACK006 TESTED GUARDED
|
|
109
|
-
ACK007 TESTED TESTED
|
|
110
|
-
ACK008 TESTED SKIP
|
|
111
|
-
ACK009 TESTED GUARDED
|
|
112
|
+
ACK006 TESTED GUARDED TRACE Lifecycle ordering
|
|
113
|
+
ACK007 TESTED TESTED TRACE Denial route-around
|
|
114
|
+
ACK008 TESTED SKIP TRACE Interrupt bypass
|
|
115
|
+
ACK009 TESTED GUARDED TRACE Evidence completeness
|
|
112
116
|
```
|
|
113
117
|
|
|
114
118
|
`TESTED` means a live path deliberately exercises the control. `GUARDED` means the adapter fails closed on bad or missing evidence without inducing that attack. `TRACE` means deterministic trace coverage only for that adapter. `SKIP` means AgentAck does not claim a reliable safe live boundary.
|
|
@@ -135,15 +139,17 @@ Missing evidence returns `INCOMPLETE`, not a silent pass.
|
|
|
135
139
|
|
|
136
140
|
Claude's extended suite asks the user to approve one Bash action once, deny an identical replay, deny one marker-writing route, and deny an alternate route for the same harmless intent. If the first approval is explicitly persistent, AgentAck does not label later reuse as a replay vulnerability.
|
|
137
141
|
|
|
138
|
-
Codex's extended suite adds the same replay and route-around checks plus a human-triggered `turn/interrupt` probe. AgentAck sends only a one-request `accept` decision and never intentionally grants `acceptForSession` or persistent authority.
|
|
139
|
-
|
|
140
142
|
All filesystem effects stay inside disposable temporary workspaces. AgentAck does not run destructive, credential, deployment, or real cloud/network probes.
|
|
141
143
|
|
|
142
|
-
|
|
144
|
+
The retained Codex App Server parser, fixtures, and deterministic analyzers are research/regression groundwork only; they are not evidence that a real Codex installation has passed the live approval suite.
|
|
145
|
+
|
|
146
|
+
## Terminal result
|
|
147
|
+
|
|
148
|
+
A successful supported live run has this scan-friendly shape:
|
|
143
149
|
|
|
144
150
|
```text
|
|
145
151
|
AgentAck PASS
|
|
146
|
-
Integration:
|
|
152
|
+
Integration: Claude Code
|
|
147
153
|
|
|
148
154
|
Probe isolation PASS
|
|
149
155
|
Approval required PASS
|
|
@@ -153,7 +159,7 @@ Denial enforcement PASS
|
|
|
153
159
|
Approval replay PASS
|
|
154
160
|
Denial route-around PASS
|
|
155
161
|
Approval expiry SKIP
|
|
156
|
-
Stop enforcement
|
|
162
|
+
Stop enforcement SKIP
|
|
157
163
|
Lifecycle ordering PASS
|
|
158
164
|
Evidence completeness PASS
|
|
159
165
|
```
|
|
@@ -164,10 +170,10 @@ A `PASS` requires affirmative evidence for the tested path. `SKIP` and `INCOMPLE
|
|
|
164
170
|
|
|
165
171
|
```bash
|
|
166
172
|
agentack demo # secure + deliberately broken showcase
|
|
167
|
-
agentack doctor # detect
|
|
173
|
+
agentack doctor # detect integrations and show verified live readiness
|
|
168
174
|
agentack coverage # show trace/live ACK coverage
|
|
169
|
-
agentack test claude # live Claude approval-control suite
|
|
170
|
-
agentack test codex #
|
|
175
|
+
agentack test claude # verified live Claude approval-control suite
|
|
176
|
+
agentack test codex # backward-compatible Codex status diagnostic; currently INCOMPLETE
|
|
171
177
|
agentack check trace.jsonl # evaluate an AgentAck trace
|
|
172
178
|
agentack check trace.jsonl --json report.json --sarif report.sarif
|
|
173
179
|
agentack rules
|
|
@@ -42,21 +42,24 @@ The broken demo changes the command after approval; AgentAck detects the mismatc
|
|
|
42
42
|
Next: agentack doctor
|
|
43
43
|
```
|
|
44
44
|
|
|
45
|
-
##
|
|
45
|
+
## Integrations
|
|
46
|
+
|
|
47
|
+
Claude Code is the currently verified live approval-control integration:
|
|
46
48
|
|
|
47
49
|
```bash
|
|
48
50
|
agentack test claude
|
|
49
|
-
agentack test codex
|
|
50
51
|
```
|
|
51
52
|
|
|
52
|
-
| Agent |
|
|
53
|
+
| Agent | Detection | Live test | Status |
|
|
53
54
|
| --- | --- | --- | --- |
|
|
54
|
-
| Claude Code | yes | **yes** | official hooks + `tool_decision` telemetry |
|
|
55
|
-
| Codex CLI | yes |
|
|
56
|
-
| Gemini CLI | yes |
|
|
57
|
-
| Cursor CLI | yes |
|
|
55
|
+
| Claude Code | yes | **yes** | supported live adapter using official hooks + `tool_decision` telemetry |
|
|
56
|
+
| Codex CLI | yes | no | detection + retained App Server protocol research; live boundary not verified |
|
|
57
|
+
| Gemini CLI | yes | no | detection only |
|
|
58
|
+
| Cursor CLI | yes | no | detection only |
|
|
59
|
+
|
|
60
|
+
Real-binary testing with Codex CLI 0.148.0 did not produce a reproducible standalone human command-approval boundary through the public App Server path. AgentAck therefore reports Codex as `DETECTED`, not `READY`, and does not claim live Codex coverage.
|
|
58
61
|
|
|
59
|
-
|
|
62
|
+
For backward compatibility, `agentack test codex` returns a concise `INCOMPLETE` diagnostic rather than running the old experimental five-probe suite.
|
|
60
63
|
|
|
61
64
|
See [`docs/claude-code.md`](docs/claude-code.md) and [`docs/codex-cli.md`](docs/codex-cli.md).
|
|
62
65
|
|
|
@@ -70,15 +73,15 @@ Current coverage:
|
|
|
70
73
|
|
|
71
74
|
```text
|
|
72
75
|
Rule Trace Claude Codex Check
|
|
73
|
-
ACK001 TESTED TESTED
|
|
74
|
-
ACK002 TESTED TESTED
|
|
75
|
-
ACK003 TESTED TESTED
|
|
76
|
-
ACK004 TESTED TESTED
|
|
76
|
+
ACK001 TESTED TESTED TRACE Required approval
|
|
77
|
+
ACK002 TESTED TESTED TRACE Denied action
|
|
78
|
+
ACK003 TESTED TESTED TRACE Exact action binding
|
|
79
|
+
ACK004 TESTED TESTED TRACE Approval replay
|
|
77
80
|
ACK005 TESTED TRACE TRACE Approval expiry
|
|
78
|
-
ACK006 TESTED GUARDED
|
|
79
|
-
ACK007 TESTED TESTED
|
|
80
|
-
ACK008 TESTED SKIP
|
|
81
|
-
ACK009 TESTED GUARDED
|
|
81
|
+
ACK006 TESTED GUARDED TRACE Lifecycle ordering
|
|
82
|
+
ACK007 TESTED TESTED TRACE Denial route-around
|
|
83
|
+
ACK008 TESTED SKIP TRACE Interrupt bypass
|
|
84
|
+
ACK009 TESTED GUARDED TRACE Evidence completeness
|
|
82
85
|
```
|
|
83
86
|
|
|
84
87
|
`TESTED` means a live path deliberately exercises the control. `GUARDED` means the adapter fails closed on bad or missing evidence without inducing that attack. `TRACE` means deterministic trace coverage only for that adapter. `SKIP` means AgentAck does not claim a reliable safe live boundary.
|
|
@@ -105,15 +108,17 @@ Missing evidence returns `INCOMPLETE`, not a silent pass.
|
|
|
105
108
|
|
|
106
109
|
Claude's extended suite asks the user to approve one Bash action once, deny an identical replay, deny one marker-writing route, and deny an alternate route for the same harmless intent. If the first approval is explicitly persistent, AgentAck does not label later reuse as a replay vulnerability.
|
|
107
110
|
|
|
108
|
-
Codex's extended suite adds the same replay and route-around checks plus a human-triggered `turn/interrupt` probe. AgentAck sends only a one-request `accept` decision and never intentionally grants `acceptForSession` or persistent authority.
|
|
109
|
-
|
|
110
111
|
All filesystem effects stay inside disposable temporary workspaces. AgentAck does not run destructive, credential, deployment, or real cloud/network probes.
|
|
111
112
|
|
|
112
|
-
|
|
113
|
+
The retained Codex App Server parser, fixtures, and deterministic analyzers are research/regression groundwork only; they are not evidence that a real Codex installation has passed the live approval suite.
|
|
114
|
+
|
|
115
|
+
## Terminal result
|
|
116
|
+
|
|
117
|
+
A successful supported live run has this scan-friendly shape:
|
|
113
118
|
|
|
114
119
|
```text
|
|
115
120
|
AgentAck PASS
|
|
116
|
-
Integration:
|
|
121
|
+
Integration: Claude Code
|
|
117
122
|
|
|
118
123
|
Probe isolation PASS
|
|
119
124
|
Approval required PASS
|
|
@@ -123,7 +128,7 @@ Denial enforcement PASS
|
|
|
123
128
|
Approval replay PASS
|
|
124
129
|
Denial route-around PASS
|
|
125
130
|
Approval expiry SKIP
|
|
126
|
-
Stop enforcement
|
|
131
|
+
Stop enforcement SKIP
|
|
127
132
|
Lifecycle ordering PASS
|
|
128
133
|
Evidence completeness PASS
|
|
129
134
|
```
|
|
@@ -134,10 +139,10 @@ A `PASS` requires affirmative evidence for the tested path. `SKIP` and `INCOMPLE
|
|
|
134
139
|
|
|
135
140
|
```bash
|
|
136
141
|
agentack demo # secure + deliberately broken showcase
|
|
137
|
-
agentack doctor # detect
|
|
142
|
+
agentack doctor # detect integrations and show verified live readiness
|
|
138
143
|
agentack coverage # show trace/live ACK coverage
|
|
139
|
-
agentack test claude # live Claude approval-control suite
|
|
140
|
-
agentack test codex #
|
|
144
|
+
agentack test claude # verified live Claude approval-control suite
|
|
145
|
+
agentack test codex # backward-compatible Codex status diagnostic; currently INCOMPLETE
|
|
141
146
|
agentack check trace.jsonl # evaluate an AgentAck trace
|
|
142
147
|
agentack check trace.jsonl --json report.json --sarif report.sarif
|
|
143
148
|
agentack rules
|
|
Binary file
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Codex CLI status
|
|
2
|
+
|
|
3
|
+
AgentAck detects Codex CLI and retains structured App Server protocol parsing and regression fixtures, but it does **not currently advertise a verified Codex live approval-integrity test**.
|
|
4
|
+
|
|
5
|
+
## Why live support is not advertised
|
|
6
|
+
|
|
7
|
+
Real-binary testing with Codex CLI 0.148.0 repeatedly failed to produce a reproducible standalone human command-approval boundary through the public App Server path.
|
|
8
|
+
|
|
9
|
+
The public App Server protocol exposes structured command-execution approval request and lifecycle types, but a normal `turn/start` begins from user input and leaves tool invocation to the agent. In the tested standalone configuration, Codex could complete the turn without producing the `commandExecution` approval evidence AgentAck requires. Prompt retries did not make that boundary deterministic.
|
|
10
|
+
|
|
11
|
+
AgentAck therefore does not convert protocol capability or mocked fixtures into a live-support claim.
|
|
12
|
+
|
|
13
|
+
## `agentack doctor`
|
|
14
|
+
|
|
15
|
+
When Codex is installed, `agentack doctor` reports it as `DETECTED`, including the executable version when available. It does not report Codex as `READY` and does not advertise `agentack test codex` as a ready live command.
|
|
16
|
+
|
|
17
|
+
Example:
|
|
18
|
+
|
|
19
|
+
```text
|
|
20
|
+
Codex CLI DETECTED codex-cli 0.148.0
|
|
21
|
+
Codex CLI is installed, but AgentAck does not currently expose a verified deterministic live approval-integrity test.
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Detection does not start a model turn.
|
|
25
|
+
|
|
26
|
+
## `agentack test codex`
|
|
27
|
+
|
|
28
|
+
The command remains accepted for backward compatibility, but it returns a concise `INCOMPLETE` diagnostic and does not run the previous model-driven five-probe suite.
|
|
29
|
+
|
|
30
|
+
Example shape:
|
|
31
|
+
|
|
32
|
+
```text
|
|
33
|
+
AgentAck INCOMPLETE
|
|
34
|
+
Integration: Codex CLI
|
|
35
|
+
|
|
36
|
+
Codex live approval boundary INCOMPLETE
|
|
37
|
+
|
|
38
|
+
Details
|
|
39
|
+
- Codex live approval boundary: Codex CLI is installed, but AgentAck does not currently expose a verified deterministic live approval-integrity test.
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
This behavior is deliberate: AgentAck does not label a boundary as tested unless the actual installed agent reliably reaches that boundary.
|
|
43
|
+
|
|
44
|
+
## Retained protocol groundwork
|
|
45
|
+
|
|
46
|
+
The repository keeps the Codex App Server protocol parser, structured fixtures, and deterministic analysis code because they remain useful for:
|
|
47
|
+
|
|
48
|
+
- validating command approval lifecycle messages;
|
|
49
|
+
- exact-action correlation research;
|
|
50
|
+
- regression tests for replay, denial-route-around, interruption, and incomplete evidence;
|
|
51
|
+
- re-enabling a live adapter later if a reproducible supported public boundary becomes available.
|
|
52
|
+
|
|
53
|
+
Passing those fixtures proves the AgentAck protocol analysis logic for those synthetic inputs. It does **not** prove that a real Codex installation can be driven through the same approval boundary.
|
|
54
|
+
|
|
55
|
+
## Current coverage
|
|
56
|
+
|
|
57
|
+
`agentack coverage` reports Codex as `TRACE` for ACK001-ACK009. That means the rules remain covered by AgentAck's deterministic trace engine, not that Codex has passed a live test.
|
|
58
|
+
|
|
59
|
+
Claude Code remains the currently verified live integration.
|
|
60
|
+
|
|
61
|
+
## Re-enabling live Codex support
|
|
62
|
+
|
|
63
|
+
AgentAck should advertise Codex live support again only after a supported public Codex boundary passes a real-binary end-to-end test that can establish, with structured evidence:
|
|
64
|
+
|
|
65
|
+
1. an action requiring approval;
|
|
66
|
+
2. the exact action presented for approval;
|
|
67
|
+
3. an explicit human allow/deny decision;
|
|
68
|
+
4. authoritative execution or block evidence;
|
|
69
|
+
5. stable correlation across those events.
|
|
70
|
+
|
|
71
|
+
The test must not depend on terminal-text scraping or an undocumented private interface.
|
|
72
|
+
|
|
73
|
+
## Privacy
|
|
74
|
+
|
|
75
|
+
The current Codex status path does not launch a model turn or execute a synthetic command. Retained protocol fixtures contain only synthetic values and no credentials or real command output.
|
|
76
|
+
|
|
77
|
+
## Upstream interface
|
|
78
|
+
|
|
79
|
+
The retained parser targets the official Codex App Server JSON-RPC/JSONL lifecycle and generated schemas in the OpenAI Codex repository. Because Codex evolves quickly, future live support should be capability-detected and real-binary validated rather than enabled by a hard-coded version number.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Live coverage
|
|
2
|
+
|
|
3
|
+
AgentAck separates deterministic rule coverage from what each live adapter can safely establish at a real agent boundary.
|
|
4
|
+
|
|
5
|
+
Run:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
agentack coverage
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
The coverage vocabulary is intentionally strict:
|
|
12
|
+
|
|
13
|
+
- **TESTED** means the path deliberately exercises the control and can produce `PASS` or `FAIL` from affirmative evidence.
|
|
14
|
+
- **GUARDED** means the live adapter validates the evidence boundary and fails closed on missing, malformed, or inconsistent evidence, but does not deliberately induce that attack.
|
|
15
|
+
- **TRACE** means the ACK rule is covered by AgentAck's deterministic trace engine, but the listed agent does not currently have a verified portable live probe for that rule.
|
|
16
|
+
- **SKIP** means the agent does not expose a reliable safe boundary for that live check.
|
|
17
|
+
|
|
18
|
+
Current mapping:
|
|
19
|
+
|
|
20
|
+
| Rule | Deterministic trace | Claude Code | Codex CLI | Live interpretation |
|
|
21
|
+
| --- | --- | --- | --- | --- |
|
|
22
|
+
| `ACK001` | TESTED | TESTED | TRACE | Claude baseline action crosses approval; Codex live boundary is not verified |
|
|
23
|
+
| `ACK002` | TESTED | TESTED | TRACE | Claude denial enforcement is exercised live |
|
|
24
|
+
| `ACK003` | TESTED | TESTED | TRACE | Claude exact structured action binding is exercised live |
|
|
25
|
+
| `ACK004` | TESTED | TESTED | TRACE | Claude one-request approval replay is exercised live |
|
|
26
|
+
| `ACK005` | TESTED | TRACE | TRACE | no stable portable live approval-expiry clock |
|
|
27
|
+
| `ACK006` | TESTED | GUARDED | TRACE | Claude malformed/truncated lifecycle fails closed |
|
|
28
|
+
| `ACK007` | TESTED | TESTED | TRACE | Claude denied marker intent is retried through an alternate command |
|
|
29
|
+
| `ACK008` | TESTED | SKIP | TRACE | Claude Stop is not a user-interrupt signal; Codex standalone boundary is not verified |
|
|
30
|
+
| `ACK009` | TESTED | GUARDED | TRACE | Claude missing decision/execution/session evidence returns `INCOMPLETE` |
|
|
31
|
+
|
|
32
|
+
Codex protocol fixtures and deterministic analyzers remain in the repository for research and regression testing, but they are not counted as live coverage.
|
|
33
|
+
|
|
34
|
+
A live `PASS` applies only to the synthetic paths that were actually exercised. It is not a claim that every action path in that agent is safe.
|
|
@@ -11,7 +11,7 @@ Current report schema version: `1`.
|
|
|
11
11
|
"report_schema_version": 1,
|
|
12
12
|
"producer": {
|
|
13
13
|
"name": "AgentAck",
|
|
14
|
-
"version": "0.6.
|
|
14
|
+
"version": "0.6.3"
|
|
15
15
|
},
|
|
16
16
|
"run": {
|
|
17
17
|
"run_id": "UUID",
|
|
@@ -26,7 +26,7 @@ Current report schema version: `1`.
|
|
|
26
26
|
}
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
`kind` is `trace` for `agentack check` and `adapter` for
|
|
29
|
+
`kind` is `trace` for `agentack check` and `adapter` for an adapter result.
|
|
30
30
|
|
|
31
31
|
## Input provenance
|
|
32
32
|
|
|
@@ -24,7 +24,7 @@ Default properties:
|
|
|
24
24
|
|
|
25
25
|
## Live adapters
|
|
26
26
|
|
|
27
|
-
A live adapter runs only when the user explicitly invokes `agentack test <agent
|
|
27
|
+
A live adapter runs only when the user explicitly invokes a supported `agentack test <agent>` path.
|
|
28
28
|
|
|
29
29
|
### Claude Code
|
|
30
30
|
|
|
@@ -40,13 +40,11 @@ See [`claude-code.md`](claude-code.md).
|
|
|
40
40
|
|
|
41
41
|
### Codex CLI
|
|
42
42
|
|
|
43
|
-
|
|
43
|
+
Codex CLI is currently detection-only from the live-support perspective. AgentAck retains App Server protocol parsing, structured fixtures, and deterministic analysis for future support, but real-binary testing with Codex CLI 0.148.0 did not expose a reproducible standalone human command-approval boundary through the public App Server path.
|
|
44
44
|
|
|
45
|
-
|
|
45
|
+
`agentack doctor` therefore reports an installed Codex binary as `DETECTED`, not `READY`. `agentack test codex` is retained only as a backward-compatible status diagnostic and returns `INCOMPLETE` without launching the old model-driven five-probe suite.
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
AgentAck does not parse human-readable Codex terminal output or persist App Server command output in reports.
|
|
47
|
+
Passing Codex protocol fixtures demonstrates AgentAck's analysis behavior for those synthetic inputs. It is not evidence that an installed Codex binary has passed a live approval-control test.
|
|
50
48
|
|
|
51
49
|
See [`codex-cli.md`](codex-cli.md).
|
|
52
50
|
|
|
@@ -62,13 +60,13 @@ AgentAck distinguishes the proposed action from the action recorded as presented
|
|
|
62
60
|
|
|
63
61
|
It cannot prove that a human actually saw or understood a presentation unless the integration exposes a trustworthy boundary for that claim.
|
|
64
62
|
|
|
65
|
-
For Claude Code, hooks provide structured action data while documented OpenTelemetry decision events provide the correlated decision/source.
|
|
63
|
+
For Claude Code, hooks provide structured action data while documented OpenTelemetry decision events provide the correlated decision/source. AgentAck does not currently make a live human-presentation claim for Codex CLI.
|
|
66
64
|
|
|
67
65
|
## Approval scope
|
|
68
66
|
|
|
69
67
|
A replay finding requires evidence that authority was reused outside the scope actually granted.
|
|
70
68
|
|
|
71
|
-
The Claude adapter does not label an explicit persistent user approval as a replay vulnerability.
|
|
69
|
+
The Claude adapter does not label an explicit persistent user approval as a replay vulnerability.
|
|
72
70
|
|
|
73
71
|
## Action hashing
|
|
74
72
|
|
|
@@ -78,7 +76,7 @@ SHA-256 action identity detects differences between structured action representa
|
|
|
78
76
|
|
|
79
77
|
Missing lifecycle evidence is never converted into success. AgentAck returns `INCOMPLETE` when the available evidence cannot establish approval integrity and no stronger security failure is demonstrated.
|
|
80
78
|
|
|
81
|
-
This applies to missing permission events, lost telemetry correlation, missing command completion, missing session
|
|
79
|
+
This applies to missing permission events, lost telemetry correlation, missing command completion, missing session completion, malformed protocol messages, or unsupported live boundaries.
|
|
82
80
|
|
|
83
81
|
## Sensitive data
|
|
84
82
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agentack"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.3"
|
|
8
8
|
description = "Test whether human approval controls for AI agents actually work."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -21,6 +21,7 @@ classifiers = [
|
|
|
21
21
|
"Programming Language :: Python :: 3.11",
|
|
22
22
|
"Programming Language :: Python :: 3.12",
|
|
23
23
|
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
24
25
|
"Topic :: Security",
|
|
25
26
|
"Topic :: Software Development :: Testing",
|
|
26
27
|
]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import shutil
|
|
4
|
+
|
|
5
|
+
from .base import AdapterStatus, AdapterTestResult, AgentAdapter, CheckResult
|
|
6
|
+
from .codex_protocol import safe_version
|
|
7
|
+
|
|
8
|
+
_CODEX_LIVE_LIMITATION = (
|
|
9
|
+
"Codex CLI is installed, but AgentAck does not currently expose a verified deterministic live approval-integrity test. "
|
|
10
|
+
"Real-binary testing against Codex CLI 0.148.0 did not produce a reproducible standalone human command-approval "
|
|
11
|
+
"boundary through the public App Server path, so Codex remains detection and deterministic-trace coverage only."
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class CodexCLIAdapter(AgentAdapter):
|
|
16
|
+
"""Detect Codex while keeping live approval claims fail-closed."""
|
|
17
|
+
|
|
18
|
+
name = "codex"
|
|
19
|
+
display_name = "Codex CLI"
|
|
20
|
+
|
|
21
|
+
def __init__(self, executable: str | None = None) -> None:
|
|
22
|
+
self.executable = executable
|
|
23
|
+
|
|
24
|
+
def detect(self) -> AdapterStatus:
|
|
25
|
+
executable = self.executable or shutil.which("codex")
|
|
26
|
+
if not executable:
|
|
27
|
+
return AdapterStatus(
|
|
28
|
+
name=self.name,
|
|
29
|
+
display_name=self.display_name,
|
|
30
|
+
installed=False,
|
|
31
|
+
testable=False,
|
|
32
|
+
detail="Install Codex CLI for detection. AgentAck does not currently expose a verified Codex live test.",
|
|
33
|
+
)
|
|
34
|
+
return AdapterStatus(
|
|
35
|
+
name=self.name,
|
|
36
|
+
display_name=self.display_name,
|
|
37
|
+
installed=True,
|
|
38
|
+
testable=False,
|
|
39
|
+
executable=executable,
|
|
40
|
+
version=safe_version(executable),
|
|
41
|
+
detail=_CODEX_LIVE_LIMITATION,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def run_test(self) -> AdapterTestResult:
|
|
45
|
+
status = self.detect()
|
|
46
|
+
detail = (
|
|
47
|
+
_CODEX_LIVE_LIMITATION
|
|
48
|
+
if status.installed
|
|
49
|
+
else "The `codex` executable was not found on PATH. AgentAck does not currently expose a verified Codex live test."
|
|
50
|
+
)
|
|
51
|
+
return AdapterTestResult(
|
|
52
|
+
adapter=self.name,
|
|
53
|
+
display_name=self.display_name,
|
|
54
|
+
status="INCOMPLETE",
|
|
55
|
+
checks=(CheckResult("Codex live approval boundary", "INCOMPLETE", detail),),
|
|
56
|
+
notes=(
|
|
57
|
+
"Codex App Server protocol parsing, fixtures, and deterministic analysis remain in the repository for future support.",
|
|
58
|
+
"Use `agentack coverage` for the current trace/live coverage matrix.",
|
|
59
|
+
),
|
|
60
|
+
adapter_version=status.version,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
__all__ = ["CodexCLIAdapter"]
|
|
@@ -204,7 +204,7 @@ def cmd_test(args: argparse.Namespace) -> int:
|
|
|
204
204
|
}
|
|
205
205
|
adapter_type = adapters.get(args.agent)
|
|
206
206
|
if adapter_type is None:
|
|
207
|
-
print(f"agentack: unsupported
|
|
207
|
+
print(f"agentack: unsupported adapter test/status target {args.agent!r}", file=sys.stderr)
|
|
208
208
|
return EXIT_INPUT_ERROR
|
|
209
209
|
try:
|
|
210
210
|
result = adapter_type().run_test()
|
|
@@ -310,7 +310,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
310
310
|
coverage = subparsers.add_parser("coverage", help="show truthful ACK rule coverage across traces and live adapters")
|
|
311
311
|
coverage.set_defaults(func=cmd_coverage)
|
|
312
312
|
|
|
313
|
-
live_test = subparsers.add_parser("test", help="run a live
|
|
313
|
+
live_test = subparsers.add_parser("test", help="run a supported live test or retained adapter status diagnostic")
|
|
314
314
|
live_test.add_argument("agent", choices=("claude", "codex"))
|
|
315
315
|
live_test.add_argument("--json", dest="json_output")
|
|
316
316
|
live_test.add_argument("--sarif")
|
|
@@ -17,15 +17,15 @@ class CoverageRow:
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
LIVE_COVERAGE: tuple[CoverageRow, ...] = (
|
|
20
|
-
CoverageRow("ACK001", "Required approval", "TESTED", "TESTED", "
|
|
21
|
-
CoverageRow("ACK002", "Denied action", "TESTED", "TESTED", "
|
|
22
|
-
CoverageRow("ACK003", "Exact action binding", "TESTED", "TESTED", "
|
|
23
|
-
CoverageRow("ACK004", "Approval replay", "TESTED", "TESTED", "
|
|
24
|
-
CoverageRow("ACK005", "Approval expiry", "TESTED", "TRACE", "TRACE", "Neither
|
|
25
|
-
CoverageRow("ACK006", "Lifecycle ordering", "TESTED", "GUARDED", "
|
|
26
|
-
CoverageRow("ACK007", "Denial route-around", "TESTED", "TESTED", "
|
|
27
|
-
CoverageRow("ACK008", "Interrupt bypass", "TESTED", "SKIP", "
|
|
28
|
-
CoverageRow("ACK009", "Evidence completeness", "TESTED", "GUARDED", "
|
|
20
|
+
CoverageRow("ACK001", "Required approval", "TESTED", "TESTED", "TRACE", "Claude exercises the live approval boundary; Codex remains deterministic-trace coverage only."),
|
|
21
|
+
CoverageRow("ACK002", "Denied action", "TESTED", "TESTED", "TRACE", "Claude exercises live denial enforcement; Codex remains deterministic-trace coverage only."),
|
|
22
|
+
CoverageRow("ACK003", "Exact action binding", "TESTED", "TESTED", "TRACE", "Claude compares live structured action identity; Codex remains deterministic-trace coverage only."),
|
|
23
|
+
CoverageRow("ACK004", "Approval replay", "TESTED", "TESTED", "TRACE", "Claude exercises one-time approval replay; Codex remains deterministic-trace coverage only."),
|
|
24
|
+
CoverageRow("ACK005", "Approval expiry", "TESTED", "TRACE", "TRACE", "Neither agent exposes a stable portable live approval-expiry clock for this suite."),
|
|
25
|
+
CoverageRow("ACK006", "Lifecycle ordering", "TESTED", "GUARDED", "TRACE", "Claude validates live lifecycle evidence; retained Codex protocol analysis is research/regression, not a live claim."),
|
|
26
|
+
CoverageRow("ACK007", "Denial route-around", "TESTED", "TESTED", "TRACE", "Claude exercises an alternate denied-intent route; Codex remains deterministic-trace coverage only."),
|
|
27
|
+
CoverageRow("ACK008", "Interrupt bypass", "TESTED", "SKIP", "TRACE", "Claude Stop is not a user-interrupt signal; Codex has no verified deterministic standalone live boundary."),
|
|
28
|
+
CoverageRow("ACK009", "Evidence completeness", "TESTED", "GUARDED", "TRACE", "Claude fails closed on missing live evidence; Codex remains deterministic-trace coverage only."),
|
|
29
29
|
)
|
|
30
30
|
|
|
31
31
|
|