agent-acceptance 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. agent_acceptance-1.0.0/.gitattributes +5 -0
  2. agent_acceptance-1.0.0/.github/workflows/ci.yml +32 -0
  3. agent_acceptance-1.0.0/.gitignore +11 -0
  4. agent_acceptance-1.0.0/CHANGELOG.md +67 -0
  5. agent_acceptance-1.0.0/LICENSE +201 -0
  6. agent_acceptance-1.0.0/PKG-INFO +164 -0
  7. agent_acceptance-1.0.0/README.md +136 -0
  8. agent_acceptance-1.0.0/action/action.yml +57 -0
  9. agent_acceptance-1.0.0/action/runner/Dockerfile +12 -0
  10. agent_acceptance-1.0.0/action/scripts/post_comment.sh +18 -0
  11. agent_acceptance-1.0.0/action/scripts/run.sh +127 -0
  12. agent_acceptance-1.0.0/conftest.py +7 -0
  13. agent_acceptance-1.0.0/docs/GATE_MODE.md +90 -0
  14. agent_acceptance-1.0.0/docs/THREAT_MODEL.md +57 -0
  15. agent_acceptance-1.0.0/pyproject.toml +54 -0
  16. agent_acceptance-1.0.0/src/agent_acceptance/__init__.py +10 -0
  17. agent_acceptance-1.0.0/src/agent_acceptance/cli.py +180 -0
  18. agent_acceptance-1.0.0/src/agent_acceptance/config.py +91 -0
  19. agent_acceptance-1.0.0/src/agent_acceptance/core/__init__.py +1 -0
  20. agent_acceptance-1.0.0/src/agent_acceptance/core/evaluate.py +204 -0
  21. agent_acceptance-1.0.0/src/agent_acceptance/core/mutation.py +117 -0
  22. agent_acceptance-1.0.0/src/agent_acceptance/core/overlay.py +253 -0
  23. agent_acceptance-1.0.0/src/agent_acceptance/exit_codes.py +17 -0
  24. agent_acceptance-1.0.0/src/agent_acceptance/report/__init__.py +1 -0
  25. agent_acceptance-1.0.0/src/agent_acceptance/report/render.py +68 -0
  26. agent_acceptance-1.0.0/tests/test_evaluate.py +149 -0
  27. agent_acceptance-1.0.0/tests/test_mutation_config.py +231 -0
  28. agent_acceptance-1.0.0/tests/test_report_cli.py +88 -0
  29. agent_acceptance-1.0.0/tests/test_run_backend.py +160 -0
@@ -0,0 +1,5 @@
1
+ # Shell scripts and workflow files must keep LF endings: CRLF breaks bash on Linux.
2
+ *.sh text eol=lf
3
+ *.yml text eol=lf
4
+ Dockerfile text eol=lf
5
+ *.py text eol=lf
@@ -0,0 +1,32 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - uses: actions/setup-python@v5
18
+ with:
19
+ python-version: ${{ matrix.python-version }}
20
+ - name: Install
21
+ run: |
22
+ python -m pip install --upgrade pip
23
+ python -m pip install -e ".[dev]"
24
+ - name: Lint (ruff)
25
+ run: ruff check src tests
26
+ - name: Type-check (mypy)
27
+ run: mypy src/agent_acceptance
28
+ - name: Tests with coverage
29
+ run: pytest --cov=agent_acceptance --cov-report=term-missing --cov-fail-under=80
30
+
31
+ # The composite action is exercised end-to-end by the demo repository
32
+ # (KlldYKG/agent-acceptance-demo, pull_request_target) — see P1a D2.
@@ -0,0 +1,11 @@
1
+ # Build artifacts
2
+ dist/
3
+ build/
4
+ *.egg-info/
5
+ __pycache__/
6
+ .pytest_cache/
7
+ .mypy_cache/
8
+ .ruff_cache/
9
+ .coverage
10
+ htmlcov/
11
+ .venv/
@@ -0,0 +1,67 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+ The format follows [Keep a Changelog](https://keepachangelog.com/); versioning
5
+ follows [SemVer](https://semver.org/). Policy and protocol semantics follow
6
+ `protocol/SPEC.md` v2 (frozen) in the project workspace.
7
+
8
+ ## [1.0.0] — 2026-11-16
9
+
10
+ First stable release. The validated execution path (GitHub Action, Docker
11
+ sandbox) and the local CLI now come from the same source of truth.
12
+
13
+ ### Added
14
+
15
+ - **Bidirectional test-evidence verification.** Added/modified tests from a PR
16
+ are extracted, overlaid onto the pre-PR revision, and executed twice in a
17
+ network-isolated sandbox. `fail-before → pass-after` = **VALID**;
18
+ already-passes-on-base = **PRE-PASS** (the AI-slop signature); fails on both =
19
+ **BROKEN**; collection failure = **ERROR**. Verdicts are deterministic — no
20
+ LLM in the loop.
21
+ - **Sampled mutation testing** (mutmut 2.5.x) over the PR's changed source
22
+ files only, with a wall-clock budget (default 600 s). Budget overrun renders
23
+ as `TIMEOUT`, engine unavailable as `SKIPPED` — the report never fabricates
24
+ numbers.
25
+ - **GitHub Action** (`action/`): composite action with Docker sandbox
26
+ (no network, read-only FS, dropped capabilities, non-root), PR-comment report
27
+ with upsert semantics, and opt-in **gate mode**. Deterministic report
28
+ rendering (`report/render.py`).
29
+ - **CLI** (`agent-acceptance` on PyPI): `run` (git-worktree overlay backend,
30
+ local), `evaluate`, `report` subcommands; documented exit codes
31
+ (0 OK / 10 INVALID_EVIDENCE / 11 BROKEN / 20 NO_EVIDENCE / 2 usage).
32
+ - **Base-branch-only policy** (`.agent-acceptance.yml`): test dirs, mutation
33
+ budget, flaky-n, gate flag — read from the base revision only, so a PR can
34
+ never alter the checks applied to itself.
35
+ - **Gate-mode flaky rule** (SPEC §2.5 extension): each phase runs n=3 times;
36
+ unstable tests are reported as errors and never counted as evidence.
37
+ Report mode always runs each side once.
38
+ - **Security documentation**: `docs/THREAT_MODEL.md` (STRIDE-lite, T1–T8 →
39
+ mitigations, residual risks) and `docs/GATE_MODE.md` (branch-protection
40
+ integration, fail-closed semantics).
41
+ - **CI**: ruff + mypy + pytest with ≥80% coverage gate on Python 3.10–3.13.
42
+ 50 tests, 93.78% coverage at release.
43
+
44
+ ### Changed
45
+
46
+ - Report/eval I/O pins UTF-8 explicitly (reports contain emoji; GBK-default
47
+ Windows hosts crashed before).
48
+ - The demo repository consumes this action by full-commit-SHA reference
49
+ instead of a vendored copy (single source of truth).
50
+
51
+ ### Security
52
+
53
+ - Action code always executes from the pinned ref of this repository, never
54
+ from the PR (safe `pull_request_target` pattern, double checkout).
55
+ - The default is **report mode** (comment only, never blocks). Gate mode is
56
+ opt-in and fails closed.
57
+ - Residual risks (notably: `install-deps` executes packaging code of
58
+ dependencies, networked) are documented in `docs/THREAT_MODEL.md`.
59
+
60
+ ### Known limitations
61
+
62
+ - pytest-ecosystem projects only (JUnit XML interface).
63
+ - `SKIPPED`/`TIMEOUT` mutation results are neutral, not negative.
64
+ - CLI `run` backend executes tests locally without sandboxing — use the
65
+ Action for untrusted code.
66
+
67
+ [1.0.0]: https://github.com/KlldYKG/agent-acceptance/releases/tag/v1.0.0
@@ -0,0 +1,201 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright 2026 agent-acceptance contributors
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
@@ -0,0 +1,164 @@
1
+ Metadata-Version: 2.5
2
+ Name: agent-acceptance
3
+ Version: 1.0.0
4
+ Summary: Deterministic test-evidence checks for AI-contributed PRs: verify added tests fail before and pass after the change. Report mode never blocks.
5
+ Project-URL: Repository, https://github.com/KlldYKG/agent-acceptance
6
+ Author: agent-acceptance contributors
7
+ License-Expression: Apache-2.0
8
+ License-File: LICENSE
9
+ Keywords: ai-generated-code,ci,github-actions,mutation-testing,testing
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Software Development :: Quality Assurance
19
+ Classifier: Topic :: Software Development :: Testing
20
+ Requires-Python: >=3.10
21
+ Provides-Extra: action
22
+ Provides-Extra: dev
23
+ Requires-Dist: mypy>=1.11; extra == 'dev'
24
+ Requires-Dist: pytest-cov>=5; extra == 'dev'
25
+ Requires-Dist: pytest>=8; extra == 'dev'
26
+ Requires-Dist: ruff>=0.6; extra == 'dev'
27
+ Description-Content-Type: text/markdown
28
+
29
+ # agent-acceptance
30
+
31
+ > 76% of open source projects with an AI contribution policy ask contributors
32
+ > to "take responsibility for tests". This Action makes that sentence execute —
33
+ > and checks whether the submitted evidence is real.
34
+
35
+ Deterministic, execution-layer test-evidence checks for AI-contributed (or
36
+ human) pull requests. No LLM anywhere in the loop.
37
+
38
+ ## What it checks
39
+
40
+ **1. Bidirectional test validity.** The PR's added/changed tests are run
41
+ against *both* revisions in a network-isolated sandbox:
42
+
43
+ | on base | on PR | verdict | meaning |
44
+ |---|---|---|---|
45
+ | ❌ fails | ✅ passes | **VALID** | real evidence: the test pins the change |
46
+ | ✅ passes | ✅ passes | **PRE-PASS** | proves nothing about the change — the classic AI-slop signature |
47
+ | ❌ fails | ❌ fails | **BROKEN** | the test itself is broken |
48
+ | — | error | **ERROR** | collection/execution failure |
49
+
50
+ **2. Sampled mutation testing** (mutmut 2.5.x, time-budgeted, scoped to the
51
+ PR's changed source files): quantifies how tightly the new tests pin down the
52
+ changed behavior. **Never fabricates numbers** — an unfinished phase renders
53
+ as `SKIPPED`, not a made-up score.
54
+
55
+ Verified end-to-end in
56
+ [agent-acceptance-demo](https://github.com/KlldYKG/agent-acceptance-demo):
57
+ PR #1 (real fix + real tests) → `VALID`, mutation score 100%; PR #2 (tests
58
+ that pass without any fix, i.e. slop) → `PRE-PASS`, detected.
59
+
60
+ ## Safety model (summary — full version below)
61
+
62
+ - **Untrusted code never gets network access.** Test and mutation phases run
63
+ in a container with `--network none`, read-only repo mount, dropped
64
+ capabilities, non-root UID, CPU/memory/PID limits.
65
+ - **The only networked phase is opt-in dependency installation** (`install-deps`:
66
+ separate container, binary wheels only, hard timeout).
67
+ ⚠️ **Residual risk**: installing a package executes its packaging code
68
+ (`setup.py`/`build` hooks). Only enable `install-deps` when you accept
69
+ running the PR's declared dependencies' build code on the runner.
70
+ - **The Action's own code always executes from your default branch**, never
71
+ from the PR (safe `pull_request_target` double-checkout pattern).
72
+ - **Report mode (default) never blocks merges.** Gate mode is opt-in and must
73
+ be enabled by a maintainer — see [docs/GATE_MODE.md](docs/GATE_MODE.md).
74
+
75
+ Full analysis: [docs/THREAT_MODEL.md](docs/THREAT_MODEL.md).
76
+
77
+ ## Quick start (GitHub Action)
78
+
79
+ In the target repository, create `.github/workflows/agent-acceptance.yml`:
80
+
81
+ ```yaml
82
+ name: agent-acceptance
83
+ on: pull_request_target
84
+ permissions:
85
+ contents: read
86
+ pull-requests: write
87
+ jobs:
88
+ evidence:
89
+ runs-on: ubuntu-latest
90
+ steps:
91
+ # Code under test: the PR merge snapshot (full history for git archive).
92
+ - uses: actions/checkout@v4
93
+ with:
94
+ ref: refs/pull/${{ github.event.pull_request.number }}/merge
95
+ fetch-depth: 0
96
+ # Action code: ALWAYS from the trusted default branch, never from the PR.
97
+ - uses: actions/checkout@v4
98
+ with:
99
+ ref: ${{ github.event.repository.default_branch }}
100
+ path: aa-trusted
101
+ - uses: ./aa-trusted/tool/action # pin a release tag / SHA in production
102
+ with:
103
+ github-token: ${{ github.token }}
104
+ ```
105
+
106
+ The action posts **one** PR comment (upserted in place) with the per-test
107
+ verdict table, mutation results, and a reading guide.
108
+
109
+ ### Inputs
110
+
111
+ | input | default | notes |
112
+ |---|---|---|
113
+ | `github-token` | `${{ github.token }}` | used only to upsert the PR comment |
114
+ | `test-dirs` | `tests` | comma-separated |
115
+ | `mutation-budget-seconds` | `600` | overrun ⇒ `SKIPPED` (no score) |
116
+ | `install-deps` | `false` | ⚠️ see safety model |
117
+ | `gate` | `false` | opt-in blocking — [docs/GATE_MODE.md](docs/GATE_MODE.md) |
118
+
119
+ ## Quick start (CLI)
120
+
121
+ The same checks, runnable locally against any git repository:
122
+
123
+ ```bash
124
+ pip install agent-acceptance
125
+
126
+ # Verify a branch against main (writes out/base.xml, out/pr.xml,
127
+ # out/evidence.json):
128
+ agent-acceptance run --repo . --base main --head my-branch --out out
129
+
130
+ # Or evaluate pre-captured JUnit XML:
131
+ agent-acceptance evaluate out --base-sha <sha> --head-sha <sha>
132
+
133
+ # Render the deterministic PR-comment Markdown:
134
+ agent-acceptance report out
135
+ ```
136
+
137
+ Exit codes: `0` ok · `10` gate: invalid evidence · `11` gate: broken test ·
138
+ `20` harness misconfiguration · `2` usage/infrastructure error.
139
+
140
+ Policy (`.agent-acceptance.yml`) is loaded **only from the base revision** —
141
+ a PR can never alter the checks applied to itself.
142
+
143
+ ## How this differs from review bots and static gates
144
+
145
+ | | LLM review bots | static policy gates (e.g. [Agent Gate](https://github.com/sjh9714/Agent-Gate)) | **agent-acceptance** |
146
+ |---|---|---|---|
147
+ | What it inspects | diff + prose | files/patterns | **executed behavior** |
148
+ | Can be fooled by plausible-sounding tests | yes | n/a | no — tests must actually fail-then-pass |
149
+ | Deterministic / replayable | no | yes | yes |
150
+ | Sees whether tests pin behavior | no | no | yes (mutation sampling) |
151
+
152
+ agent-acceptance is complementary to static gates: it verifies *execution
153
+ evidence*, they verify *policy compliance*.
154
+
155
+ ## Status
156
+
157
+ **v1.0.0** — first stable release. Install from
158
+ [PyPI](https://pypi.org/project/agent-acceptance/) or use the action
159
+ (pinned by commit SHA, see the quick start above). See
160
+ [CHANGELOG.md](CHANGELOG.md) for the release notes.
161
+
162
+ ## Contributing & license
163
+
164
+ Apache-2.0. Issues and PRs welcome — evidence-based tests especially.
@@ -0,0 +1,136 @@
1
+ # agent-acceptance
2
+
3
+ > 76% of open source projects with an AI contribution policy ask contributors
4
+ > to "take responsibility for tests". This Action makes that sentence execute —
5
+ > and checks whether the submitted evidence is real.
6
+
7
+ Deterministic, execution-layer test-evidence checks for AI-contributed (or
8
+ human) pull requests. No LLM anywhere in the loop.
9
+
10
+ ## What it checks
11
+
12
+ **1. Bidirectional test validity.** The PR's added/changed tests are run
13
+ against *both* revisions in a network-isolated sandbox:
14
+
15
+ | on base | on PR | verdict | meaning |
16
+ |---|---|---|---|
17
+ | ❌ fails | ✅ passes | **VALID** | real evidence: the test pins the change |
18
+ | ✅ passes | ✅ passes | **PRE-PASS** | proves nothing about the change — the classic AI-slop signature |
19
+ | ❌ fails | ❌ fails | **BROKEN** | the test itself is broken |
20
+ | — | error | **ERROR** | collection/execution failure |
21
+
22
+ **2. Sampled mutation testing** (mutmut 2.5.x, time-budgeted, scoped to the
23
+ PR's changed source files): quantifies how tightly the new tests pin down the
24
+ changed behavior. **Never fabricates numbers** — an unfinished phase renders
25
+ as `SKIPPED`, not a made-up score.
26
+
27
+ Verified end-to-end in
28
+ [agent-acceptance-demo](https://github.com/KlldYKG/agent-acceptance-demo):
29
+ PR #1 (real fix + real tests) → `VALID`, mutation score 100%; PR #2 (tests
30
+ that pass without any fix, i.e. slop) → `PRE-PASS`, detected.
31
+
32
+ ## Safety model (summary — full version below)
33
+
34
+ - **Untrusted code never gets network access.** Test and mutation phases run
35
+ in a container with `--network none`, read-only repo mount, dropped
36
+ capabilities, non-root UID, CPU/memory/PID limits.
37
+ - **The only networked phase is opt-in dependency installation** (`install-deps`:
38
+ separate container, binary wheels only, hard timeout).
39
+ ⚠️ **Residual risk**: installing a package executes its packaging code
40
+ (`setup.py`/`build` hooks). Only enable `install-deps` when you accept
41
+ running the PR's declared dependencies' build code on the runner.
42
+ - **The Action's own code always executes from your default branch**, never
43
+ from the PR (safe `pull_request_target` double-checkout pattern).
44
+ - **Report mode (default) never blocks merges.** Gate mode is opt-in and must
45
+ be enabled by a maintainer — see [docs/GATE_MODE.md](docs/GATE_MODE.md).
46
+
47
+ Full analysis: [docs/THREAT_MODEL.md](docs/THREAT_MODEL.md).
48
+
49
+ ## Quick start (GitHub Action)
50
+
51
+ In the target repository, create `.github/workflows/agent-acceptance.yml`:
52
+
53
+ ```yaml
54
+ name: agent-acceptance
55
+ on: pull_request_target
56
+ permissions:
57
+ contents: read
58
+ pull-requests: write
59
+ jobs:
60
+ evidence:
61
+ runs-on: ubuntu-latest
62
+ steps:
63
+ # Code under test: the PR merge snapshot (full history for git archive).
64
+ - uses: actions/checkout@v4
65
+ with:
66
+ ref: refs/pull/${{ github.event.pull_request.number }}/merge
67
+ fetch-depth: 0
68
+ # Action code: ALWAYS from the trusted default branch, never from the PR.
69
+ - uses: actions/checkout@v4
70
+ with:
71
+ ref: ${{ github.event.repository.default_branch }}
72
+ path: aa-trusted
73
+ - uses: ./aa-trusted/tool/action # pin a release tag / SHA in production
74
+ with:
75
+ github-token: ${{ github.token }}
76
+ ```
77
+
78
+ The action posts **one** PR comment (upserted in place) with the per-test
79
+ verdict table, mutation results, and a reading guide.
80
+
81
+ ### Inputs
82
+
83
+ | input | default | notes |
84
+ |---|---|---|
85
+ | `github-token` | `${{ github.token }}` | used only to upsert the PR comment |
86
+ | `test-dirs` | `tests` | comma-separated |
87
+ | `mutation-budget-seconds` | `600` | overrun ⇒ `SKIPPED` (no score) |
88
+ | `install-deps` | `false` | ⚠️ see safety model |
89
+ | `gate` | `false` | opt-in blocking — [docs/GATE_MODE.md](docs/GATE_MODE.md) |
90
+
91
+ ## Quick start (CLI)
92
+
93
+ The same checks, runnable locally against any git repository:
94
+
95
+ ```bash
96
+ pip install agent-acceptance
97
+
98
+ # Verify a branch against main (writes out/base.xml, out/pr.xml,
99
+ # out/evidence.json):
100
+ agent-acceptance run --repo . --base main --head my-branch --out out
101
+
102
+ # Or evaluate pre-captured JUnit XML:
103
+ agent-acceptance evaluate out --base-sha <sha> --head-sha <sha>
104
+
105
+ # Render the deterministic PR-comment Markdown:
106
+ agent-acceptance report out
107
+ ```
108
+
109
+ Exit codes: `0` ok · `10` gate: invalid evidence · `11` gate: broken test ·
110
+ `20` harness misconfiguration · `2` usage/infrastructure error.
111
+
112
+ Policy (`.agent-acceptance.yml`) is loaded **only from the base revision** —
113
+ a PR can never alter the checks applied to itself.
114
+
115
+ ## How this differs from review bots and static gates
116
+
117
+ | | LLM review bots | static policy gates (e.g. [Agent Gate](https://github.com/sjh9714/Agent-Gate)) | **agent-acceptance** |
118
+ |---|---|---|---|
119
+ | What it inspects | diff + prose | files/patterns | **executed behavior** |
120
+ | Can be fooled by plausible-sounding tests | yes | n/a | no — tests must actually fail-then-pass |
121
+ | Deterministic / replayable | no | yes | yes |
122
+ | Sees whether tests pin behavior | no | no | yes (mutation sampling) |
123
+
124
+ agent-acceptance is complementary to static gates: it verifies *execution
125
+ evidence*, they verify *policy compliance*.
126
+
127
+ ## Status
128
+
129
+ **v1.0.0** — first stable release. Install from
130
+ [PyPI](https://pypi.org/project/agent-acceptance/) or use the action
131
+ (pinned by commit SHA, see the quick start above). See
132
+ [CHANGELOG.md](CHANGELOG.md) for the release notes.
133
+
134
+ ## Contributing & license
135
+
136
+ Apache-2.0. Issues and PRs welcome — evidence-based tests especially.
@@ -0,0 +1,57 @@
1
+ name: 'agent-acceptance'
2
+ description: >
3
+ Execution-layer test-evidence gate for AI-contributed PRs: verifies that added or
4
+ modified tests fail on the base revision and pass on the PR revision, with sampled
5
+ mutation testing. Report mode (default) never blocks. This action is the validated
6
+ execution path; the sandboxed orchestration lives next to the package source it
7
+ evaluates (single source of truth, SHA-pinned by consumers).
8
+ branding:
9
+ icon: 'check-circle'
10
+ color: 'blue'
11
+
12
+ inputs:
13
+ github-token:
14
+ description: 'Token used to upsert the PR comment.'
15
+ required: false
16
+ default: ${{ github.token }}
17
+ test-dirs:
18
+ description: 'Comma-separated directories that contain test code.'
19
+ required: false
20
+ default: 'tests'
21
+ install-deps:
22
+ description: 'Opt-in: install requirements.txt from PyPI in a separate networked setup container. Residual risk documented in docs/THREAT_MODEL.md.'
23
+ required: false
24
+ default: 'false'
25
+ mutation-budget-seconds:
26
+ description: 'Wall-clock budget for sampled mutation testing.'
27
+ required: false
28
+ default: '600'
29
+ gate:
30
+ description: 'When true, the action fails on INVALID_EVIDENCE / ERROR / BROKEN evidence. Default: report-only.'
31
+ required: false
32
+ default: 'false'
33
+
34
+ runs:
35
+ using: composite
36
+ steps:
37
+ - name: Build sandbox runner image
38
+ shell: bash
39
+ run: docker build -t aa-runner:local "${{ github.action_path }}/runner"
40
+
41
+ - name: Evaluate test evidence
42
+ shell: bash
43
+ env:
44
+ AA_PR_NUMBER: ${{ github.event.pull_request.number }}
45
+ AA_REPO: ${{ github.repository }}
46
+ AA_BASE_SHA: ${{ github.event.pull_request.base.sha }}
47
+ AA_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
48
+ AA_TEST_DIRS: ${{ inputs.test-dirs }}
49
+ AA_INSTALL_DEPS: ${{ inputs.install-deps }}
50
+ AA_MUTATION_BUDGET_S: ${{ inputs.mutation-budget-seconds }}
51
+ AA_GATE: ${{ inputs.gate }}
52
+ AA_TOKEN: ${{ inputs.github-token }}
53
+ GH_TOKEN: ${{ inputs.github-token }}
54
+ # The composite action is checked out from this repository at the pinned
55
+ # ref; the Python package source sits at <repo>/src next to <repo>/action.
56
+ AA_PKG_SRC: ${{ github.action_path }}/../src
57
+ run: bash "${{ github.action_path }}/scripts/run.sh"