aramid 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aramid-0.2.0/LICENSE +21 -0
- aramid-0.2.0/PKG-INFO +426 -0
- aramid-0.2.0/README.md +390 -0
- aramid-0.2.0/pyproject.toml +133 -0
- aramid-0.2.0/setup.cfg +4 -0
- aramid-0.2.0/src/aramid/__init__.py +1 -0
- aramid-0.2.0/src/aramid/__main__.py +3 -0
- aramid-0.2.0/src/aramid/autolearn.py +313 -0
- aramid-0.2.0/src/aramid/cli.py +264 -0
- aramid-0.2.0/src/aramid/commands/__init__.py +0 -0
- aramid-0.2.0/src/aramid/commands/arm.py +222 -0
- aramid-0.2.0/src/aramid/commands/autolearn_cmd.py +76 -0
- aramid-0.2.0/src/aramid/commands/check.py +175 -0
- aramid-0.2.0/src/aramid/commands/doctor.py +723 -0
- aramid-0.2.0/src/aramid/commands/drain.py +270 -0
- aramid-0.2.0/src/aramid/commands/hooks_template.py +134 -0
- aramid-0.2.0/src/aramid/commands/init.py +472 -0
- aramid-0.2.0/src/aramid/commands/ledger_cmd.py +245 -0
- aramid-0.2.0/src/aramid/commands/mutation_score.py +65 -0
- aramid-0.2.0/src/aramid/commands/override.py +131 -0
- aramid-0.2.0/src/aramid/commands/pack_cmd.py +76 -0
- aramid-0.2.0/src/aramid/commands/rebaseline.py +46 -0
- aramid-0.2.0/src/aramid/commands/schedule.py +262 -0
- aramid-0.2.0/src/aramid/commands/status.py +342 -0
- aramid-0.2.0/src/aramid/commands/triage_cmd.py +91 -0
- aramid-0.2.0/src/aramid/commands/uninstall.py +51 -0
- aramid-0.2.0/src/aramid/commands/update_rules.py +52 -0
- aramid-0.2.0/src/aramid/config.py +271 -0
- aramid-0.2.0/src/aramid/consumers/__init__.py +0 -0
- aramid-0.2.0/src/aramid/consumers/base.py +54 -0
- aramid-0.2.0/src/aramid/consumers/dast.py +83 -0
- aramid-0.2.0/src/aramid/consumers/fuzz.py +221 -0
- aramid-0.2.0/src/aramid/consumers/js_mutation.py +236 -0
- aramid-0.2.0/src/aramid/consumers/llm_review.py +415 -0
- aramid-0.2.0/src/aramid/consumers/mutation.py +285 -0
- aramid-0.2.0/src/aramid/consumers/regression_pack.py +51 -0
- aramid-0.2.0/src/aramid/dast_probe.py +265 -0
- aramid-0.2.0/src/aramid/data/ARAMID.md.tmpl +192 -0
- aramid-0.2.0/src/aramid/data/block_rules.toml +30 -0
- aramid-0.2.0/src/aramid/data/defaults.toml +219 -0
- aramid-0.2.0/src/aramid/detectors.py +250 -0
- aramid-0.2.0/src/aramid/diagnostics.py +49 -0
- aramid-0.2.0/src/aramid/fingerprint.py +14 -0
- aramid-0.2.0/src/aramid/fuzzdriver.py +121 -0
- aramid-0.2.0/src/aramid/fuzzgen.py +117 -0
- aramid-0.2.0/src/aramid/gitutil.py +153 -0
- aramid-0.2.0/src/aramid/hooks.py +522 -0
- aramid-0.2.0/src/aramid/jsmutate.py +371 -0
- aramid-0.2.0/src/aramid/ledger.py +263 -0
- aramid-0.2.0/src/aramid/models.py +84 -0
- aramid-0.2.0/src/aramid/mutation.py +109 -0
- aramid-0.2.0/src/aramid/mutation_gate.py +103 -0
- aramid-0.2.0/src/aramid/mutation_score.py +114 -0
- aramid-0.2.0/src/aramid/mutation_score_gate.py +103 -0
- aramid-0.2.0/src/aramid/normalizer.py +91 -0
- aramid-0.2.0/src/aramid/pack.py +117 -0
- aramid-0.2.0/src/aramid/pipeline.py +921 -0
- aramid-0.2.0/src/aramid/policy.py +263 -0
- aramid-0.2.0/src/aramid/providers/__init__.py +0 -0
- aramid-0.2.0/src/aramid/providers/base.py +91 -0
- aramid-0.2.0/src/aramid/providers/claude_cli.py +70 -0
- aramid-0.2.0/src/aramid/providers/codex_cli.py +99 -0
- aramid-0.2.0/src/aramid/providers/ollama_cloud.py +91 -0
- aramid-0.2.0/src/aramid/providers/openrouter.py +119 -0
- aramid-0.2.0/src/aramid/providers/spend.py +51 -0
- aramid-0.2.0/src/aramid/queue.py +137 -0
- aramid-0.2.0/src/aramid/red_proof.py +310 -0
- aramid-0.2.0/src/aramid/redact.py +26 -0
- aramid-0.2.0/src/aramid/registry.py +48 -0
- aramid-0.2.0/src/aramid/reporter.py +106 -0
- aramid-0.2.0/src/aramid/review.py +519 -0
- aramid-0.2.0/src/aramid/rules/owasp.yml +362 -0
- aramid-0.2.0/src/aramid/runners/__init__.py +0 -0
- aramid-0.2.0/src/aramid/runners/_util.py +96 -0
- aramid-0.2.0/src/aramid/runners/base.py +217 -0
- aramid-0.2.0/src/aramid/runners/clippy.py +326 -0
- aramid-0.2.0/src/aramid/runners/deps.py +656 -0
- aramid-0.2.0/src/aramid/runners/eslint.py +136 -0
- aramid-0.2.0/src/aramid/runners/gitleaks.py +105 -0
- aramid-0.2.0/src/aramid/runners/ruff.py +93 -0
- aramid-0.2.0/src/aramid/runners/semgrep.py +189 -0
- aramid-0.2.0/src/aramid/runners/tests.py +471 -0
- aramid-0.2.0/src/aramid/runners/typecheck.py +185 -0
- aramid-0.2.0/src/aramid/tdd.py +115 -0
- aramid-0.2.0/src/aramid/tests_gate.py +74 -0
- aramid-0.2.0/src/aramid/toolpath.py +92 -0
- aramid-0.2.0/src/aramid/toolset.py +159 -0
- aramid-0.2.0/src/aramid/triage.py +185 -0
- aramid-0.2.0/src/aramid.egg-info/PKG-INFO +426 -0
- aramid-0.2.0/src/aramid.egg-info/SOURCES.txt +92 -0
- aramid-0.2.0/src/aramid.egg-info/dependency_links.txt +1 -0
- aramid-0.2.0/src/aramid.egg-info/entry_points.txt +2 -0
- aramid-0.2.0/src/aramid.egg-info/requires.txt +9 -0
- aramid-0.2.0/src/aramid.egg-info/top_level.txt +1 -0
aramid-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 FBMac
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
aramid-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aramid
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A deterministic, offline security and quality gate for git hooks -- gitleaks, semgrep, ruff/eslint, pip-audit and your own suite, at pre-commit and pre-push.
|
|
5
|
+
Author: FBMac
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/jared0565/aramid
|
|
8
|
+
Project-URL: Repository, https://github.com/jared0565/aramid
|
|
9
|
+
Project-URL: Changelog, https://github.com/jared0565/aramid/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Issues, https://github.com/jared0565/aramid/issues
|
|
11
|
+
Keywords: security,sast,secrets,git-hooks,pre-commit,static-analysis,linting,devsecops
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Security
|
|
22
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
23
|
+
Classifier: Topic :: Software Development :: Testing
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: tomli-w<2,>=1.0
|
|
28
|
+
Requires-Dist: pip-audit<3,>=2.7
|
|
29
|
+
Requires-Dist: ruff<0.17,>=0.6
|
|
30
|
+
Requires-Dist: semgrep<2,>=1.100
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
33
|
+
Requires-Dist: pyyaml>=6; extra == "dev"
|
|
34
|
+
Requires-Dist: packaging>=23; extra == "dev"
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# aramid
|
|
38
|
+
|
|
39
|
+
A red/blue-team security & quality oversight engine for application development. The
|
|
40
|
+
foundation is a **deterministic gate**: git-hook enforcement that runs industry-standard
|
|
41
|
+
tools — gitleaks (secrets), semgrep (SAST), ruff/eslint (lint), pip-audit (dependency
|
|
42
|
+
CVEs), and the project's own test suite — at `pre-commit` and `pre-push`. Findings are
|
|
43
|
+
severity-tiered: **security blocks, quality warns.** The gate itself makes **zero LLM
|
|
44
|
+
calls and burns zero tokens** — fully offline-capable. Riding on top of it is a
|
|
45
|
+
token-economical **red team**: a scheduled, budgeted drain that spends LLM quota only on
|
|
46
|
+
the small, novel, high-risk slice of commits, never on every push (see the roadmap below).
|
|
47
|
+
|
|
48
|
+
## Install
|
|
49
|
+
|
|
50
|
+
aramid is not on PyPI, so `pip install aramid` does not work. Install the wheel
|
|
51
|
+
attached to a [GitHub Release](https://github.com/jared0565/aramid/releases):
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install https://github.com/jared0565/aramid/releases/download/v0.1.0/aramid-0.1.0-py3-none-any.whl
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Or straight from git, if you would rather pin a ref than a file:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install "git+https://github.com/jared0565/aramid@v0.1.0"
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
To work on aramid itself, install it editable from a checkout — this is a
|
|
64
|
+
development install, not the way to deploy it:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install -e ".[dev]"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Any of these pulls in `ruff`, `semgrep`, and `pip-audit` as aramid's own dependencies. Secret
|
|
71
|
+
scanning additionally requires a `gitleaks` binary on `PATH` (see `aramid doctor`).
|
|
72
|
+
The vendored OWASP semgrep ruleset ships inside the wheel; `aramid update-rules` reports
|
|
73
|
+
its pinned source and install path (refreshing it is a re-vendor + rebuild, offline by
|
|
74
|
+
design — not a runtime fetch).
|
|
75
|
+
|
|
76
|
+
## Quickstart
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
aramid init <repo> # onboard a repo: writes aramid.toml, installs git hooks, baselines
|
|
80
|
+
aramid doctor # probe the toolchain (gitleaks/semgrep/ruff/eslint/pip-audit) and offer repair
|
|
81
|
+
aramid check --all # run the full gate on demand (also: --staged, --range, --gate pre-push)
|
|
82
|
+
aramid status # report ledger and config state
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Once installed, `git commit` and `git push` trigger the gate automatically via the
|
|
86
|
+
installed hooks. Local hooks are convenience, not enforcement — `--no-verify` exists.
|
|
87
|
+
The authoritative backstop is re-running `aramid check --all --strict --json` in CI.
|
|
88
|
+
|
|
89
|
+
## Documentation
|
|
90
|
+
|
|
91
|
+
- **[User Guide](https://github.com/jared0565/aramid/blob/main/docs/user-guide.md)** — task-oriented walkthrough: install, onboarding, the gate, running checks, the red-team drain, and each consumer.
|
|
92
|
+
- **[Knowledge Base](https://github.com/jared0565/aramid/blob/main/docs/knowledge-base.md)** — reference: concepts glossary, full configuration reference, consumer reference, CLI commands, and exit codes.
|
|
93
|
+
- **Design specs & implementation plans** — `docs/superpowers/specs/` and `docs/superpowers/plans/`.
|
|
94
|
+
|
|
95
|
+
## Exit-code contract
|
|
96
|
+
|
|
97
|
+
| Code | Meaning |
|
|
98
|
+
|---|---|
|
|
99
|
+
| 0 | pass |
|
|
100
|
+
| 1 | blocking verdict — real findings, or (pre-push only) degraded BLOCK-tier tooling |
|
|
101
|
+
| 2 | pass-but-degraded — a WARN-tier tool was skipped or timed out |
|
|
102
|
+
| 3 | engine or config error |
|
|
103
|
+
|
|
104
|
+
`--strict` (CI mode) remaps 2 and 3 onto 1, so a run that "couldn't tell" fails the
|
|
105
|
+
build the same as a run that found something. The engine never exits 0 silently on
|
|
106
|
+
its own failure.
|
|
107
|
+
|
|
108
|
+
## Scope & roadmap
|
|
109
|
+
|
|
110
|
+
The **deterministic gate** covers the mechanical slice of OWASP: secrets, SAST,
|
|
111
|
+
dependency CVEs, and lint. It deliberately does **not** try to reason about access
|
|
112
|
+
control, security misconfiguration, or authentication logic in a regex — that
|
|
113
|
+
adversarial, judgment-based slice is the red team's job (Phase 2b), run at drain time
|
|
114
|
+
under a budget rather than on every commit. Four phases:
|
|
115
|
+
|
|
116
|
+
1. **Phase 1 — done:** deterministic blue-team gate engine.
|
|
117
|
+
2. **Phase 2 — red team**, staged into three:
|
|
118
|
+
- **2a — done:** zero-token watcher chassis — commit triage → risk-scored review
|
|
119
|
+
queue → budgeted scheduled drain → pluggable consumers, plus the regression attack pack.
|
|
120
|
+
- **2b — done:** the LLM reviewer — evidence-bound adversarial review over a provider
|
|
121
|
+
chain, cross-provider refute (self-refute fallback on single-provider installs), bake-then-arm blocking (detailed below).
|
|
122
|
+
- **2c — in progress:** the heavy adversarial tier, each a new drain consumer —
|
|
123
|
+
mutation (2c-1), JS/TS mutation (2c-1b), fuzz/property harness (2c-2), and
|
|
124
|
+
DAST passive web-hygiene probing (2c-3) are all shipped. Remaining within 2c:
|
|
125
|
+
an explicit-config app auto-start runtime, nuclei enrichment, and armed-BLOCK
|
|
126
|
+
wiring for DAST.
|
|
127
|
+
3. **Phase 3:** harness advisory layer — non-blocking, mid-development early warning.
|
|
128
|
+
4. **Phase 4:** metering & governance — token budgets, ledger-derived regression tests.
|
|
129
|
+
|
|
130
|
+
Full design specs and implementation plans: `docs/superpowers/specs/` and
|
|
131
|
+
`docs/superpowers/plans/`.
|
|
132
|
+
|
|
133
|
+
## Upgrading / re-baselining
|
|
134
|
+
|
|
135
|
+
A finding's identity is `sha256(tool + rule + normalized-path + sha256(normalized-line) + occurrence-index)`. Rule-id and path normalization feed that hash, so an aramid upgrade that changes them re-fingerprints already-accepted findings — the ratchet then sees them as new and can escalate them to BLOCK. After such an upgrade, run:
|
|
136
|
+
|
|
137
|
+
aramid rebaseline --yes
|
|
138
|
+
|
|
139
|
+
to re-snapshot the current findings as the accepted baseline. This discards prior ratchet grandfathering (that is the point), so review the gate output first. Without `--yes` the command only reports what it would discard and exits non-zero.
|
|
140
|
+
|
|
141
|
+
## Phase 2a: watcher chassis
|
|
142
|
+
|
|
143
|
+
Phase 2 starts with a zero-token chassis — the code has landed, and this repo
|
|
144
|
+
carries its config (`aramid.toml`). The triage hook and scheduled drain are a
|
|
145
|
+
per-clone local step (`.git/hooks` is not version-controlled): run `aramid init .`
|
|
146
|
+
to install the post-commit triage shim and `aramid schedule install` to register
|
|
147
|
+
the drain job.
|
|
148
|
+
Once installed, every commit is scored at zero cost by a post-commit hook
|
|
149
|
+
(security-surface paths, risky content, novelty, graphite blast radius). Commits
|
|
150
|
+
scoring >= 40 join a review queue drained on a schedule (`aramid drain`, Task
|
|
151
|
+
Scheduler task `aramid-drain`).
|
|
152
|
+
The post-commit hook self-kills after 15s (`--budget`), so a wedged triage can
|
|
153
|
+
never hang `git commit`; shims installed before this feature pick it up on the
|
|
154
|
+
next `aramid init` (idempotent shim regeneration).
|
|
155
|
+
The regression attack pack (`.aramid-rules/regression.yml`) replays rules
|
|
156
|
+
compiled from resolved findings — `aramid pack compile` writes it, and an
|
|
157
|
+
adopting repo commits it (this repo has none yet: no findings resolved). It
|
|
158
|
+
reintroduces a rotated secret or banned dependency as a pre-push block.
|
|
159
|
+
`aramid status` shows queue depth and drain history; `aramid pack list|add|compile`
|
|
160
|
+
manages rules.
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
aramid triage HEAD # score a commit (or range) and enqueue if risky
|
|
164
|
+
aramid drain --repo . --dry-run # preview what a drain would consume
|
|
165
|
+
aramid schedule install # register the Task Scheduler drain job (Windows)
|
|
166
|
+
aramid pack list # show compiled regression rules
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Still deterministic, still zero LLM calls — 2a is the chassis (triage → queue →
|
|
170
|
+
drain) that Phase 2b (LLM adversarial review, shipped) and Phase 2c ride as
|
|
171
|
+
drain-time consumers. 2c-1 (shipped) adds the mutation consumer: diff-touched
|
|
172
|
+
functions are mutated in a throwaway worktree and mutants the full test suite
|
|
173
|
+
cannot kill are recorded as WARN-tier test-gap findings (`[mutation]` config:
|
|
174
|
+
budgets, two-stage targeted/confirm execution; Python repos with pytest).
|
|
175
|
+
2c-1b (shipped) extends mutation to JavaScript/TypeScript: an owned token-level
|
|
176
|
+
mutator (no AST) mutates the diff-touched lines inside a throwaway worktree with
|
|
177
|
+
the repo's own `node_modules` junctioned in, running the project's `<pm> test`
|
|
178
|
+
once per mutant; survivors the suite cannot kill are WARN-tier test-gap findings
|
|
179
|
+
(`[js_mutation]` config: budgets; JS/TS repos with an npm/pnpm/yarn test script).
|
|
180
|
+
2c-2 (shipped) adds the fuzz consumer: diff-touched type-hinted functions are
|
|
181
|
+
called with deterministic seeded inputs in a throwaway worktree, and deep-crash
|
|
182
|
+
exceptions (IndexError, KeyError, …) are recorded as WARN-tier findings — the
|
|
183
|
+
seed is the repro (`[fuzz]` config: budgets, a scary-name skip-list; Python
|
|
184
|
+
repos with type hints, no test suite required). Repro caveat: the seed
|
|
185
|
+
reproduces a crash only for targets that are deterministic in their arguments —
|
|
186
|
+
functions depending on external state (files, network, globals, time) may not
|
|
187
|
+
replay from the recorded seed.
|
|
188
|
+
2c-3 (shipped) adds the DAST consumer: an owned stdlib passive web-hygiene prober
|
|
189
|
+
scans a user-declared `base_url` (never auto-started) with bounded one-shot HTTP
|
|
190
|
+
requests, reporting missing security headers, insecure cookie flags, plaintext
|
|
191
|
+
transport, exposed sensitive paths (`.git/config`, `.env`, …), and server version
|
|
192
|
+
banners as WARN-tier findings. Evidence is metadata only — never response bodies
|
|
193
|
+
or secret values. It OK-skips when no target is configured (a non-web repo never
|
|
194
|
+
pins the queue) and gives up after repeated unreachable/erroring drains
|
|
195
|
+
(`[dast]` config: `base_url`, `paths`, `timeout_s`; off by default until a target
|
|
196
|
+
is set).
|
|
197
|
+
|
|
198
|
+
### `aramid mutation-score`: advisory drift report
|
|
199
|
+
|
|
200
|
+
`aramid mutation-score` (add `--json` for machine-readable output) is a
|
|
201
|
+
**read-only, advisory** report over the mutation consumer's ledger history —
|
|
202
|
+
it surfaces per-function mutation-score drift and flags regressions, but it
|
|
203
|
+
is not a gate: it never blocks, never arms, and never writes to the ledger
|
|
204
|
+
(exit 0 on a readable ledger, 3 on engine error). Two signals, both computed
|
|
205
|
+
from the mutation consumer's existing per-run taxonomy: a per-mutant
|
|
206
|
+
**transition** (a mutant killed in the most-recent-prior fully-mutated run
|
|
207
|
+
now confirmed-surviving on a line whose content hasn't changed — precise,
|
|
208
|
+
truncation-proof) and a per-function **rate-delta** (stage-1 kill-rate
|
|
209
|
+
dropped against that same baseline — richer but noisier, compared only
|
|
210
|
+
between `fully_mutated` runs).
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
aramid mutation-score # human-readable per-function scores + regressions
|
|
214
|
+
aramid mutation-score --json # machine-readable
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
Four documented limitations (it measures drift, it doesn't enforce anything
|
|
218
|
+
— read the numbers, don't trust the silence):
|
|
219
|
+
1. **Code-change-triggered:** only re-mutated (diff-touched) functions are
|
|
220
|
+
measured, so test-weakening against unchanged code is invisible to this
|
|
221
|
+
metric.
|
|
222
|
+
2. **Rate-delta is a narrow-oracle self-delta:** it is silent on any
|
|
223
|
+
function whose mutants were budget-dropped, timed out, or errored
|
|
224
|
+
(`fully_mutated == False`) — such a function never gets a fresh rate to
|
|
225
|
+
compare against its baseline.
|
|
226
|
+
3. **Function-key baseline is lost on rename:** the baseline key is
|
|
227
|
+
`"<rel>::<func>"`; renaming a function or its file drops the prior
|
|
228
|
+
baseline, missing one signal at the rename boundary (normal again on the
|
|
229
|
+
next drain).
|
|
230
|
+
4. **Transition recall is bounded by `confirm_cap`:** an unconfirmed
|
|
231
|
+
(cap-truncated) stage-1 survivor isn't counted yet, so a regression it
|
|
232
|
+
represents fires on a later drain once it's confirmed — not the first.
|
|
233
|
+
|
|
234
|
+
#### 2b: regression teeth at pre-push
|
|
235
|
+
|
|
236
|
+
Every `pre-push` gate recomputes the regressions above straight from drain
|
|
237
|
+
history — nothing is stored, so no stale record can be wrongly resolved and
|
|
238
|
+
only a re-drain that re-measures the function truly clears a regression.
|
|
239
|
+
|
|
240
|
+
- **Transition regressions** (a previously-killed mutant now survives) are
|
|
241
|
+
findings under tool `mutation-score`, rule `transition`, severity high.
|
|
242
|
+
They WARN during the bake and BLOCK once the repo opts in with
|
|
243
|
+
`aramid arm --mutation-score` (sets `[mutation].score_block_armed = true`).
|
|
244
|
+
- **Rate regressions** (stage-1 kill-rate dropped between fully-measured
|
|
245
|
+
runs) are permanent WARN, rule `rate`, severity low. They never block;
|
|
246
|
+
arming rate needs real-drain evidence: today's trigger is a bare
|
|
247
|
+
`current.rate < baseline.rate` with no minimum sample size or delta
|
|
248
|
+
threshold, over mutant batches regenerated from the function's current
|
|
249
|
+
source each drain — not a fixed population between the runs being
|
|
250
|
+
compared. Arming on that alone risks blocking on sampling noise instead
|
|
251
|
+
of proven test-weakening; a calibrated threshold is what real-drain
|
|
252
|
+
history would provide.
|
|
253
|
+
- **The only escape valve is ephemeral, and it's transition-only:** rate
|
|
254
|
+
regressions are permanent WARN and never block (above), so there is
|
|
255
|
+
nothing for them to escape. A push whose range adds or modifies the
|
|
256
|
+
module-mapped test (`test_<module>.py` / `<module>_test.py`) suppresses
|
|
257
|
+
the transition for that gate run only. Touching the source file does
|
|
258
|
+
not suppress — that is exactly the optimistic-resolution hole the
|
|
259
|
+
surviving-mutant gate has and this gate closes.
|
|
260
|
+
|
|
261
|
+
Additional limitations beyond the advisory ones above:
|
|
262
|
+
|
|
263
|
+
1. Two same-operator mutants on one identical line share a fingerprint, so
|
|
264
|
+
an armed transition may conflate them (the killing test for one kills
|
|
265
|
+
the class).
|
|
266
|
+
2. Regression findings are derived per-gate and never persisted: they do
|
|
267
|
+
not appear in `aramid status` and cannot be overridden via
|
|
268
|
+
`aramid override` — the escape hatches are the mapped test or disarming.
|
|
269
|
+
3. A function rewritten without its mapped test keeps blocking on the old
|
|
270
|
+
measurement until a re-drain re-measures it. Because a disabled engine
|
|
271
|
+
could then never clear it, `[mutation].enabled = false` disables this
|
|
272
|
+
gate entirely; use `score_block_armed = false` to drop only the teeth.
|
|
273
|
+
4. Detection reads only the stage-1 killed/survived counts, the
|
|
274
|
+
fully-mutated flag, and mutant fingerprints — never the under-counted
|
|
275
|
+
errors/timeouts buckets, so noisy timeout/error runs cannot fake a
|
|
276
|
+
regression.
|
|
277
|
+
|
|
278
|
+
### Red-first proof (TDD gate, sub-project 3)
|
|
279
|
+
|
|
280
|
+
At every `pre-push`, changed test files are examined — but only if at least
|
|
281
|
+
one of their changed lines is itself a test **definition** line (`def`/`async
|
|
282
|
+
def` whose name starts with `test`, found by walking the real `ast`, never by
|
|
283
|
+
matching text against the diff — limitation 8). A qualifying file's head
|
|
284
|
+
version is then run — against a throwaway worktree at the range's *base*. A
|
|
285
|
+
file whose tests all pass on the pre-change tree was never red, so it proves
|
|
286
|
+
nothing about the change: one finding per such file (tool `red-proof`, rule
|
|
287
|
+
`test-not-red`, severity medium). Collection errors count as red — a test
|
|
288
|
+
importing a brand-new module *is* red on the base tree.
|
|
289
|
+
|
|
290
|
+
Findings WARN during the bake and BLOCK once the repo opts in with
|
|
291
|
+
`aramid arm --red-proof` (sets `[red_proof].red_proof_block_armed = true`).
|
|
292
|
+
Disarmed WARNs never auto-escalate, `aramid override` works as the standard
|
|
293
|
+
escape hatch, and only files changed in the push are ever examined, so
|
|
294
|
+
arming can never wall-block pre-existing repo state. `[red_proof]` also
|
|
295
|
+
carries `wall_budget_s` / `test_timeout_s` caps for the per-file test runs
|
|
296
|
+
(the one-time worktree setup and git reads sit outside the budget, like
|
|
297
|
+
every other git call in the gate); when the budget runs out, remaining
|
|
298
|
+
files are skipped silently.
|
|
299
|
+
|
|
300
|
+
Limitations:
|
|
301
|
+
|
|
302
|
+
1. The verdict is per test *file*: an old test in a changed file failing on
|
|
303
|
+
base masks a never-red new test (a missed signal). Before the content
|
|
304
|
+
gate (limitation 8) existed, this whole-file design also produced
|
|
305
|
+
genuine false alarms: any changed line in a test file triggered a full
|
|
306
|
+
base rerun regardless of what changed, so a fixture repair, a comment,
|
|
307
|
+
or any other non-test-adding edit to an already-green file could be
|
|
308
|
+
flagged as never-red — this is why the gate exists. It closes that
|
|
309
|
+
specific class, but one masking residue survives it: once a file passes
|
|
310
|
+
the gate and a finding fires, the whole-file verdict still can't say
|
|
311
|
+
*which* test definition in the file was the one that never went red, if
|
|
312
|
+
the file holds more than one.
|
|
313
|
+
2. Any import failure on base counts as red, including files trivially
|
|
314
|
+
broken on base for unrelated reasons.
|
|
315
|
+
3. Only the changed test files themselves are materialized at head — a new
|
|
316
|
+
test depending on head changes to non-test files it imports (a root
|
|
317
|
+
`conftest.py`, a new fixture module) usually collection-errors, which
|
|
318
|
+
counts as red.
|
|
319
|
+
4. Range mode only: first pushes and `--all`/`--staged` runs skip silently.
|
|
320
|
+
5. Tests run once, no flake retries — bake before arming.
|
|
321
|
+
6. The base run inherits the repo's own pytest config: an `addopts` gate
|
|
322
|
+
(coverage threshold, warnings-as-errors) can force any single-file base
|
|
323
|
+
run non-zero — read as red. As a detector it still never raises a false
|
|
324
|
+
alarm, but a *persistent* gate costs recall the way limitation 1 costs
|
|
325
|
+
it for one file: every base run reads red, so no genuine never-red
|
|
326
|
+
violation is ever flagged. And as a resolver it is not harmless: it
|
|
327
|
+
durably resolves any existing open red-proof finding on the file and
|
|
328
|
+
cannot self-correct,
|
|
329
|
+
since a gated base run can never come back green to re-open it. This
|
|
330
|
+
already happens during the bake, not only once armed.
|
|
331
|
+
7. The base run's import path is forced to the base worktree
|
|
332
|
+
(`<wt>/src`, then `<wt>`, then the inherited `PYTHONPATH`). Without
|
|
333
|
+
this the base run imports whatever is *installed*, which under a pip
|
|
334
|
+
editable install is the live source the push is changing — so a
|
|
335
|
+
src-layout package resolved to head code, every genuinely red-first
|
|
336
|
+
test passed on "base", and the producer raised a false alarm for every
|
|
337
|
+
changed test file. That inverted the guarantee in limitations 1 and 2,
|
|
338
|
+
and it is fixed. Two residues remain: a PEP 660 **strict** editable
|
|
339
|
+
install hooks a `MetaPathFinder` rather than adding a `sys.path` entry,
|
|
340
|
+
and nothing on `PYTHONPATH` outranks that; and a package installed
|
|
341
|
+
non-editably still shadows the worktree unless its layout puts the
|
|
342
|
+
source under `<wt>/src` or `<wt>`.
|
|
343
|
+
8. A subject is only examined if at least one of its changed lines is itself
|
|
344
|
+
a test **definition** line — `def`/`async def` whose name starts with
|
|
345
|
+
`test`, found by walking the real `ast`, never by matching text against
|
|
346
|
+
the diff (a string literal or docstring that merely *contains*
|
|
347
|
+
`def test_x():`-shaped text does not count, nor does a line added inside
|
|
348
|
+
an existing test's body). "Changed" includes a pure modification of an
|
|
349
|
+
already-existing def line (a reformat, a rename), not only a freshly
|
|
350
|
+
added one — that is not a distinct false-positive class, it is limitation
|
|
351
|
+
1's whole-file behavior under the same name, since the base run still
|
|
352
|
+
proves nothing more than "the whole file passed". This closes the
|
|
353
|
+
false-alarm class in limitation 1 for edits that touch no test
|
|
354
|
+
definition at all, at a deliberate recall cost: a new
|
|
355
|
+
`@pytest.mark.parametrize` case added to an existing test function, or a
|
|
356
|
+
strengthened assertion in an existing test's body, is not scanned at all.
|
|
357
|
+
That is not an oversight — this producer's contract is recall loss only,
|
|
358
|
+
never a false positive, and this trades one false-positive class (any
|
|
359
|
+
edit to an already-green test file) for a symmetric false-negative class
|
|
360
|
+
(an edit that only touches an existing test's body) one layer earlier,
|
|
361
|
+
before a subprocess is even spent on it. One resolution-side consequence
|
|
362
|
+
follows: such an edit can no longer prove a file's open red-proof finding
|
|
363
|
+
red either, so it can no longer auto-resolve that finding — only a push
|
|
364
|
+
that changes a test-definition line can. A BOM-prefixed file's otherwise-
|
|
365
|
+
qualifying change is invisible to this gate too: the BOM makes `ast.parse`
|
|
366
|
+
raise a `SyntaxError`, so the file is silently never scanned — correct
|
|
367
|
+
fail-open behavior, but a real recall cost. The name check is hard-coded
|
|
368
|
+
to `test`; a repo that configures pytest's `python_functions` to
|
|
369
|
+
something else has its differently-named tests invisible to this gate
|
|
370
|
+
regardless of that setting.
|
|
371
|
+
|
|
372
|
+
### Phase 2b: the LLM reviewer
|
|
373
|
+
|
|
374
|
+
The `llm-review` drain-time consumer covers exactly the OWASP slice 2a's
|
|
375
|
+
deterministic tools can't: broken access control (A01), security
|
|
376
|
+
misconfiguration (A05), authentication failures (A07), and business-logic
|
|
377
|
+
flaws — adversarial, judgment-based review that a regex or an AST rule
|
|
378
|
+
cannot do. Every queued item's diff and touched files are assembled into a
|
|
379
|
+
redacted, byte-capped packet and sent down a provider chain
|
|
380
|
+
(selected by risk tier — low to high: `ollama-cloud` → `codex-cli` → `claude-cli`, degrading to nearest available); every
|
|
381
|
+
finding must cite a verbatim evidence quote that is mechanically verified
|
|
382
|
+
against the packet and the file's HEAD content before it's trusted, and
|
|
383
|
+
every fresh CRITICAL gets one cross-provider refute call before it can be
|
|
384
|
+
marked `confirmed` (when only one provider is installed the refute falls
|
|
385
|
+
back to the same provider — flagged `self_refute` in selection telemetry
|
|
386
|
+
and `self-refute:` in the finding record). Findings land in the ledger as
|
|
387
|
+
WARN — same bake
|
|
388
|
+
discipline as semgrep's: they surface at `pre-push` without blocking until
|
|
389
|
+
the operator explicitly ends the bake with `aramid arm --llm`, after which
|
|
390
|
+
`confirmed`-and-`critical` LLM findings BLOCK. A finding whose evidence quote
|
|
391
|
+
no longer appears in the file is auto-resolved before the block check runs,
|
|
392
|
+
so a fix is never held hostage by a stale finding.
|
|
393
|
+
|
|
394
|
+
The reviewer arm is selected deterministically by a risk-tiered ladder based
|
|
395
|
+
on the item's triage score: low-risk items (score 40–59) use ollama-cloud
|
|
396
|
+
(cheap tier), mid-risk (60–79) use codex-cli, and high-risk (80+) use
|
|
397
|
+
claude-cli (frontier tier). OpenRouter is available for opt-in use only —
|
|
398
|
+
not part of the default provider chain per the model-source policy; to enable
|
|
399
|
+
it, add `"openrouter"` to `[llm].provider_order` in `aramid.toml` and define an
|
|
400
|
+
`openrouter` arm in `[[llm.ladder]]` (with a model and min_score band).
|
|
401
|
+
|
|
402
|
+
**Auto-learn (learned uplift).** The deterministic ladder is a *floor*, not
|
|
403
|
+
the final answer: the auto-learn engine measures each arm's real-world miss
|
|
404
|
+
rate with **audit sampling** (1 in N below-frontier reviews is double-reviewed
|
|
405
|
+
by the frontier arm and the finding sets diffed — audit findings are filed for
|
|
406
|
+
real) and applies an escalate-only Thompson **uplift**: an item may be served
|
|
407
|
+
by a *higher* tier than its triage score suggests, never a lower one. It ships
|
|
408
|
+
shadow-first (bake-then-arm): with the default `[llm.autolearn] enabled = true,
|
|
409
|
+
armed = false` it records telemetry, shadow decisions, and audits but never
|
|
410
|
+
changes selection; `aramid arm --autolearn` arms it per-repo once
|
|
411
|
+
`aramid autolearn` shows a shadow record you trust. A **cascade** re-review
|
|
412
|
+
escalates one tier mid-drain (armed only) when a served review shows danger
|
|
413
|
+
signs (a verified CRITICAL, heavy hallucination rejections, a truncated
|
|
414
|
+
packet). Learning state is machine-global (`~/.aramid/autolearn_state.json`),
|
|
415
|
+
derived entirely from per-repo ledgers, and rebuildable at any time with
|
|
416
|
+
`aramid autolearn --rebuild`. Cold start, missing state, and any policy error
|
|
417
|
+
all degrade to exactly the deterministic ladder.
|
|
418
|
+
|
|
419
|
+
Setup: install the `claude` and/or `codex` CLI on `PATH` (`aramid doctor`
|
|
420
|
+
reports what it sees, informationally — LLM tooling never gates BLOCK-tier
|
|
421
|
+
status). Set `OLLAMA_API_KEY` in the environment to enable ollama-cloud.
|
|
422
|
+
OpenRouter is opt-in: set `OPENROUTER_API_KEY` and optionally cap spend via
|
|
423
|
+
`aramid.toml`'s `[llm].openrouter_monthly_cap_usd` (default `$5.00`/month,
|
|
424
|
+
checked against a local spend log before every call). All 2b knobs — provider
|
|
425
|
+
order, per-model overrides, timeouts, packet size cap, items-per-drain budget,
|
|
426
|
+
and the `llm_block_armed` bake flag itself — live under `[llm]` in `aramid.toml`.
|