ai-test-loop 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_test_loop/__init__.py +10 -0
- ai_test_loop/__main__.py +55 -0
- ai_test_loop/scripts/loop_runner.py +276 -0
- ai_test_loop-1.1.0.dist-info/METADATA +250 -0
- ai_test_loop-1.1.0.dist-info/RECORD +9 -0
- ai_test_loop-1.1.0.dist-info/WHEEL +5 -0
- ai_test_loop-1.1.0.dist-info/entry_points.txt +2 -0
- ai_test_loop-1.1.0.dist-info/licenses/LICENSE +21 -0
- ai_test_loop-1.1.0.dist-info/top_level.txt +1 -0
ai_test_loop/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""ai-test-loop — deterministic critic for AI agents.
|
|
2
|
+
|
|
3
|
+
Este pacote é um casco fino em volta do mesmo `loop_runner.py` que o pacote npm usa (lá em
|
|
4
|
+
versão Node). O contrato é idêntico: `SCORE=` / `GAPS=` / `EVIDENCE=` e exit code 0/1 para CI.
|
|
5
|
+
|
|
6
|
+
Por que o casco existe: `pip install ai-test-loop` precisa dar o mesmo comando que
|
|
7
|
+
`npx ai-test-loop`, senão o produto tem dois nomes e as estatísticas ficam partidas.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__version__ = "1.1.0"
|
ai_test_loop/__main__.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Console entry point for `ai-test-loop`.
|
|
2
|
+
|
|
3
|
+
Delega ao `loop_runner.py`, que continua sendo a implementação canônica em Python. Nada de
|
|
4
|
+
lógica duplicada aqui: se o runner mudar, este arquivo não precisa mudar.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _runner_path() -> str:
|
|
14
|
+
return os.path.join(os.path.dirname(os.path.abspath(__file__)), "scripts", "loop_runner.py")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def main() -> int:
|
|
18
|
+
argv = sys.argv[1:]
|
|
19
|
+
|
|
20
|
+
# --version e --help do PRODUTO, não do argparse interno: quem digita
|
|
21
|
+
# `ai-test-loop --version` quer a versão do pacote.
|
|
22
|
+
if "--version" in argv or "-V" in argv:
|
|
23
|
+
print(__version__)
|
|
24
|
+
return 0
|
|
25
|
+
if not argv or "--help" in argv or "-h" in argv:
|
|
26
|
+
print(
|
|
27
|
+
f"ai-test-loop {__version__} — nota 0-100 no seu agente antes de entregar.\n"
|
|
28
|
+
"\n"
|
|
29
|
+
"USO\n"
|
|
30
|
+
" ai-test-loop --config <arquivo.json> roda as checagens e imprime a nota\n"
|
|
31
|
+
" ai-test-loop --config <arq> --dry-run mostra o que rodaria, sem executar\n"
|
|
32
|
+
" ai-test-loop --config <arq> --quiet só SCORE/GAPS/EVIDENCE\n"
|
|
33
|
+
" ai-test-loop --config <arq> --json relatório completo em JSON\n"
|
|
34
|
+
" ai-test-loop --show-state última rodada e placar guardados\n"
|
|
35
|
+
" ai-test-loop --version versão do pacote\n"
|
|
36
|
+
"\n"
|
|
37
|
+
"Pacote completo, packs e suporte: "
|
|
38
|
+
"https://skills.seu.pet/p/ai-test-loop"
|
|
39
|
+
"?utm_source=cli&utm_medium=help&utm_campaign=ai-test-loop\n"
|
|
40
|
+
)
|
|
41
|
+
return 0
|
|
42
|
+
|
|
43
|
+
runner = _runner_path()
|
|
44
|
+
if not os.path.exists(runner):
|
|
45
|
+
sys.stderr.write(f"ai-test-loop: runner não encontrado em {runner}\n")
|
|
46
|
+
return 2
|
|
47
|
+
|
|
48
|
+
# exec em vez de subprocess: o exit code do runner (0/1/2) é o exit code do comando,
|
|
49
|
+
# que é exatamente o que o CI consome.
|
|
50
|
+
os.execv(sys.executable, [sys.executable, runner, *argv])
|
|
51
|
+
return 2 # inalcançável
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
if __name__ == "__main__":
|
|
55
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""loop_runner — the deterministic critic of the AI Test Loop.
|
|
3
|
+
|
|
4
|
+
Runs a list of weighed checks for real, captures their raw output as evidence, and prints:
|
|
5
|
+
|
|
6
|
+
SCORE=<0-100>
|
|
7
|
+
GAPS=["<check>: <why it failed>", ...]
|
|
8
|
+
EVIDENCE=<path to the JSON report>
|
|
9
|
+
|
|
10
|
+
The agent (builder) is not allowed to grade its own work; this script measures. It is
|
|
11
|
+
deliberately dumb: exit codes + optional output patterns decide pass/fail. No model calls,
|
|
12
|
+
no network, no telemetry.
|
|
13
|
+
|
|
14
|
+
Config (JSON):
|
|
15
|
+
|
|
16
|
+
{
|
|
17
|
+
"name": "my-project",
|
|
18
|
+
"threshold": 85,
|
|
19
|
+
"max_rounds": 8,
|
|
20
|
+
"checks": [
|
|
21
|
+
{"name": "unit tests", "cmd": "pytest -q", "weight": 40, "timeout_s": 900},
|
|
22
|
+
{"name": "build", "cmd": "npm run build", "weight": 25, "timeout_s": 600},
|
|
23
|
+
{"name": "lint", "cmd": "ruff check .", "weight": 10},
|
|
24
|
+
{"name": "no secrets", "cmd": "gitleaks detect --no-banner", "weight": 15,
|
|
25
|
+
"fail_if_output_matches": "leaks found: [1-9]"},
|
|
26
|
+
{"name": "api responds", "cmd": "curl -sS localhost:8000/health", "weight": 10,
|
|
27
|
+
"must_match": "\"status\"\\s*:\\s*\"ok\""}
|
|
28
|
+
]
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
Check keys:
|
|
32
|
+
cmd (required) shell command; runs with pipefail semantics via bash -o pipefail -c
|
|
33
|
+
weight (required) points; all weights are normalised, so they need not sum to 100
|
|
34
|
+
timeout_s (optional, default 900) hard timeout — a timeout is a FAIL, never a skip
|
|
35
|
+
expect_exit (optional, default 0) exit code that means pass
|
|
36
|
+
must_match (optional) regex that MUST appear in combined output
|
|
37
|
+
fail_if_output_matches (optional) regex that fails the check when it appears
|
|
38
|
+
required (optional, default true) when false, a FAIL scores 0 but is reported as a
|
|
39
|
+
non-blocking gap (useful for "nice to have" checks)
|
|
40
|
+
cwd (optional) working directory for this check
|
|
41
|
+
|
|
42
|
+
Usage:
|
|
43
|
+
python loop_runner.py --config loop.json [--round 2] [--json] [--quiet]
|
|
44
|
+
python loop_runner.py --config loop.json --show-state # print round history and stop
|
|
45
|
+
python loop_runner.py --config loop.json --dry-run # list checks, run nothing
|
|
46
|
+
|
|
47
|
+
Exit code: 0 when SCORE >= threshold (gate passed), 1 when below, 2 on usage/config error.
|
|
48
|
+
|
|
49
|
+
State: <config dir>/.<config-stem>-state.json accumulates every round, so a crash or a restart
|
|
50
|
+
never loses the evidence trail. The report of the latest round is written to
|
|
51
|
+
<config dir>/<config-stem>-report-r<round>.json. Names are derived from the config file so two
|
|
52
|
+
projects in the same folder never share a history.
|
|
53
|
+
"""
|
|
54
|
+
from __future__ import annotations
|
|
55
|
+
|
|
56
|
+
import argparse
|
|
57
|
+
import json
|
|
58
|
+
import os
|
|
59
|
+
import pathlib
|
|
60
|
+
import re
|
|
61
|
+
import shlex
|
|
62
|
+
import subprocess
|
|
63
|
+
import sys
|
|
64
|
+
import time
|
|
65
|
+
from datetime import datetime, timezone
|
|
66
|
+
|
|
67
|
+
DEFAULT_TIMEOUT = 900
|
|
68
|
+
MAX_OUTPUT_CHARS = 8000
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def load_config(path: str) -> dict:
|
|
72
|
+
try:
|
|
73
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
74
|
+
cfg = json.load(fh)
|
|
75
|
+
except FileNotFoundError:
|
|
76
|
+
sys.exit(f"[loop_runner] config not found: {path}")
|
|
77
|
+
except json.JSONDecodeError as exc:
|
|
78
|
+
sys.exit(f"[loop_runner] config is not valid JSON: {exc}")
|
|
79
|
+
checks = cfg.get("checks")
|
|
80
|
+
if not isinstance(checks, list) or not checks:
|
|
81
|
+
sys.exit("[loop_runner] config needs a non-empty 'checks' list")
|
|
82
|
+
for i, chk in enumerate(checks):
|
|
83
|
+
if not chk.get("cmd"):
|
|
84
|
+
sys.exit(f"[loop_runner] check #{i} has no 'cmd'")
|
|
85
|
+
if not isinstance(chk.get("weight", None), (int, float)) or chk["weight"] <= 0:
|
|
86
|
+
sys.exit(f"[loop_runner] check {chk.get('name', i)!r} needs a positive 'weight'")
|
|
87
|
+
cfg.setdefault("name", os.path.basename(os.path.dirname(os.path.abspath(path))) or "loop")
|
|
88
|
+
cfg.setdefault("threshold", 85)
|
|
89
|
+
cfg.setdefault("max_rounds", 8)
|
|
90
|
+
return cfg
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def run_check(chk: dict) -> dict:
|
|
94
|
+
"""Run one check for real, return its result record with raw evidence."""
|
|
95
|
+
name = chk.get("name") or chk["cmd"][:40]
|
|
96
|
+
timeout = int(chk.get("timeout_s", DEFAULT_TIMEOUT))
|
|
97
|
+
cwd = chk.get("cwd") or None
|
|
98
|
+
expect_exit = int(chk.get("expect_exit", 0))
|
|
99
|
+
# bash -o pipefail: without it, `pytest | tail` returns tail's exit code (0) and a failing
|
|
100
|
+
# check scores as passing — the single most common false green in this kind of runner.
|
|
101
|
+
wrapped = ["bash", "-o", "pipefail", "-c", chk["cmd"]]
|
|
102
|
+
started = time.time()
|
|
103
|
+
timed_out = False
|
|
104
|
+
try:
|
|
105
|
+
proc = subprocess.run(
|
|
106
|
+
wrapped, cwd=cwd, capture_output=True, text=True, timeout=timeout,
|
|
107
|
+
stdin=subprocess.DEVNULL,
|
|
108
|
+
)
|
|
109
|
+
rc, out, err = proc.returncode, proc.stdout or "", proc.stderr or ""
|
|
110
|
+
except subprocess.TimeoutExpired as exc:
|
|
111
|
+
timed_out = True
|
|
112
|
+
rc = None
|
|
113
|
+
out = (exc.stdout or b"").decode("utf-8", "replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")
|
|
114
|
+
err = (exc.stderr or b"").decode("utf-8", "replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "")
|
|
115
|
+
duration = round(time.time() - started, 2)
|
|
116
|
+
combined = (out or "") + ("\n" + err if err else "")
|
|
117
|
+
|
|
118
|
+
reasons: list[str] = []
|
|
119
|
+
if timed_out:
|
|
120
|
+
reasons.append(f"timed out after {timeout}s")
|
|
121
|
+
elif rc != expect_exit:
|
|
122
|
+
reasons.append(f"exit code {rc}, expected {expect_exit}")
|
|
123
|
+
|
|
124
|
+
must = chk.get("must_match")
|
|
125
|
+
if must and not re.search(must, combined):
|
|
126
|
+
reasons.append(f"output does not match required pattern /{must}/")
|
|
127
|
+
|
|
128
|
+
fail_pat = chk.get("fail_if_output_matches")
|
|
129
|
+
if fail_pat:
|
|
130
|
+
hit = re.search(fail_pat, combined)
|
|
131
|
+
if hit:
|
|
132
|
+
reasons.append(f"output matched failure pattern /{fail_pat}/: {hit.group(0)!r}")
|
|
133
|
+
|
|
134
|
+
passed = not reasons
|
|
135
|
+
evidence = combined.strip()
|
|
136
|
+
if len(evidence) > MAX_OUTPUT_CHARS:
|
|
137
|
+
evidence = evidence[: MAX_OUTPUT_CHARS // 2] + "\n...[truncated]...\n" + evidence[-MAX_OUTPUT_CHARS // 2:]
|
|
138
|
+
return {
|
|
139
|
+
"name": name,
|
|
140
|
+
"cmd": chk["cmd"],
|
|
141
|
+
"weight": float(chk["weight"]),
|
|
142
|
+
"required": bool(chk.get("required", True)),
|
|
143
|
+
"passed": passed,
|
|
144
|
+
"reasons": reasons,
|
|
145
|
+
"exit_code": rc,
|
|
146
|
+
"duration_s": duration,
|
|
147
|
+
"evidence": evidence,
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def score(checks: list[dict]) -> float:
|
|
152
|
+
total = sum(c["weight"] for c in checks) or 1.0
|
|
153
|
+
earned = sum(c["weight"] for c in checks if c["passed"])
|
|
154
|
+
return round(earned / total * 100, 1)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def main() -> int:
|
|
158
|
+
ap = argparse.ArgumentParser(
|
|
159
|
+
add_help=True,
|
|
160
|
+
description="Deterministic critic for the AI Test Loop",
|
|
161
|
+
# 09/11/2026: o --help era o UNICO ponto de contato do dev com o pacote e
|
|
162
|
+
# nao dizia nada sobre o resto do produto. Medido: 555 downloads/npm no mes
|
|
163
|
+
# e ZERO hits com utm_source=npm na loja. O epilog e o unico lugar honesto
|
|
164
|
+
# de por o link (nao interrompe a saida, nao quebra script de terceiro).
|
|
165
|
+
epilog=("Full version, packs and paid support: "
|
|
166
|
+
"https://skills.seu.pet/p/ai-test-loop"
|
|
167
|
+
"?utm_source=cli&utm_medium=help&utm_campaign=ai-test-loop"),
|
|
168
|
+
)
|
|
169
|
+
ap.add_argument("--config", required=True)
|
|
170
|
+
ap.add_argument("--round", type=int)
|
|
171
|
+
ap.add_argument("--json", action="store_true", help="also dump the full report as JSON to stdout")
|
|
172
|
+
ap.add_argument("--quiet", action="store_true", help="only SCORE/GAPS/EVIDENCE lines")
|
|
173
|
+
ap.add_argument("--dry-run", action="store_true")
|
|
174
|
+
ap.add_argument("--show-state", action="store_true")
|
|
175
|
+
args = ap.parse_args()
|
|
176
|
+
|
|
177
|
+
cfg_path = os.path.abspath(args.config)
|
|
178
|
+
cfg = load_config(cfg_path)
|
|
179
|
+
base = os.path.dirname(cfg_path)
|
|
180
|
+
# Per-config filenames: two configs in the same folder must not share one state file,
|
|
181
|
+
# otherwise rounds from project A pollute project B's history and round numbers.
|
|
182
|
+
stem = os.path.splitext(os.path.basename(cfg_path))[0]
|
|
183
|
+
state_path = os.path.join(base, f".{stem}-state.json")
|
|
184
|
+
|
|
185
|
+
if args.show_state:
|
|
186
|
+
if not os.path.exists(state_path):
|
|
187
|
+
print("[loop_runner] no state file yet")
|
|
188
|
+
return 0
|
|
189
|
+
with open(state_path, "r", encoding="utf-8") as fh:
|
|
190
|
+
st = json.load(fh)
|
|
191
|
+
for r in st.get("rounds", []):
|
|
192
|
+
print(f"round {r['round']}: SCORE={r['score']} threshold={r['threshold']} "
|
|
193
|
+
f"failed={[c['name'] for c in r['checks'] if not c['passed']]}")
|
|
194
|
+
return 0
|
|
195
|
+
|
|
196
|
+
if args.dry_run:
|
|
197
|
+
for c in cfg["checks"]:
|
|
198
|
+
print(f"[dry-run] {c.get('name', c['cmd'][:40])} weight={c['weight']} cmd={c['cmd']}")
|
|
199
|
+
print(f"[dry-run] threshold={cfg['threshold']} max_rounds={cfg['max_rounds']}")
|
|
200
|
+
return 0
|
|
201
|
+
|
|
202
|
+
round_no = args.round
|
|
203
|
+
if round_no is None:
|
|
204
|
+
prev = 0
|
|
205
|
+
if os.path.exists(state_path):
|
|
206
|
+
with open(state_path, "r", encoding="utf-8") as fh:
|
|
207
|
+
prev = len(json.load(fh).get("rounds", []))
|
|
208
|
+
round_no = prev + 1
|
|
209
|
+
|
|
210
|
+
if round_no > int(cfg["max_rounds"]):
|
|
211
|
+
print(f"ROUND: {round_no} ABORT: exceeded max_rounds={cfg['max_rounds']}")
|
|
212
|
+
print("GAPS=[\"round budget exhausted — the check set or the threshold needs revisiting\"]")
|
|
213
|
+
return 1
|
|
214
|
+
|
|
215
|
+
results = []
|
|
216
|
+
for chk in cfg["checks"]:
|
|
217
|
+
if not args.quiet:
|
|
218
|
+
print(f"[loop_runner] round {round_no}: running {chk.get('name', chk['cmd'][:40])!r} ...",
|
|
219
|
+
file=sys.stderr, flush=True)
|
|
220
|
+
results.append(run_check(chk))
|
|
221
|
+
|
|
222
|
+
value = score(results)
|
|
223
|
+
threshold = float(cfg["threshold"])
|
|
224
|
+
gate = value >= threshold
|
|
225
|
+
gaps = [f"{c['name']}: {'; '.join(c['reasons'])}" for c in results if not c["passed"] and c["required"]]
|
|
226
|
+
soft = [f"{c['name']}: {'; '.join(c['reasons'])}" for c in results if not c["passed"] and not c["required"]]
|
|
227
|
+
|
|
228
|
+
report = {
|
|
229
|
+
"project": cfg["name"],
|
|
230
|
+
"round": round_no,
|
|
231
|
+
"ts": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
232
|
+
"threshold": threshold,
|
|
233
|
+
"score": value,
|
|
234
|
+
"gate": "passed" if gate else "failed",
|
|
235
|
+
"checks": results,
|
|
236
|
+
"gaps": gaps,
|
|
237
|
+
"non_blocking_gaps": soft,
|
|
238
|
+
}
|
|
239
|
+
report_path = os.path.join(base, f"{stem}-report-r{round_no}.json")
|
|
240
|
+
pathlib.Path(report_path).write_text(
|
|
241
|
+
json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
242
|
+
|
|
243
|
+
state = {"project": cfg["name"], "rounds": []}
|
|
244
|
+
if os.path.exists(state_path):
|
|
245
|
+
try:
|
|
246
|
+
with open(state_path, "r", encoding="utf-8") as fh:
|
|
247
|
+
state = json.load(fh)
|
|
248
|
+
state.setdefault("rounds", [])
|
|
249
|
+
except json.JSONDecodeError:
|
|
250
|
+
pass
|
|
251
|
+
state["rounds"].append({"round": round_no, "score": value, "threshold": threshold,
|
|
252
|
+
"gate": report["gate"], "ts": report["ts"], "report": report_path,
|
|
253
|
+
"checks": results})
|
|
254
|
+
pathlib.Path(state_path).write_text(
|
|
255
|
+
json.dumps(state, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
256
|
+
|
|
257
|
+
if not args.quiet:
|
|
258
|
+
for c in results:
|
|
259
|
+
status = "PASS" if c["passed"] else "FAIL"
|
|
260
|
+
detail = c["evidence"].strip().splitlines()[-1][:160] if c["evidence"].strip() else "(no output)"
|
|
261
|
+
print(f" {status} {c['name']:<28} w={c['weight']:<5} {c['duration_s']:>7}s {detail}")
|
|
262
|
+
|
|
263
|
+
print(f"SCORE={value}")
|
|
264
|
+
print(f"THRESHOLD={threshold}")
|
|
265
|
+
print(f"GATE={'passed' if gate else 'failed'} (round {round_no}/{cfg['max_rounds']})")
|
|
266
|
+
print(f"GAPS={json.dumps(gaps, ensure_ascii=False)}")
|
|
267
|
+
if soft:
|
|
268
|
+
print(f"NON_BLOCKING_GAPS={json.dumps(soft, ensure_ascii=False)}")
|
|
269
|
+
print(f"EVIDENCE={report_path}")
|
|
270
|
+
if args.json:
|
|
271
|
+
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
272
|
+
return 0 if gate else 1
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
if __name__ == "__main__":
|
|
276
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ai-test-loop
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Deterministic critic for AI agents: runs weighed checks for real, scores 0-100 and blocks delivery below the threshold.
|
|
5
|
+
Author: Samuel Medeiros
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://skills.seu.pet/en/p/ai-test-loop?utm_source=pypi&utm_medium=readme&utm_campaign=ai-test-loop
|
|
8
|
+
Project-URL: Repository, https://github.com/Samuelfmedeiros/ai-test-loop
|
|
9
|
+
Project-URL: Changelog, https://github.com/Samuelfmedeiros/ai-test-loop/releases
|
|
10
|
+
Keywords: mcp,ai-agent,ai-agents,llm,testing,test-loop,quality-gate,delivery-gate,critic,claude-code,cursor,code-review
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# ai-test-loop
|
|
23
|
+
|
|
24
|
+
**Your agent grades its own homework.** This makes that impossible: a critic script runs real checks,
|
|
25
|
+
scores the artifact 0–100, and a fail-closed gate blocks delivery below the threshold.
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
npx ai-test-loop --config loop.json # zero install, no Python needed
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Real output, not a mockup:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
PASS unit tests w=40 12.4s 41 passed in 12.31s
|
|
35
|
+
FAIL build w=20 3.1s error TS2307: Cannot find module './api'
|
|
36
|
+
PASS no secrets w=15 0.8s no leaks found
|
|
37
|
+
SCORE=75.0
|
|
38
|
+
GATE=failed (round 1/8)
|
|
39
|
+
GAPS=["build: exit code 2, expected 0"]
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
> **Want this wired into your repo, with the visual-diff gate and the delivery report?**
|
|
43
|
+
> The ready-made packs are at **https://skills.seu.pet/en/p/pack-completo?utm_source=npm&utm_medium=readme&utm_campaign=ai-test-loop**
|
|
44
|
+
> — or grab the free runner first and see if the loop earns its place.
|
|
45
|
+
|
|
46
|
+
No API keys, no dependencies, no model calls. The runner is deliberately dumb: exit codes and
|
|
47
|
+
optional output patterns decide pass/fail. The model never gets to *say* the work is good — it has
|
|
48
|
+
to *show* it.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## Install
|
|
53
|
+
|
|
54
|
+
Same package name everywhere. Both registries, same name:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
npx ai-test-loop --help # Node 18+ — nothing to install
|
|
58
|
+
pip install ai-test-loop # Python 3.9+ — same engine (PyPI)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
`npx` is the fastest path: it downloads and runs in about five seconds, no virtualenv, no Python.
|
|
62
|
+
The npm package carries a Node port of the runner; the PyPI package carries the Python one. Both
|
|
63
|
+
print the same `SCORE=` / `GAPS=` / `EVIDENCE=` contract, so the gate you run locally is the gate CI
|
|
64
|
+
runs.
|
|
65
|
+
|
|
66
|
+
Verify with a dry run — it lists the checks and weights without executing anything:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
npx ai-test-loop --config templates/loop.example.json --dry-run
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
> Older docs and articles mention `pip install agent-test-loop`. That is the previous PyPI name and
|
|
73
|
+
> still works — it is the same engine, kept as a mirror so existing installs do not break. New code
|
|
74
|
+
> should use `ai-test-loop`.
|
|
75
|
+
|
|
76
|
+
## The problem
|
|
77
|
+
|
|
78
|
+
Ask an agent "is this done?" and it answers about its own reasoning. The same context that produced
|
|
79
|
+
the artifact also decides it is correct, so real defects survive to delivery:
|
|
80
|
+
|
|
81
|
+
- a handler that is never called (tests pass, feature dead)
|
|
82
|
+
- a test that passes because it asserts nothing
|
|
83
|
+
- a page that breaks only at mobile width
|
|
84
|
+
- a migration that works on the author's data and nowhere else
|
|
85
|
+
- a "fix" that reverted a config someone changed on purpose
|
|
86
|
+
|
|
87
|
+
Running "one more review pass" does not help, because the reviewer and the builder are the same
|
|
88
|
+
process. Separating them is the whole trick.
|
|
89
|
+
|
|
90
|
+
## Quickstart
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
npx ai-test-loop --config loop.json
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Describe your checks once — name, command, weight:
|
|
97
|
+
|
|
98
|
+
```json
|
|
99
|
+
{
|
|
100
|
+
"name": "my-project",
|
|
101
|
+
"threshold": 85,
|
|
102
|
+
"max_rounds": 8,
|
|
103
|
+
"checks": [
|
|
104
|
+
{"name": "unit tests", "cmd": "pytest -q", "weight": 35, "timeout_s": 900},
|
|
105
|
+
{"name": "type check", "cmd": "mypy src", "weight": 10},
|
|
106
|
+
{"name": "build", "cmd": "npm run build", "weight": 20},
|
|
107
|
+
{"name": "api up", "cmd": "curl -sS 127.0.0.1:8000/health", "weight": 15,
|
|
108
|
+
"must_match": "\"status\"\\s*:\\s*\"ok\""},
|
|
109
|
+
{"name": "no secrets", "cmd": "gitleaks detect --no-banner", "weight": 20,
|
|
110
|
+
"fail_if_output_matches": "leaks found: [1-9]", "required": true}
|
|
111
|
+
]
|
|
112
|
+
}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Output is machine-readable, so the *agent* can act on it and a CI job can gate on it:
|
|
116
|
+
|
|
117
|
+
```
|
|
118
|
+
SCORE=75.0
|
|
119
|
+
THRESHOLD=85
|
|
120
|
+
GATE=failed (round 1/8)
|
|
121
|
+
GAPS=["build: exit code 2, expected 0"]
|
|
122
|
+
EVIDENCE=loop-report-r1.json
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Exit code is `0` when the gate passes and `1` when it fails — so `npx ai-test-loop --config
|
|
126
|
+
loop.json` is a valid CI step on its own, with no parsing.
|
|
127
|
+
|
|
128
|
+
Fix the **single largest** gap, run `--round 2`, repeat. Cap the rounds (8 is plenty): if the score
|
|
129
|
+
never converges, the check set is wrong, not the code.
|
|
130
|
+
|
|
131
|
+
## Config reference
|
|
132
|
+
|
|
133
|
+
| Key | Meaning |
|
|
134
|
+
|---|---|
|
|
135
|
+
| `cmd` | (required) shell command, run with `bash -o pipefail -c` |
|
|
136
|
+
| `weight` | (required) points; weights are normalised, they need not sum to 100 |
|
|
137
|
+
| `timeout_s` | default 900 — a timeout is a **FAIL**, never a skip |
|
|
138
|
+
| `expect_exit` | default 0 |
|
|
139
|
+
| `must_match` | regex that must appear in the combined stdout+stderr |
|
|
140
|
+
| `fail_if_output_matches` | regex that fails the check when it appears |
|
|
141
|
+
| `required` | default `true`; `false` scores 0 but reports as a non-blocking gap |
|
|
142
|
+
| `cwd` | working directory for that check |
|
|
143
|
+
|
|
144
|
+
`bash -o pipefail` is not decoration: without it `pytest | tail` returns tail's exit code (0), so a
|
|
145
|
+
failing suite scores as passing. That is the most common false green in runners like this one.
|
|
146
|
+
|
|
147
|
+
## The four roles (never merge them)
|
|
148
|
+
|
|
149
|
+
| Role | Who | Job |
|
|
150
|
+
|---|---|---|
|
|
151
|
+
| Orchestrator | the agent running the task | picks checks, runs rounds, owns the report |
|
|
152
|
+
| Builder | the agent/model that wrote the artifact | fixes one class of gap per pass |
|
|
153
|
+
| Critic | this script + your checks | runs real commands, returns score + gaps |
|
|
154
|
+
| Reviewer | a **separate** context/model | audits critic and builder, can reopen the loop |
|
|
155
|
+
|
|
156
|
+
Rule zero: builder ≠ critic ≠ reviewer. If one context does all three, the loop is theatre.
|
|
157
|
+
|
|
158
|
+
## Three rules that make it honest
|
|
159
|
+
|
|
160
|
+
1. **Every point is earned by a command that actually ran.** No points for "read the code". The
|
|
161
|
+
report stores the raw output next to each score.
|
|
162
|
+
2. **`NOT VERIFIABLE` is not a pass.** If a check cannot run (missing device, no network, jail), it
|
|
163
|
+
scores 0 and the report says why. Silent skips are how a green board lies.
|
|
164
|
+
3. **The security gate is fail-closed.** A secret in the diff or history, a high-severity SAST
|
|
165
|
+
finding, or an auth/path-traversal/injection issue found in review blocks delivery **even with a
|
|
166
|
+
perfect score**. An inconclusive answer counts as blocked, never as pass.
|
|
167
|
+
|
|
168
|
+
## Reviewer panel
|
|
169
|
+
|
|
170
|
+
For anything other people will use, ask 2–3 specialists instead of one generalist — quality and
|
|
171
|
+
security always, performance only if there is a build or runtime cost, UX only if there is a
|
|
172
|
+
user-facing surface. Each returns strict JSON:
|
|
173
|
+
|
|
174
|
+
```json
|
|
175
|
+
{"parecer":"CONCORDO|DISCORDO","nota":0-100,"gaps":["..."],"texto":"one paragraph with evidence"}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Consensus → deliver. Majority disagree → reopen the loop with the combined gaps. Prompt in
|
|
179
|
+
`references/reviewer-prompt.md`, starting rubric in `references/scoring-rubric.md`.
|
|
180
|
+
|
|
181
|
+
## Telemetry (opt-out, and what it is not)
|
|
182
|
+
|
|
183
|
+
The `npx ai-test-loop` wrapper sends **one anonymous event per run** so the project knows whether
|
|
184
|
+
the tool is actually used — downloads alone are not usage (npm counts mirrors and CI).
|
|
185
|
+
|
|
186
|
+
What is sent: event type (`run`/`help`/`version`/`erro`), package version, Node major, OS name,
|
|
187
|
+
duration in ms, and — only on a real run — the final score and the number of gaps.
|
|
188
|
+
|
|
189
|
+
What is **never** sent: your IP (not logged server-side), e-mail, project path, file names, the
|
|
190
|
+
content of your checks, the command output, or anything from your code.
|
|
191
|
+
|
|
192
|
+
The install id is a random UUID written to `~/.ai-test-loop/telemetry-id` once. It counts machines,
|
|
193
|
+
not people. Delete that file and you are a new machine.
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
ai-test-loop --no-telemetry ... # that run only
|
|
197
|
+
export AI_TEST_LOOP_NO_TELEMETRY=1 # permanently, in your shell profile
|
|
198
|
+
export DO_NOT_TRACK=1 # the ecosystem-wide convention, also respected
|
|
199
|
+
# CI=1 is detected automatically: pipelines are not people
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Telemetry never blocks and never waits: if the network is down the CLI runs exactly the same.
|
|
203
|
+
|
|
204
|
+
## When not to use it
|
|
205
|
+
|
|
206
|
+
Throwaway exploration, spike branches, and one-line answers where the cost of the loop exceeds the
|
|
207
|
+
cost of being wrong. It is for artifacts a human will rely on.
|
|
208
|
+
|
|
209
|
+
## Files
|
|
210
|
+
|
|
211
|
+
| Path | What |
|
|
212
|
+
|---|---|
|
|
213
|
+
| `bin/cli.js` | the `npx` entry point (wrapper + telemetry) |
|
|
214
|
+
| `scripts/loop_runner.js` | the deterministic critic, Node port — zero dependencies |
|
|
215
|
+
| `scripts/loop_runner.py` | the same critic in Python (used by the pip package) |
|
|
216
|
+
| `scripts/selftest.js` | 22 assertions covering every way a check can fail — `npm test` |
|
|
217
|
+
| `SKILL.md` | the loop, the scoring rules, the fail-closed gate — drop it in your agent's skills dir |
|
|
218
|
+
| `references/reviewer-prompt.md` | independent-review prompt, strict JSON output |
|
|
219
|
+
| `references/scoring-rubric.md` | starting weights |
|
|
220
|
+
| `templates/loop.example.json` | config with the usual check set |
|
|
221
|
+
|
|
222
|
+
`npm test` runs the selftest. It needs nothing but node.
|
|
223
|
+
|
|
224
|
+
## Get the full pack
|
|
225
|
+
|
|
226
|
+
This repo is the free, complete tool. The store has the ready-made packs for the cases where
|
|
227
|
+
writing the checks is the slow part — CI gates, visual regression, MCP servers, MLOps guards:
|
|
228
|
+
|
|
229
|
+
- Free download / e-mail delivery: **https://skills.seu.pet/gratis/ai-test-loop?utm_source=npm&utm_medium=readme&utm_campaign=ai-test-loop**
|
|
230
|
+
- Full pack: **https://skills.seu.pet/en/p/pack-completo?utm_source=npm&utm_medium=readme&utm_campaign=ai-test-loop**
|
|
231
|
+
- Catalog: **https://skills.seu.pet/?utm_source=npm&utm_medium=readme&utm_campaign=ai-test-loop**
|
|
232
|
+
- Want it installed and wired into your repo: **https://skills.seu.pet/servico?utm_source=npm&utm_medium=readme&utm_campaign=ai-test-loop**
|
|
233
|
+
|
|
234
|
+
## License
|
|
235
|
+
|
|
236
|
+
MIT — see `LICENSE`. Use it commercially, fork it, ship it.
|
|
237
|
+
|
|
238
|
+
---
|
|
239
|
+
|
|
240
|
+
### Em português
|
|
241
|
+
|
|
242
|
+
O loop de teste para agentes: o construtor não assina o próprio trabalho. Um crítico roda checagens
|
|
243
|
+
de verdade, dá nota 0–100 e barra a entrega abaixo do limite.
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
npx ai-test-loop --config loop.json # não precisa de Python
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
Sem chave de API, sem dependência, sem chamada de modelo. Versão com entrega por e-mail e os packs
|
|
250
|
+
prontos: https://skills.seu.pet/gratis/ai-test-loop?utm_source=npm&utm_medium=readme&utm_campaign=ai-test-loop
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
ai_test_loop/__init__.py,sha256=THy6dEUGQpwun6Xv3iqbrqjZ3s60iLIJIbnNsHr0vug,446
|
|
2
|
+
ai_test_loop/__main__.py,sha256=J-hOnFx_H11V94W-NDHR2gJu4XLEe2PEB_M1br_g-VM,2062
|
|
3
|
+
ai_test_loop/scripts/loop_runner.py,sha256=O_UCvaIirTTF5ILC8kbGFyHU0PSMSky_fJySuj2SJHE,11681
|
|
4
|
+
ai_test_loop-1.1.0.dist-info/licenses/LICENSE,sha256=wRooGlQmxnT_ULZwshgiibjk5TQpOtzmpTcx7kHmMt0,1072
|
|
5
|
+
ai_test_loop-1.1.0.dist-info/METADATA,sha256=pf-IDy5QdT8gxPK4LIram-inyi6RXvaDzSG4v-H5pJ0,10751
|
|
6
|
+
ai_test_loop-1.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
7
|
+
ai_test_loop-1.1.0.dist-info/entry_points.txt,sha256=LQAh2300QdMAIp1r6eqB3lYQRTywqrNV74NR-DWocGI,60
|
|
8
|
+
ai_test_loop-1.1.0.dist-info/top_level.txt,sha256=oMZYHxa6Z0FE_AkxJqmmKvviqz7ZRHvR32l28EPO52c,13
|
|
9
|
+
ai_test_loop-1.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Samuel Medeiros
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ai_test_loop
|