ai-test-loop 1.1.0__tar.gz → 1.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ai_test_loop-1.1.0/ai_test_loop.egg-info → ai_test_loop-1.1.2}/PKG-INFO +3 -3
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/ai_test_loop/__init__.py +1 -1
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/ai_test_loop/scripts/loop_runner.py +31 -3
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2/ai_test_loop.egg-info}/PKG-INFO +3 -3
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/ai_test_loop.egg-info/SOURCES.txt +1 -2
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/pyproject.toml +6 -5
- ai_test_loop-1.1.0/tests/test_cli.py +0 -117
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/LICENSE +0 -0
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/MANIFEST.in +0 -0
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/README.md +0 -0
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/ai_test_loop/__main__.py +0 -0
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/ai_test_loop.egg-info/dependency_links.txt +0 -0
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/ai_test_loop.egg-info/entry_points.txt +0 -0
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/ai_test_loop.egg-info/top_level.txt +0 -0
- {ai_test_loop-1.1.0 → ai_test_loop-1.1.2}/setup.cfg +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ai-test-loop
|
|
3
|
-
Version: 1.1.
|
|
4
|
-
Summary: Deterministic critic for AI agents: runs
|
|
3
|
+
Version: 1.1.2
|
|
4
|
+
Summary: Deterministic critic for AI agents: runs weighted checks for real, scores 0-100 and blocks the delivery when it should.
|
|
5
5
|
Author: Samuel Medeiros
|
|
6
6
|
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://skills.seu.pet/
|
|
7
|
+
Project-URL: Homepage, https://skills.seu.pet/p/ai-test-loop?utm_source=pypi&utm_medium=readme&utm_campaign=ai-test-loop
|
|
8
8
|
Project-URL: Repository, https://github.com/Samuelfmedeiros/ai-test-loop
|
|
9
9
|
Project-URL: Changelog, https://github.com/Samuelfmedeiros/ai-test-loop/releases
|
|
10
10
|
Keywords: mcp,ai-agent,ai-agents,llm,testing,test-loop,quality-gate,delivery-gate,critic,claude-code,cursor,code-review
|
|
@@ -66,6 +66,7 @@ from datetime import datetime, timezone
|
|
|
66
66
|
|
|
67
67
|
DEFAULT_TIMEOUT = 900
|
|
68
68
|
MAX_OUTPUT_CHARS = 8000
|
|
69
|
+
_SHELL: list[str] | None = None
|
|
69
70
|
|
|
70
71
|
|
|
71
72
|
def load_config(path: str) -> dict:
|
|
@@ -90,15 +91,42 @@ def load_config(path: str) -> dict:
|
|
|
90
91
|
return cfg
|
|
91
92
|
|
|
92
93
|
|
|
94
|
+
def _shell() -> list[str]:
|
|
95
|
+
"""Shell wrapper for one check, decided once per process.
|
|
96
|
+
|
|
97
|
+
bash -o pipefail is the strong default: without it `pytest | tail` returns tail's
|
|
98
|
+
exit code (0) and a failing check scores as passing — the most common false green.
|
|
99
|
+
|
|
100
|
+
But hardcoding "bash" crashes on machines that have no bash (minimal containers,
|
|
101
|
+
alpine, stripped PATH). Measured: FileNotFoundError traceback, not a gate failure.
|
|
102
|
+
So: bash when available, /bin/sh on POSIX (pipefail lost but the check runs), and
|
|
103
|
+
cmd.exe on Windows (where bash only exists if Git Bash is installed).
|
|
104
|
+
"""
|
|
105
|
+
global _SHELL
|
|
106
|
+
if _SHELL is not None:
|
|
107
|
+
return _SHELL
|
|
108
|
+
def works(cmd: str) -> bool:
|
|
109
|
+
try:
|
|
110
|
+
return subprocess.run([cmd, "-c", "exit 0"], capture_output=True).returncode == 0
|
|
111
|
+
except (FileNotFoundError, PermissionError, OSError):
|
|
112
|
+
return False
|
|
113
|
+
if os.name == "nt":
|
|
114
|
+
_SHELL = ["bash", "-o", "pipefail", "-c"] if works("bash") else [
|
|
115
|
+
os.environ.get("ComSpec", "cmd.exe"), "/d", "/s", "/c"]
|
|
116
|
+
else:
|
|
117
|
+
_SHELL = ["bash", "-o", "pipefail", "-c"] if works("bash") else ["/bin/sh", "-c"]
|
|
118
|
+
return _SHELL
|
|
119
|
+
|
|
120
|
+
|
|
93
121
|
def run_check(chk: dict) -> dict:
|
|
94
122
|
"""Run one check for real, return its result record with raw evidence."""
|
|
95
123
|
name = chk.get("name") or chk["cmd"][:40]
|
|
96
124
|
timeout = int(chk.get("timeout_s", DEFAULT_TIMEOUT))
|
|
97
125
|
cwd = chk.get("cwd") or None
|
|
98
126
|
expect_exit = int(chk.get("expect_exit", 0))
|
|
99
|
-
# bash -o pipefail:
|
|
100
|
-
#
|
|
101
|
-
wrapped =
|
|
127
|
+
# bash -o pipefail: sem ele `pytest | tail` devolve o exit code do tail (0) e o check
|
|
128
|
+
# que falhou pontua como aprovado — o falso verde mais comum deste tipo de runner.
|
|
129
|
+
wrapped = _shell() + [chk["cmd"]]
|
|
102
130
|
started = time.time()
|
|
103
131
|
timed_out = False
|
|
104
132
|
try:
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ai-test-loop
|
|
3
|
-
Version: 1.1.
|
|
4
|
-
Summary: Deterministic critic for AI agents: runs
|
|
3
|
+
Version: 1.1.2
|
|
4
|
+
Summary: Deterministic critic for AI agents: runs weighted checks for real, scores 0-100 and blocks the delivery when it should.
|
|
5
5
|
Author: Samuel Medeiros
|
|
6
6
|
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://skills.seu.pet/
|
|
7
|
+
Project-URL: Homepage, https://skills.seu.pet/p/ai-test-loop?utm_source=pypi&utm_medium=readme&utm_campaign=ai-test-loop
|
|
8
8
|
Project-URL: Repository, https://github.com/Samuelfmedeiros/ai-test-loop
|
|
9
9
|
Project-URL: Changelog, https://github.com/Samuelfmedeiros/ai-test-loop/releases
|
|
10
10
|
Keywords: mcp,ai-agent,ai-agents,llm,testing,test-loop,quality-gate,delivery-gate,critic,claude-code,cursor,code-review
|
|
@@ -3,9 +3,11 @@ requires = ["setuptools>=68", "wheel"]
|
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
|
+
# Nome canônico do produto. `agent-test-loop` é o espelho aposentado que existe só
|
|
7
|
+
# para não quebrar requirements.txt antigo — ele depende deste pacote.
|
|
6
8
|
name = "ai-test-loop"
|
|
7
|
-
version = "1.1.
|
|
8
|
-
description = "Deterministic critic for AI agents: runs
|
|
9
|
+
version = "1.1.2"
|
|
10
|
+
description = "Deterministic critic for AI agents: runs weighted checks for real, scores 0-100 and blocks the delivery when it should."
|
|
9
11
|
readme = "README.md"
|
|
10
12
|
requires-python = ">=3.9"
|
|
11
13
|
license = { text = "MIT" }
|
|
@@ -22,10 +24,9 @@ classifiers = [
|
|
|
22
24
|
"Topic :: Software Development :: Quality Assurance",
|
|
23
25
|
"Topic :: Software Development :: Testing",
|
|
24
26
|
]
|
|
25
|
-
dependencies = []
|
|
26
27
|
|
|
27
28
|
[project.urls]
|
|
28
|
-
Homepage = "https://skills.seu.pet/
|
|
29
|
+
Homepage = "https://skills.seu.pet/p/ai-test-loop?utm_source=pypi&utm_medium=readme&utm_campaign=ai-test-loop"
|
|
29
30
|
Repository = "https://github.com/Samuelfmedeiros/ai-test-loop"
|
|
30
31
|
Changelog = "https://github.com/Samuelfmedeiros/ai-test-loop/releases"
|
|
31
32
|
|
|
@@ -33,7 +34,7 @@ Changelog = "https://github.com/Samuelfmedeiros/ai-test-loop/releases"
|
|
|
33
34
|
ai-test-loop = "ai_test_loop.__main__:main"
|
|
34
35
|
|
|
35
36
|
[tool.setuptools]
|
|
36
|
-
packages = ["ai_test_loop"]
|
|
37
|
+
packages = ["ai_test_loop", "ai_test_loop.scripts"]
|
|
37
38
|
|
|
38
39
|
[tool.setuptools.package-data]
|
|
39
40
|
ai_test_loop = ["scripts/*.py"]
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Testes do CLI do ai-test-loop — o que o usuario ve ao rodar `npx ai-test-loop`.
|
|
3
|
-
|
|
4
|
-
Por que existe: a 1.0.2 ficou semanas publicada com `--version` quebrado e sem
|
|
5
|
-
argumento cuspindo erro cru do argparse, porque o pacote nao tinha NENHUM teste.
|
|
6
|
-
555 downloads no mes e nenhum sinal de que o comando documentado funcionava.
|
|
7
|
-
|
|
8
|
-
Nao usa pytest: o pacote e stdlib-only por design, e estes testes rodam com
|
|
9
|
-
`python3 tests/test_cli.py` em qualquer maquina com node.
|
|
10
|
-
"""
|
|
11
|
-
import json
|
|
12
|
-
import os
|
|
13
|
-
import subprocess
|
|
14
|
-
import sys
|
|
15
|
-
import tempfile
|
|
16
|
-
import unittest
|
|
17
|
-
|
|
18
|
-
RAIZ = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
19
|
-
CLI = os.path.join(RAIZ, "bin", "cli.js")
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def roda(*args, env_extra=None, espera_erro=False):
|
|
23
|
-
"""Executa o CLI e devolve (rc, stdout, stderr)."""
|
|
24
|
-
env = dict(os.environ)
|
|
25
|
-
# telemetria desligada: teste nao deve escrever no servidor do site
|
|
26
|
-
env["AI_TEST_LOOP_NO_TELEMETRY"] = "1"
|
|
27
|
-
if env_extra:
|
|
28
|
-
env.update(env_extra)
|
|
29
|
-
p = subprocess.run(
|
|
30
|
-
["node", CLI, *args],
|
|
31
|
-
capture_output=True, text=True, timeout=120, env=env,
|
|
32
|
-
)
|
|
33
|
-
return p.returncode, p.stdout, p.stderr
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class TestAjuda(unittest.TestCase):
|
|
37
|
-
def test_help_mostra_a_ajuda_do_produto(self):
|
|
38
|
-
"""`--help` tem de explicar o que a ferramenta faz, nao o usage do Python."""
|
|
39
|
-
rc, out, _ = roda("--help")
|
|
40
|
-
self.assertEqual(rc, 0)
|
|
41
|
-
self.assertIn("ai-test-loop", out)
|
|
42
|
-
self.assertIn("USO", out)
|
|
43
|
-
self.assertIn("--config", out)
|
|
44
|
-
self.assertIn("PRIMEIRA VEZ", out)
|
|
45
|
-
# o usage do argparse do runner nao deve vazar
|
|
46
|
-
self.assertNotIn("loop_runner.py [-h]", out)
|
|
47
|
-
self.assertNotIn("show this help message and exit", out)
|
|
48
|
-
|
|
49
|
-
def test_help_leva_para_a_loja(self):
|
|
50
|
-
"""O help e o degrau mais barato do funil: tem de ter o link."""
|
|
51
|
-
_, out, _ = roda("--help")
|
|
52
|
-
self.assertIn("skills.seu.pet/p/ai-test-loop", out)
|
|
53
|
-
self.assertIn("utm_source=cli", out)
|
|
54
|
-
|
|
55
|
-
def test_sem_argumento_mostra_ajuda_em_vez_de_erro(self):
|
|
56
|
-
"""Sem argumento o usuario via 'error: --config is required' e ia embora."""
|
|
57
|
-
rc, out, err = roda()
|
|
58
|
-
self.assertEqual(rc, 0, f"sem argumento deveria sair limpo, veio rc={rc}")
|
|
59
|
-
self.assertIn("USO", out)
|
|
60
|
-
self.assertNotIn("the following arguments are required", out + err)
|
|
61
|
-
self.assertNotIn("usage: loop_runner.py", out + err)
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
class TestVersao(unittest.TestCase):
|
|
65
|
-
def test_version_responde_a_versao_do_pacote(self):
|
|
66
|
-
rc, out, err = roda("--version")
|
|
67
|
-
self.assertEqual(rc, 0)
|
|
68
|
-
with open(os.path.join(RAIZ, "package.json"), encoding="utf-8") as f:
|
|
69
|
-
versao = json.load(f)["version"]
|
|
70
|
-
self.assertEqual(out.strip(), versao, "a saida tem de ser SO a versao")
|
|
71
|
-
# o Python nao pode rodar depois e imprimir o usage dele
|
|
72
|
-
self.assertNotIn("loop_runner.py", out + err)
|
|
73
|
-
self.assertNotIn("--config", out + err)
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
class TestRunReal(unittest.TestCase):
|
|
77
|
-
def test_dry_run_lista_os_checks(self):
|
|
78
|
-
rc, out, _ = roda("--config", os.path.join(RAIZ, "templates", "loop.example.json"),
|
|
79
|
-
"--dry-run")
|
|
80
|
-
self.assertEqual(rc, 0)
|
|
81
|
-
self.assertIn("[dry-run]", out)
|
|
82
|
-
self.assertIn("threshold=", out)
|
|
83
|
-
|
|
84
|
-
def test_run_de_verdade_da_nota_e_gaps(self):
|
|
85
|
-
"""Um run real tem de produzir SCORE, GATE e GAPS — nao so ecoar."""
|
|
86
|
-
with tempfile.TemporaryDirectory() as d:
|
|
87
|
-
cfg = os.path.join(d, "mini.json")
|
|
88
|
-
with open(cfg, "w", encoding="utf-8") as f:
|
|
89
|
-
json.dump({
|
|
90
|
-
"threshold": 50, "max_rounds": 1,
|
|
91
|
-
"checks": [
|
|
92
|
-
{"name": "passa", "cmd": "true", "weight": 50, "pass_when": "exit0"},
|
|
93
|
-
{"name": "falha", "cmd": "false", "weight": 50, "pass_when": "exit0"},
|
|
94
|
-
],
|
|
95
|
-
}, f)
|
|
96
|
-
rc, out, _ = roda("--config", cfg)
|
|
97
|
-
self.assertIn("SCORE=", out)
|
|
98
|
-
self.assertIn("GATE=", out)
|
|
99
|
-
self.assertIn("GAPS=", out)
|
|
100
|
-
# 50 de 100: um passa, um falha, e o gap nomeia qual
|
|
101
|
-
self.assertIn("SCORE=50", out)
|
|
102
|
-
self.assertIn("falha", out.split("GAPS=")[1])
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
class TestPythonAusente(unittest.TestCase):
|
|
106
|
-
def test_python_inexistente_da_erro_claro(self):
|
|
107
|
-
"""Sem python o usuario precisa saber o que instalar, nao ver stack trace."""
|
|
108
|
-
rc, out, err = roda("--config", "x.json",
|
|
109
|
-
env_extra={"AI_TEST_LOOP_PYTHON": "/caminho/que/nao/existe"})
|
|
110
|
-
self.assertNotEqual(rc, 0)
|
|
111
|
-
juntos = out + err
|
|
112
|
-
self.assertIn("Python 3", juntos)
|
|
113
|
-
self.assertIn("AI_TEST_LOOP_PYTHON", juntos)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
if __name__ == "__main__":
|
|
117
|
-
unittest.main(verbosity=2)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|